)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"da036bf2236dbf69e7b320b850ac07ff99fb5cec","unresolved":true,"context_lines":[{"line_number":4,"context_line":"Commit:     chandan kumar \u003cchkumar@redhat.com\u003e"},{"line_number":5,"context_line":"CommitDate: 2026-09-04 06:07:53 +0000"},{"line_number":6,"context_line":""},{"line_number":7,"context_line":"Add device_state bind guard and fix fake driver topology"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"When an ARQ is bound, ext_arq.bind() marks the whole Cyborg device as"},{"line_number":10,"context_line":"allocated. The bind path now rejects devices that are not AVAILABLE,"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":32,"id":"caf3ede5_530616a9","line":7,"range":{"start_line":7,"start_character":28,"end_line":7,"end_character":56},"updated":"2026-09-07 11:48:44.000000000","message":"the fake driver topltogy is not incorrect. more on that later","commit_id":"9b92cd31098fdb5ca2a3dec610a133186fa88295"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"da036bf2236dbf69e7b320b850ac07ff99fb5cec","unresolved":true,"context_lines":[{"line_number":6,"context_line":""},{"line_number":7,"context_line":"Add device_state bind guard and fix fake driver topology"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"When an ARQ is bound, ext_arq.bind() marks the whole Cyborg device as"},{"line_number":10,"context_line":"allocated. The bind path now rejects devices that are not AVAILABLE,"},{"line_number":11,"context_line":"rolls back attach handle, device_state, and placement reservation on"},{"line_number":12,"context_line":"failure, and init_host() backfills missing device_state rows while"},{"line_number":13,"context_line":"reconciling stale lifecycle states at startup."},{"line_number":14,"context_line":""},{"line_number":15,"context_line":"The fake driver still reported one device with 16 attach handles, so"},{"line_number":16,"context_line":"placement could schedule multiple VMs to the same device. The first bind"},{"line_number":17,"context_line":"succeeded, but a second VM on the same host failed with HTTP 409"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":32,"id":"d3debe98_96e25084","line":14,"range":{"start_line":9,"start_character":0,"end_line":14,"end_character":1},"updated":"2026-09-07 11:48:44.000000000","message":"the fake driver emulsted multiple device assocated with multipel atachmetn handels with 1 deployable that corrates to a single placement resouce provider with an inventory of N of the fpga resouce class\n\ntaht is a valid toplgoy","commit_id":"9b92cd31098fdb5ca2a3dec610a133186fa88295"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"da036bf2236dbf69e7b320b850ac07ff99fb5cec","unresolved":true,"context_lines":[{"line_number":16,"context_line":"placement could schedule multiple VMs to the same device. The first bind"},{"line_number":17,"context_line":"succeeded, but a second VM on the same host failed with HTTP 409"},{"line_number":18,"context_line":"(DeviceNotAvailable) in grenade/Tempest test_server_basic_ops. The"},{"line_number":19,"context_line":"driver now reports 16 separate devices, each with one deployable and one"},{"line_number":20,"context_line":"attach handle, and drops the old multi-handle helper that is no longer"},{"line_number":21,"context_line":"used."},{"line_number":22,"context_line":""},{"line_number":23,"context_line":"The first split device keeps the legacy FakeDevice deployable name so"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":32,"id":"3a56c0ae_29349e1e","line":20,"range":{"start_line":19,"start_character":39,"end_line":20,"end_character":14},"updated":"2026-09-07 11:48:44.000000000","message":"this is incorect.\n\nthe defice to deployable ration is many devices to one deployable and there is a 1:1 relasthipship betwen atchemnt handels and devices\n\nyou shoudl not change taht to a 1 device to one deployable \n\nthe many to one is correct and its a bug if the nvme code does not support that","commit_id":"9b92cd31098fdb5ca2a3dec610a133186fa88295"}],"cyborg/accelerator/drivers/fake.py":[{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"da036bf2236dbf69e7b320b850ac07ff99fb5cec","unresolved":true,"context_lines":[{"line_number":43,"context_line":"        return driver_cpid"},{"line_number":44,"context_line":""},{"line_number":45,"context_line":"    def _get_pci_slot(self, num_accelerator):"},{"line_number":46,"context_line":"        \"\"\"Return a unique PCI BDF for each fake device."},{"line_number":47,"context_line":""},{"line_number":48,"context_line":"        discover() now reports one device per accelerator instead of one"},{"line_number":49,"context_line":"        device with many attach handles. Each device needs its own PCI"},{"line_number":50,"context_line":"        slot so Cyborg can track device_state independently per bind."},{"line_number":51,"context_line":"        \"\"\""},{"line_number":52,"context_line":"        num_pci_fn_per_pci_device \u003d 8"},{"line_number":53,"context_line":"        pci_slot \u003d {"},{"line_number":54,"context_line":"            \u0027domain\u0027: \u00270000\u0027,"}],"source_content_type":"text/x-python","patch_set":32,"id":"335fdd4e_bbc8ef88","line":51,"range":{"start_line":46,"start_character":6,"end_line":51,"end_character":11},"updated":"2026-09-07 11:48:44.000000000","message":"frist of all you shoudl not be documenting what you cahnged temporally between the prior versonadn the current version like this in the docstring.\nThey shoudl alwasy be in present tense as evergreen statmes of current behvior.\n\nsecond while facotring this out into a helper fucntion is likely ok i dont see a reason why you shoudl remvoed _generate_attach_handles and inlien that into _generate_dep_list\n\nit was cleaner before and its not valid to assuem one attachment per deployable\n\nthe fake driver was enulating the correct toplogy for a PR with multipel VFs\n\nwe expect 1 deploragble per PF with an invetory of n VFs each with there own attachmetn handel\n\nyou can add supprot for the toplogy used by the nvme driver but you shoudl nto change the defautl FPGA topltogy that the fake driver is creating in general.","commit_id":"9b92cd31098fdb5ca2a3dec610a133186fa88295"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"da036bf2236dbf69e7b320b850ac07ff99fb5cec","unresolved":true,"context_lines":[{"line_number":69,"context_line":""},{"line_number":70,"context_line":"    def _generate_dep_list(self, pci):"},{"line_number":71,"context_line":"        driver_dep \u003d driver_deployable.DriverDeployable()"},{"line_number":72,"context_line":"        # One attach handle per deployable: ext_arq.bind() marks the whole"},{"line_number":73,"context_line":"        # device allocated, so multiple handles on one device would still"},{"line_number":74,"context_line":"        # allow only a single concurrent bind."},{"line_number":75,"context_line":"        driver_ah \u003d driver_attach_handle.DriverAttachHandle()"},{"line_number":76,"context_line":"        driver_ah.attach_type \u003d constants.AH_TYPE_TEST_PCI"},{"line_number":77,"context_line":"        driver_ah.in_use \u003d False"}],"source_content_type":"text/x-python","patch_set":32,"id":"de57e0e1_1700a65a","line":74,"range":{"start_line":72,"start_character":7,"end_line":74,"end_character":46},"updated":"2026-09-07 11:48:44.000000000","message":"markign a device shoudl incremetn reserved rather then settign resreved \u003d totoal\n\nfor the nvme driver there is a 1:1 correspondance but we cant assume that in general\n\nso you need to modify it to be smarter and check the number of inuse device and increment the placment value.\n\nthis is part of why i said thie really shoudl be manged by the compute agent as it can keep track fo the amount of free/used devices in memory vai the resouce tracker so it can caluate the correct value without haveign to count the inuse devices for a given deployable via a db query using the resouce tracker.\n but that is the correct thing to do in the api version.\n\nbut im not sure why you are seeign this affect the fake driver at all\n\nthe fake drver presents FPGAs not nvme devices\n\nthe call to set_nvme_placement_reserved\nhttps://review.opendev.org/c/openstack/cyborg/+/999939/25/cyborg/api/controllers/v2/arqs.py#452\n\nmay be unconditional btu it internally skips for non nvme devices\n\nhttps://review.opendev.org/c/openstack/cyborg/+/999939/25/cyborg/common/placement_client.py#454","commit_id":"9b92cd31098fdb5ca2a3dec610a133186fa88295"}],"cyborg/agent/manager.py":[{"author":{"_account_id":28006,"name":"teim-ci","display_name":"teim-ci","email":"ci@seanmooney.info","username":"ci-sean-mooney","status":"this is a third-party ci account run by sean-k-mooney on irc\nhosted at zuul.teim.app"},"tag":"autogenerated:zuul:automatic-ci","change_message_id":"b4abf22f398c1192a587eb17f975f00bd7f700d4","unresolved":false,"context_lines":[{"line_number":136,"context_line":"        drv \u003d self._rt.get_driver_for_type(device.type)"},{"line_number":137,"context_line":"        drv.cleanup(device)"},{"line_number":138,"context_line":""},{"line_number":139,"context_line":"    def _reconcile_device_states(self):"},{"line_number":140,"context_line":"        context \u003d cyborg_context.get_admin_context()"},{"line_number":141,"context_line":"        devices \u003d objects.Device.get_list_by_hostname("},{"line_number":142,"context_line":"            context,"}],"source_content_type":"text/x-python","patch_set":5,"id":"2f27c529_8ce636cb","line":139,"updated":"2026-08-06 17:16:53.000000000","message":"The new _reconcile_device_states method performs multiple DB queries (Device.get_list_by_hostname, Deployable.get_list_by_device_id, AttachHandle.get_ah_list_by_deployable_id) and save operations without any try/except. If any DB operation fails, the exception propagates through init_host() and p...\n\n**Severity**: WARNING | **Confidence**: 0.8\n\n**Impact**: A transient DB connectivity issue during agent startup causes the agent to fail to start entirely, with no self-healing. In production, this means the compute node\u0027s accelerator management is offline until manual intervention.\n\n**Suggestion**:\nWrap the body of _reconcile_device_states in a try/except that logs the exception and allows init_host to complete, so the agent can start and retry reconciliation later. Follow the pattern used in conductor/manager.py init_host.","commit_id":"489f005f20718c5aefe514802d25c711b39cb55f"},{"author":{"_account_id":28006,"name":"teim-ci","display_name":"teim-ci","email":"ci@seanmooney.info","username":"ci-sean-mooney","status":"this is a third-party ci account run by sean-k-mooney on irc\nhosted at zuul.teim.app"},"tag":"autogenerated:zuul:automatic-ci","change_message_id":"a279a661ed4541a8a59e4348d387a41a55127a19","unresolved":false,"context_lines":[{"line_number":194,"context_line":"                        device.uuid,"},{"line_number":195,"context_line":"                    )"},{"line_number":196,"context_line":"                    if device.supports_cleaning:"},{"line_number":197,"context_line":"                        self.cleanup_device(context, device)"},{"line_number":198,"context_line":"                    else:"},{"line_number":199,"context_line":"                        device.device_state \u003d constants.DEVICE_STATE_AVAILABLE"},{"line_number":200,"context_line":"                        device.save(context)"}],"source_content_type":"text/x-python","patch_set":7,"id":"b2f4843a_917c4ec2","line":197,"updated":"2026-08-07 16:56:31.000000000","message":"During init_host(), _do_reconcile_device_states calls self.cleanup_device(context, device) inline for each device in ALLOCATED state with no active ARQs. cleanup_device submits to a thread pool but then calls future.result(timeout\u003dCONF.nvme.cleanup_timeout), blocking for up to the cleanup timeout...\n\n**Severity**: WARNING | **Confidence**: 0.8\n\n**Impact**: Agent startup could be blocked for minutes if multiple devices need cleanup or if cleanup operations are slow. This delays the agent from accepting RPC calls, potentially causing timeouts in conductor communication.\n\n**Suggestion**:\nSubmit cleanup_device asynchronously (fire-and-forget) during reconciliation, or process cleanup in the background after init_host completes. If synchronous cleanup is intentional, add a LOG.warning before each device cleanup so operators can diagnose slow startup.","commit_id":"d32c1d46e96368254925b7b78b204a15747229a4"},{"author":{"_account_id":28006,"name":"teim-ci","display_name":"teim-ci","email":"ci@seanmooney.info","username":"ci-sean-mooney","status":"this is a third-party ci account run by sean-k-mooney on irc\nhosted at zuul.teim.app"},"tag":"autogenerated:zuul:automatic-ci","change_message_id":"400236530b0ba26cd12a4a88bacab617f0abc9af","unresolved":false,"context_lines":[{"line_number":195,"context_line":"                        device.uuid,"},{"line_number":196,"context_line":"                    )"},{"line_number":197,"context_line":"                    if device.supports_cleaning:"},{"line_number":198,"context_line":"                        self.cleanup_device(context, device)"},{"line_number":199,"context_line":"                    else:"},{"line_number":200,"context_line":"                        device.device_state \u003d constants.DEVICE_STATE_AVAILABLE"},{"line_number":201,"context_line":"                        device.save(context)"}],"source_content_type":"text/x-python","patch_set":8,"id":"1f78239f_37c6373f","line":198,"updated":"2026-08-09 05:46:24.000000000","message":"_do_reconcile_device_states calls self.cleanup_device(context, device) for each ALLOCATED device with no active ARQs that supports cleaning. cleanup_device submits work to a thread executor and then blocks on future.result(timeout\u003dCONF.nvme.cleanup_timeout). NVMe cleanup operations (sanitize, wri...\n\n**Severity**: WARNING | **Confidence**: 0.8\n\n**Impact**: After an agent crash with multiple NVMe devices left in ALLOCATED state, the agent restart could be blocked for a long time (cumulative cleanup_timeout across all devices), delaying service availability.\n\n**Suggestion**:\nConsider submitting cleanup_device tasks asynchronously for all affected devices and collecting results, or document the expected startup delay in the release note so operators can set expectations. Alternatively, set device state to CLEANING and submit cleanup without blocking, relying on the next restart to verify completion.","commit_id":"aa365ba4e164ea5ba2cde6123bd0b29840c31270"},{"author":{"_account_id":28006,"name":"teim-ci","display_name":"teim-ci","email":"ci@seanmooney.info","username":"ci-sean-mooney","status":"this is a third-party ci account run by sean-k-mooney on irc\nhosted at zuul.teim.app"},"tag":"autogenerated:zuul:automatic-ci","change_message_id":"1548399c5773e0b159af5038a88af275c2793227","unresolved":false,"context_lines":[{"line_number":169,"context_line":"            context,"},{"line_number":170,"context_line":"            self.resource_provider_name,"},{"line_number":171,"context_line":"        )"},{"line_number":172,"context_line":"        for device in devices:"},{"line_number":173,"context_line":"            if device.device_state in ("},{"line_number":174,"context_line":"                constants.DEVICE_STATE_CLEANING,"},{"line_number":175,"context_line":"                constants.DEVICE_STATE_PENDING_CLEANING,"}],"source_content_type":"text/x-python","patch_set":9,"id":"e896994b_8aa43f89","line":172,"updated":"2026-08-11 10:06:05.000000000","message":"The `_do_reconcile_device_states` loop iterates over all devices but has no per-device try/except. If processing any device raises (e.g., device.save DB error, query failure), the exception propagates to the outer `except Exception` in `_reconcile_device_states`, which logs and swallows it. All r...\n\n**Severity**: WARNING | **Confidence**: 0.8\n\n**Impact**: If a transient DB error or unexpected exception occurs while processing device 2 of 10, devices 3-10 remain in stale states (CLEANING, ALLOCATED) until the next full agent restart. For devices stuck in ALLOCATED, this means they cannot accept new binds and are effectively stranded.\n\n**Suggestion**:\nWrap the per-device processing body inside the for-loop with a try/except so that one device failure logs an error and continues to the next device, rather than aborting the entire reconciliation pass.","commit_id":"80ebe043e9f22e5e91805f3e8ab8e0f892de066e"},{"author":{"_account_id":28006,"name":"teim-ci","display_name":"teim-ci","email":"ci@seanmooney.info","username":"ci-sean-mooney","status":"this is a third-party ci account run by sean-k-mooney on irc\nhosted at zuul.teim.app"},"tag":"autogenerated:zuul:automatic-ci","change_message_id":"e3d961254ccfea871042d060af7c6b2e15ceecb1","unresolved":false,"context_lines":[{"line_number":154,"context_line":"                device.uuid,"},{"line_number":155,"context_line":"            )"},{"line_number":156,"context_line":""},{"line_number":157,"context_line":"    def _reconcile_device_states(self):"},{"line_number":158,"context_line":"        try:"},{"line_number":159,"context_line":"            self._do_reconcile_device_states()"},{"line_number":160,"context_line":"        except Exception:"}],"source_content_type":"text/x-python","patch_set":10,"id":"e08874fb_1f7852d9","line":157,"updated":"2026-08-11 16:14:09.000000000","message":"The _reconcile_device_states() method wraps _do_reconcile_device_states() in a try/except that catches all exceptions and logs them, allowing the agent to continue starting. No test verifies this behavior — if _do_reconcile_device_states raises, the catch-and-log path is unexercised.\n\n**Severity**: SUGGESTION | **Confidence**: 0.8\n\n**Benefit**: If the exception handling in _reconcile_device_states is broken or removed accidentally, no test will catch it. The agent could crash on startup when reconciliation fails, defeating the purpose of the graceful degradation design.\n\n**Recommendation**:\nAdd a test that patches _do_reconcile_device_states to raise a known exception, calls _reconcile_device_states, and asserts no exception propagates and the expected log message is emitted.","commit_id":"9f298f81dea27546ba8d442cba1d6cf343f4fbc2"},{"author":{"_account_id":28006,"name":"teim-ci","display_name":"teim-ci","email":"ci@seanmooney.info","username":"ci-sean-mooney","status":"this is a third-party ci account run by sean-k-mooney on irc\nhosted at zuul.teim.app"},"tag":"autogenerated:zuul:automatic-ci","change_message_id":"28f4e10abc29550f555a2a7611c69e62c1993988","unresolved":false,"context_lines":[{"line_number":227,"context_line":"                        device.uuid,"},{"line_number":228,"context_line":"                    )"},{"line_number":229,"context_line":"                    if device.supports_cleaning:"},{"line_number":230,"context_line":"                        device.device_state \u003d ("},{"line_number":231,"context_line":"                            constants.DEVICE_STATE_PENDING_CLEANING"},{"line_number":232,"context_line":"                        )"},{"line_number":233,"context_line":"                        device.save(context)"}],"source_content_type":"text/x-python","patch_set":14,"id":"6feefc09_8423a18f","line":230,"updated":"2026-08-13 14:52:40.000000000","message":"In _do_reconcile_device_states, when an ALLOCATED device with no ARQs supports cleaning, the code sets device_state to PENDING_CLEANING (line 230), saves (line 233), then calls cleanup_device (line 234). cleanup_device immediately sets device_state to CLEANING (line 135) and saves again, making t...\n\n**Severity**: SUGGESTION | **Confidence**: 0.9\n\n**Benefit**: The extra DB save is wasteful and the PENDING_CLEANING state is never observable by any consumer. It also makes the code harder to reason about because it implies a meaningful intermediate state.\n\n**Recommendation**:\nRemove the PENDING_CLEANING intermediate save since cleanup_device will immediately overwrite it to CLEANING, or adjust cleanup_device to accept the device in PENDING_CLEANING without re-saving.","commit_id":"9ee4eb82b7f716e7b143b044691f0311d4b89fe4"},{"author":{"_account_id":28006,"name":"teim-ci","display_name":"teim-ci","email":"ci@seanmooney.info","username":"ci-sean-mooney","status":"this is a third-party ci account run by sean-k-mooney on irc\nhosted at zuul.teim.app"},"tag":"autogenerated:zuul:automatic-ci","change_message_id":"086caf06cad3a1e20079dda408771e484ae6d831","unresolved":false,"context_lines":[{"line_number":110,"context_line":"        self._cleanup_executor.shutdown(wait\u003dTrue)"},{"line_number":111,"context_line":""},{"line_number":112,"context_line":"    def init_host(self):"},{"line_number":113,"context_line":"        \"\"\"Hook called by RPCService.start() after the RPC server is up.\"\"\""},{"line_number":114,"context_line":"        try:"},{"line_number":115,"context_line":"            while True:"},{"line_number":116,"context_line":"                found, done \u003d data_migrations.backfill_device_state()"}],"source_content_type":"text/x-python","patch_set":16,"id":"dab5d221_70214a8a","line":113,"updated":"2026-08-15 17:10:38.000000000","message":"The loop repeats while found is truthy, but the only exit besides found\u003d\u003d0 relies on backfill_device_state making progress. If device_update fails for one device (or a device row is re-selected without its state changing), each batch returns found\u003e0 with done\u003d\u003d0 and init_host spins forever, block...\n\n**Severity**: HIGH | **Confidence**: 0.8\n\n**Risk**: A single un-updatable device row (DB constraint, stale replica read) prevents cyborg-agent from ever completing init_host, taking the compute agent down with no timeout.\n\n**Priority**: Before merge\n**Why This Matters**: A single un-updatable device row (DB constraint, stale replica read) prevents cyborg-agent from ever completing init_host, taking the compute agent down with no timeout.\n\n**Recommendation**:\nBreak out of the loop when done \u003d\u003d 0 as well (log an error directing the operator to cyborg-dbsync), matching the progress-check pattern used by the conductor loop.","commit_id":"911a10a7ff7ea2d5f062199c37f1ef4633c926c7"},{"author":{"_account_id":28006,"name":"teim-ci","display_name":"teim-ci","email":"ci@seanmooney.info","username":"ci-sean-mooney","status":"this is a third-party ci account run by sean-k-mooney on irc\nhosted at zuul.teim.app"},"tag":"autogenerated:zuul:automatic-ci","change_message_id":"086caf06cad3a1e20079dda408771e484ae6d831","unresolved":false,"context_lines":[{"line_number":223,"context_line":"            elif device.device_state \u003d\u003d constants.DEVICE_STATE_ALLOCATED:"},{"line_number":224,"context_line":"                if not self._has_active_arqs(context, device):"},{"line_number":225,"context_line":"                    LOG.warning("},{"line_number":226,"context_line":"                        \u0027Device %s allocated with no ARQs, triggering cleanup\u0027,"},{"line_number":227,"context_line":"                        device.uuid,"},{"line_number":228,"context_line":"                    )"},{"line_number":229,"context_line":"                    if device.supports_cleaning:"}],"source_content_type":"text/x-python","patch_set":16,"id":"d3012f51_ee85a4d3","line":226,"updated":"2026-08-15 17:10:38.000000000","message":"_do_reconcile_device_states() treats every ALLOCATED device with no in-use attach handles as needing cleanup and calls cleanup_device() for NVMe devices, which submits a sanitize/write-zeroes/destroy operation via the driver. The only evidence is the DB rows present at agent restart, with no oper...\n\n**Severity**: HIGH | **Confidence**: 0.8\n\n**Risk**: Any condition that leaves device_state\u003dallocated with attach handles marked not in_use while data is still resident (partial unbind, failed deallocate, conductor crash between ARQ unbind and AH update, or a brand-new NVMe device imported as allocated) results in an automatic destructive sanitize/...\n\n**Priority**: Before merge\n**Why This Matters**: Any condition that leaves device_state\u003dallocated with attach handles marked not in_use while data is still resident (partial unbind, failed deallocate, conductor crash between ARQ unbind and AH update, or a brand-new NVMe device imported as allocated) results in an automatic destructive sanitize/...\n\n**Recommendation**:\nDo not auto-run destructive cleanup at startup from DB inference alone: move such devices to PENDING_CLEANING/error and surface them for operator action (or require an opt-in config like [nvme] auto_cleanup_on_startup), and/or verify device contents/namespace state before invoking drv.cleanup(). Distinguish NVMe-owned pool devices from operator data by owner/trait checks.","commit_id":"911a10a7ff7ea2d5f062199c37f1ef4633c926c7"},{"author":{"_account_id":28006,"name":"teim-ci","display_name":"teim-ci","email":"ci@seanmooney.info","username":"ci-sean-mooney","status":"this is a third-party ci account run by sean-k-mooney on irc\nhosted at zuul.teim.app"},"tag":"autogenerated:zuul:automatic-ci","change_message_id":"f21af27a0fa4cc660f97b95b3dff5380e0e99091","unresolved":false,"context_lines":[{"line_number":110,"context_line":"        self._cleanup_executor.shutdown(wait\u003dTrue)"},{"line_number":111,"context_line":""},{"line_number":112,"context_line":"    def init_host(self):"},{"line_number":113,"context_line":"        \"\"\"Hook called by RPCService.start() after the RPC server is up.\"\"\""},{"line_number":114,"context_line":"        try:"},{"line_number":115,"context_line":"            while True:"},{"line_number":116,"context_line":"                found, done \u003d data_migrations.backfill_device_state()"}],"source_content_type":"text/x-python","patch_set":17,"id":"f18e5071_d5bc0ada","line":113,"updated":"2026-08-17 08:39:23.000000000","message":"The new while-True backfill loop in AgentManager.init_host() calls data_migrations.backfill_device_state(), which scans devices globally (no hostname filter). ConductorManager.init_host() already runs the identical loop, and every cyborg-agent on every host will now run it too, concurrently mutat...\n\n**Severity**: WARNING | **Confidence**: 0.8\n\n**Impact**: N agents + conductor each perform repeated full-table scans and blind updates of the same rows on every restart; concurrent agents can race on in-use attach handle state and record inconsistent device_state values (one writes \u0027allocated\u0027 from a stale read). Startup latency and DB churn grow with...\n\n**Suggestion**:\nDrop the backfill from AgentManager.init_host() and rely on the conductor (single service, already running the same migration), or scope the agent-side query to this host\u0027s devices (hostname filter) so each agent only heals its own rows.","commit_id":"fc11b6660359304e8e52503b84feed015ec1b128"},{"author":{"_account_id":28006,"name":"teim-ci","display_name":"teim-ci","email":"ci@seanmooney.info","username":"ci-sean-mooney","status":"this is a third-party ci account run by sean-k-mooney on irc\nhosted at zuul.teim.app"},"tag":"autogenerated:zuul:automatic-ci","change_message_id":"f21af27a0fa4cc660f97b95b3dff5380e0e99091","unresolved":false,"context_lines":[{"line_number":199,"context_line":"                \u0027without reconciliation and retry on next restart\u0027"},{"line_number":200,"context_line":"            )"},{"line_number":201,"context_line":""},{"line_number":202,"context_line":"    def _do_reconcile_device_states(self):"},{"line_number":203,"context_line":"        context \u003d cyborg_context.get_admin_context()"},{"line_number":204,"context_line":"        devices \u003d objects.Device.get_list_by_hostname("},{"line_number":205,"context_line":"            context,"}],"source_content_type":"text/x-python","patch_set":17,"id":"860ced18_596b13af","line":202,"updated":"2026-08-17 08:39:23.000000000","message":"_do_reconcile_device_states() iterates devices with no per-device error handling; any exception from device.save(), _set_nvme_placement_reserved(), _has_active_arqs(), or cleanup_device() escapes the loop to the blanket except in _reconcile_device_states(), so devices after the failing one are ne...\n\n**Severity**: WARNING | **Confidence**: 0.8\n\n**Impact**: One bad or locked device row (e.g. transient DB error on save, or a placement client failure for a single device) silently leaves all subsequent devices on the host in stale cleaning/allocated states, defeating the commit\u0027s stated goal of reconciling stale lifecycle states at startup.\n\n**Suggestion**:\nMove the try/except inside the for-loop (log and continue per device), so a single device failure does not abort reconciliation for the rest of the host\u0027s devices; keep the outer handler for get_list_by_hostname failures.","commit_id":"fc11b6660359304e8e52503b84feed015ec1b128"},{"author":{"_account_id":28006,"name":"teim-ci","display_name":"teim-ci","email":"ci@seanmooney.info","username":"ci-sean-mooney","status":"this is a third-party ci account run by sean-k-mooney on irc\nhosted at zuul.teim.app"},"tag":"autogenerated:zuul:automatic-ci","change_message_id":"cdf9cfb89e55ab4a2a0a9989b2d6c7a56a5a0c26","unresolved":false,"context_lines":[{"line_number":113,"context_line":"    def init_host(self):"},{"line_number":114,"context_line":"        \"\"\"Hook called by RPCService.start() after the RPC server is up.\"\"\""},{"line_number":115,"context_line":"        try:"},{"line_number":116,"context_line":"            while True:"},{"line_number":117,"context_line":"                found, done \u003d data_migrations.backfill_device_state()"},{"line_number":118,"context_line":"                if not found:"},{"line_number":119,"context_line":"                    break"}],"source_content_type":"text/x-python","patch_set":32,"id":"5b98188c_54a6457f","line":116,"updated":"2026-09-04 07:15:15.000000000","message":"init_host() wraps data_migrations.backfill_device_state() in \u0027while True\u0027, breaking only when found \u003d\u003d 0. But backfill_device_state already loops internally and deliberately stops with (total_found \u003e 0, total_done \u003d\u003d 0) when a batch makes no progress (data_migrations.py:196-205: \u0027No progress backfilling device_state ... stopping\u0027). If the failing condition persists (e.g. a row that cannot be updated), every subsequent call returns found \u003e 0 again, so the outer loop never exits and RPCService.start() never completes: the agent process hangs at startup with no bound RPC endpoints.\n\n**Severity**: WARNING | **Confidence**: 0.85\n\n**Impact**: A persistent failure to update even one device_state row turns agent startup into an infinite loop; the compute node\u0027s agent never registers, disabling ARQ bind/cleanup for that host until an operator intervenes.\n\n**Suggestion**:\nCall backfill_device_state() once (it already batches internally) and log a warning when done \u003c found, or continue looping only while done \u003e 0 (e.g. \u0027if not done: break\u0027). Add a unit test for the found\u003e0, done\u003d\u003d0 return to pin the behavior.","commit_id":"9b92cd31098fdb5ca2a3dec610a133186fa88295"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"da036bf2236dbf69e7b320b850ac07ff99fb5cec","unresolved":true,"context_lines":[{"line_number":113,"context_line":"    def init_host(self):"},{"line_number":114,"context_line":"        \"\"\"Hook called by RPCService.start() after the RPC server is up.\"\"\""},{"line_number":115,"context_line":"        try:"},{"line_number":116,"context_line":"            while True:"},{"line_number":117,"context_line":"                found, done \u003d data_migrations.backfill_device_state()"},{"line_number":118,"context_line":"                if not found:"},{"line_number":119,"context_line":"                    break"}],"source_content_type":"text/x-python","patch_set":32,"id":"435021c1_6d7849f9","line":116,"in_reply_to":"5b98188c_54a6457f","updated":"2026-09-07 11:48:44.000000000","message":"ya this was not updated for the change we made earlier in teh seies the while is incorrect and should be removed\n\nbut there is a bigger problem\n\nthe cyborg compute agent shoudl not have db access directly\nand cyborg does not currentl implemnt the ovo indreiction api","commit_id":"9b92cd31098fdb5ca2a3dec610a133186fa88295"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"da036bf2236dbf69e7b320b850ac07ff99fb5cec","unresolved":true,"context_lines":[{"line_number":130,"context_line":"        self._rt.init_drivers()"},{"line_number":131,"context_line":"        all_devices \u003d self._rt.discover_all()"},{"line_number":132,"context_line":"        self._rt.check_duplicate_devices(all_devices)"},{"line_number":133,"context_line":"        self._reconcile_device_states()"},{"line_number":134,"context_line":""},{"line_number":135,"context_line":"    def cleanup_device(self, context, device):"},{"line_number":136,"context_line":"        \"\"\"Run driver cleanup synchronously from the RPC handler\u0027s perspective."}],"source_content_type":"text/x-python","patch_set":32,"id":"13f9977d_be233c2a","line":133,"range":{"start_line":133,"start_character":13,"end_line":133,"end_character":37},"updated":"2026-09-07 11:48:44.000000000","message":"isnt this just duplciating  data_migrations.backfill_device_state() for the most part.","commit_id":"9b92cd31098fdb5ca2a3dec610a133186fa88295"}],"cyborg/agent/resource_tracker.py":[{"author":{"_account_id":28006,"name":"teim-ci","display_name":"teim-ci","email":"ci@seanmooney.info","username":"ci-sean-mooney","status":"this is a third-party ci account run by sean-k-mooney on irc\nhosted at zuul.teim.app"},"tag":"autogenerated:zuul:automatic-ci","change_message_id":"1548399c5773e0b159af5038a88af275c2793227","unresolved":false,"context_lines":[{"line_number":89,"context_line":"                    )"},{"line_number":90,"context_line":"                seen[addr] \u003d name"},{"line_number":91,"context_line":""},{"line_number":92,"context_line":"    TYPE_TO_DRIVER \u003d {"},{"line_number":93,"context_line":"        constants.DEVICE_NVME: \u0027nvme_driver\u0027,"},{"line_number":94,"context_line":"    }"},{"line_number":95,"context_line":""}],"source_content_type":"text/x-python","patch_set":9,"id":"9b156410_70ab9a73","line":92,"updated":"2026-08-11 10:06:05.000000000","message":"The new `get_driver_for_type()` method on ResourceTracker has branching logic: it looks up a driver name in TYPE_TO_DRIVER, checks whether that driver exists in acc_drivers, returns it on success, or raises InvalidDriver on two separate failure paths. None of these branches are exercised by unit...\n\n**Severity**: SUGGESTION | **Confidence**: 0.8\n\n**Benefit**: Future changes to the driver lookup logic (e.g., adding a new device type to TYPE_TO_DRIVER) have no test coverage to catch regressions in the mapping or error paths.\n\n**Recommendation**:\nAdd unit tests covering: (1) successful lookup for a registered type, (2) InvalidDriver raised for an unregistered device type, (3) InvalidDriver raised when the mapped driver name is not present in acc_drivers.","commit_id":"80ebe043e9f22e5e91805f3e8ab8e0f892de066e"},{"author":{"_account_id":28006,"name":"teim-ci","display_name":"teim-ci","email":"ci@seanmooney.info","username":"ci-sean-mooney","status":"this is a third-party ci account run by sean-k-mooney on irc\nhosted at zuul.teim.app"},"tag":"autogenerated:zuul:automatic-ci","change_message_id":"352193e6789fbbbe210f4b5daad1d6c0373562ea","unresolved":false,"context_lines":[{"line_number":33,"context_line":"LOG \u003d logging.getLogger(__name__)"},{"line_number":34,"context_line":""},{"line_number":35,"context_line":"AGENT_RESOURCE_SEMAPHORE \u003d \"agent_resources\""},{"line_number":36,"context_line":"TYPE_TO_DRIVER \u003d {"},{"line_number":37,"context_line":"    constants.DEVICE_NVME: \u0027nvme_driver\u0027,"},{"line_number":38,"context_line":"}"},{"line_number":39,"context_line":""}],"source_content_type":"text/x-python","patch_set":19,"id":"e8da7afe_025ba588","line":36,"updated":"2026-08-21 12:23:44.000000000","message":"The change adds a class-level TYPE_TO_DRIVER to ResourceTracker and rewrites get_driver_for_type to use it, but leaves the identical module-level TYPE_TO_DRIVER dict in place. The device-type-to-driver rule now has two independent copies that can diverge, and edits to the module-level constant will silently have no effect.\n\n**Severity**: SUGGESTION | **Confidence**: 0.85\n\n**Impact**: A maintainer updating the module-level mapping (the previously authoritative name) would see no behavior change, a divergent-invariant maintenance risk for the device-type-to-driver rule.\n\n**Recommendation**:\nDelete the module-level TYPE_TO_DRIVER (it has no remaining users) or keep only the module-level constant and reference it from get_driver_for_type, so the rule has a single owner.","commit_id":"d2767238f4909b2dd88ce36590b780476e9c3889"},{"author":{"_account_id":28006,"name":"teim-ci","display_name":"teim-ci","email":"ci@seanmooney.info","username":"ci-sean-mooney","status":"this is a third-party ci account run by sean-k-mooney on irc\nhosted at zuul.teim.app"},"tag":"autogenerated:zuul:automatic-ci","change_message_id":"e59f04ba23d15b289ba6f27f907293430ad476b5","unresolved":false,"context_lines":[{"line_number":101,"context_line":"                )"},{"line_number":102,"context_line":"            seen[addr] \u003d driver_name"},{"line_number":103,"context_line":""},{"line_number":104,"context_line":"    TYPE_TO_DRIVER \u003d {"},{"line_number":105,"context_line":"        constants.DEVICE_NVME: \u0027nvme_driver\u0027,"},{"line_number":106,"context_line":"    }"},{"line_number":107,"context_line":""}],"source_content_type":"text/x-python","patch_set":28,"id":"2e06176e_75d33a25","line":104,"updated":"2026-08-31 11:27:16.000000000","message":"resource_tracker.py keeps the module-level TYPE_TO_DRIVER at line 36 (now referenced by nothing in production code) and adds an identical ResourceTracker.TYPE_TO_DRIVER class attribute at line 104 that get_driver_for_type actually uses. The same device-type-to-cleanup-driver rule now has two definitions that can be edited independently; adding a new cleanup-capable device type to one but not the other silently yields the old or new behavior depending on which copy a future change touches.\n\n**Severity**: SUGGESTION | **Confidence**: 0.9\n\n**Impact**: Divergent maintenance risk: a future device-type addition or driver rename applied to the module constant (which still looks like the shared table) would have no effect on get_driver_for_type, causing cleanup to keep raising InvalidDriver for a supported type.\n\n**Recommendation**:\nDelete the module-level TYPE_TO_DRIVER at resource_tracker.py:36 and keep only the class attribute (or keep the module constant and reference it directly); ensure the single remaining definition is the one tests patch or assert against.","commit_id":"e78df01a78803123a1ffa985c0a6cbe67cee5130"},{"author":{"_account_id":28006,"name":"teim-ci","display_name":"teim-ci","email":"ci@seanmooney.info","username":"ci-sean-mooney","status":"this is a third-party ci account run by sean-k-mooney on irc\nhosted at zuul.teim.app"},"tag":"autogenerated:zuul:automatic-ci","change_message_id":"cdf9cfb89e55ab4a2a0a9989b2d6c7a56a5a0c26","unresolved":false,"context_lines":[{"line_number":101,"context_line":"                )"},{"line_number":102,"context_line":"            seen[addr] \u003d driver_name"},{"line_number":103,"context_line":""},{"line_number":104,"context_line":"    TYPE_TO_DRIVER \u003d {"},{"line_number":105,"context_line":"        constants.DEVICE_NVME: \u0027nvme_driver\u0027,"},{"line_number":106,"context_line":"    }"},{"line_number":107,"context_line":""}],"source_content_type":"text/x-python","patch_set":32,"id":"ee5dc2b6_2787668b","line":104,"updated":"2026-09-04 07:15:15.000000000","message":"This commit adds a class-level TYPE_TO_DRIVER mapping to ResourceTracker (line 104) used by get_driver_for_type(), while the pre-existing module-level TYPE_TO_DRIVER (line 36) remains in the same file and is now unreferenced. The device-type-to-cleanup-driver rule now has two independent definitions in one module; a future edit to one copy (e.g. adding a QAT cleanup driver) would silently not take effect if the wrong copy is changed, and readers cannot tell which copy is authoritative.\n\n**Severity**: SUGGESTION | **Confidence**: 0.9\n\n**Impact**: Divergence between the two copies would make get_driver_for_type() fail or pick a stale driver for a device type, and the dead module-level constant misleads contributors editing the file.\n\n**Recommendation**:\nDelete the now-unused module-level TYPE_TO_DRIVER (or keep only it and reference it from get_driver_for_type) so the mapping has a single owner.","commit_id":"9b92cd31098fdb5ca2a3dec610a133186fa88295"}],"cyborg/common/exception.py":[{"author":{"_account_id":28006,"name":"teim-ci","display_name":"teim-ci","email":"ci@seanmooney.info","username":"ci-sean-mooney","status":"this is a third-party ci account run by sean-k-mooney on irc\nhosted at zuul.teim.app"},"tag":"autogenerated:zuul:automatic-ci","change_message_id":"400236530b0ba26cd12a4a88bacab617f0abc9af","unresolved":false,"context_lines":[{"line_number":449,"context_line":"    _msg_fmt \u003d _(\"Device cleanup failed: %(reason)s.\")"},{"line_number":450,"context_line":""},{"line_number":451,"context_line":""},{"line_number":452,"context_line":"class DeviceNotAvailable(CyborgException):"},{"line_number":453,"context_line":"    _msg_fmt \u003d _(\"Device %(device)s is not available (state\u003d%(state)s).\")"},{"line_number":454,"context_line":""},{"line_number":455,"context_line":""}],"source_content_type":"text/x-python","patch_set":8,"id":"cbde4799_188de315","line":452,"updated":"2026-08-09 05:46:24.000000000","message":"The commit message explicitly states that raising DeviceNotAvailable gives \u0027a clear 409 response.\u0027 However, DeviceNotAvailable inherits from CyborgException whose default code is HTTPStatus.INTERNAL_SERVER_ERROR (500). The codebase already provides a Conflict exception with code \u003d HTTPStatus.CONF...\n\n**Severity**: HIGH | **Confidence**: 0.9\n\n**Risk**: API clients (including Nova) will receive HTTP 500 instead of HTTP 409 when binding to an unavailable device. This prevents clients from distinguishing a transient server error from a legitimate resource conflict, and contradicts the documented behavior.\n\n**Priority**: Before merge\n**Why This Matters**: API clients (including Nova) will receive HTTP 500 instead of HTTP 409 when binding to an unavailable device. This prevents clients from distinguishing a transient server error from a legitimate resource conflict, and contradicts the documented behavior.\n\n**Recommendation**:\nEither change the parent class to Conflict (which sets code \u003d HTTPStatus.CONFLICT) or add \u0027code \u003d HTTPStatus.CONFLICT\u0027 to DeviceNotAvailable. Example: class DeviceNotAvailable(Conflict):","commit_id":"aa365ba4e164ea5ba2cde6123bd0b29840c31270"}],"cyborg/objects/ext_arq.py":[{"author":{"_account_id":28006,"name":"teim-ci","display_name":"teim-ci","email":"ci@seanmooney.info","username":"ci-sean-mooney","status":"this is a third-party ci account run by sean-k-mooney on irc\nhosted at zuul.teim.app"},"tag":"autogenerated:zuul:automatic-ci","change_message_id":"1c42ac4ef4832d77a7af971f82ef2dbe19eb0e7b","unresolved":false,"context_lines":[{"line_number":275,"context_line":"        )"},{"line_number":276,"context_line":""},{"line_number":277,"context_line":"    def bind(self, context, deployable):"},{"line_number":278,"context_line":"        device \u003d objects.Device.get_by_device_id(context, deployable.device_id)"},{"line_number":279,"context_line":"        if ("},{"line_number":280,"context_line":"            device.device_state is not None"},{"line_number":281,"context_line":"            and device.device_state !\u003d constants.DEVICE_STATE_AVAILABLE"}],"source_content_type":"text/x-python","patch_set":2,"id":"96af9652_f7d27a99","line":278,"updated":"2026-08-06 11:26:24.000000000","message":"The new bind guard raises DeviceNotAvailable before _allocate_attach_handle() is reached. The only code that transitions the ARQ to BIND_FAILED lives inside _allocate_attach_handle\u0027s except block (line 269). As a result, when the guard fires, the ARQ state is never updated to BIND_FAILED and rema...\n\n**Severity**: HIGH | **Confidence**: 0.8\n\n**Risk**: An ARQ whose target device is cleaning or errored stays in BIND_STARTED indefinitely. Nova, which expects a terminal state (Bound or BindFailed), may hang waiting for the ARQ to resolve. The ARQ is not usable until manually fixed or deleted.\n\n**Priority**: Before merge\n**Why This Matters**: An ARQ whose target device is cleaning or errored stays in BIND_STARTED indefinitely. Nova, which expects a terminal state (Bound or BindFailed), may hang waiting for the ARQ to resolve. The ARQ is not usable until manually fixed or deleted.\n\n**Recommendation**:\nBefore raising DeviceNotAvailable, call self.update_check_state(context, constants.ARQ_BIND_FAILED) to transition the ARQ to a terminal failure state, or wrap the guard so the exception is caught by the same error-handling path as _allocate_attach_handle failures.","commit_id":"747be871aa6f1c3b461784ac663dea1dfece0da2"},{"author":{"_account_id":28006,"name":"teim-ci","display_name":"teim-ci","email":"ci@seanmooney.info","username":"ci-sean-mooney","status":"this is a third-party ci account run by sean-k-mooney on irc\nhosted at zuul.teim.app"},"tag":"autogenerated:zuul:automatic-ci","change_message_id":"1c42ac4ef4832d77a7af971f82ef2dbe19eb0e7b","unresolved":false,"context_lines":[{"line_number":285,"context_line":"                state\u003ddevice.device_state,"},{"line_number":286,"context_line":"            )"},{"line_number":287,"context_line":""},{"line_number":288,"context_line":"        if device.device_state is not None:"},{"line_number":289,"context_line":"            device.device_state \u003d constants.DEVICE_STATE_ALLOCATED"},{"line_number":290,"context_line":"            device.save(context)"},{"line_number":291,"context_line":""}],"source_content_type":"text/x-python","patch_set":2,"id":"5ed48eef_cde39381","line":288,"updated":"2026-08-06 11:26:24.000000000","message":"The new code in bind() transitions the device to ALLOCATED and saves it before calling _allocate_attach_handle(). If _allocate_attach_handle() or update_check_state() subsequently raises, the device remains stuck in ALLOCATED with no rollback path. Future bind attempts are rejected by the new gua...\n\n**Severity**: HIGH | **Confidence**: 0.8\n\n**Risk**: A failed bind (e.g. attach-handle allocation failure) leaves the device in ALLOCATED state. It becomes unavailable for future binds because the guard rejects non-AVAILABLE states. Recovery requires an agent restart so _reconcile_device_states can detect and fix the orphaned ALLOCATED device.\n\n**Priority**: Before merge\n**Why This Matters**: A failed bind (e.g. attach-handle allocation failure) leaves the device in ALLOCATED state. It becomes unavailable for future binds because the guard rejects non-AVAILABLE states. Recovery requires an agent restart so _reconcile_device_states can detect and fix the orphaned ALLOCATED device.\n\n**Recommendation**:\nWrap the post-guard logic in a try/except that reverts device.device_state to its previous value (e.g. AVAILABLE) and saves before re-raising on failure. Alternatively, set ALLOCATED only after _allocate_attach_handle succeeds.","commit_id":"747be871aa6f1c3b461784ac663dea1dfece0da2"},{"author":{"_account_id":28006,"name":"teim-ci","display_name":"teim-ci","email":"ci@seanmooney.info","username":"ci-sean-mooney","status":"this is a third-party ci account run by sean-k-mooney on irc\nhosted at zuul.teim.app"},"tag":"autogenerated:zuul:automatic-ci","change_message_id":"06e27c3c94c934e718a9274621bd17b09420246c","unresolved":false,"context_lines":[{"line_number":288,"context_line":""},{"line_number":289,"context_line":"        self._allocate_attach_handle(context, deployable)"},{"line_number":290,"context_line":""},{"line_number":291,"context_line":"        if device.device_state is not None:"},{"line_number":292,"context_line":"            device.device_state \u003d constants.DEVICE_STATE_ALLOCATED"},{"line_number":293,"context_line":"            device.save(context)"},{"line_number":294,"context_line":""}],"source_content_type":"text/x-python","patch_set":6,"id":"79ccc1b1_e72d4ae1","line":291,"updated":"2026-08-07 07:37:52.000000000","message":"The bind method sets device.device_state to ALLOCATED after a successful bind (lines 291-293). The bind guard at lines 279-287 rejects any bind where the device is not AVAILABLE. So a second ARQ targeting a different deployable on the same physical device is rejected with DeviceNotAvailable. Cybo...\n\n**Severity**: HIGH | **Confidence**: 0.8\n\n**Risk**: Any device with multiple deployables (multi-VF GPUs, multi-slot FPGAs, multi-port accelerators) can only serve one ARQ at a time. Additional ARQs for the same physical device will receive DeviceNotAvailable errors and fail to bind, breaking multi-instance accelerator sharing.\n\n**Priority**: Before merge\n**Why This Matters**: Any device with multiple deployables (multi-VF GPUs, multi-slot FPGAs, multi-port accelerators) can only serve one ARQ at a time. Additional ARQs for the same physical device will receive DeviceNotAvailable errors and fail to bind, breaking multi-instance accelerator sharing.\n\n**Recommendation**:\nEither (a) do not set device to ALLOCATED in bind() — instead rely solely on the cleaning/error guard to reject binds during cleanup, or (b) track the count of active ARQs per device and only set ALLOCATED on first bind, returning to AVAILABLE when the last ARQ unbinds, or (c) move the state tracking to the deployable level rather than the device level.","commit_id":"56a46f8cc76b7172f473bf27edec3d40fc5fc6f6"},{"author":{"_account_id":28006,"name":"teim-ci","display_name":"teim-ci","email":"ci@seanmooney.info","username":"ci-sean-mooney","status":"this is a third-party ci account run by sean-k-mooney on irc\nhosted at zuul.teim.app"},"tag":"autogenerated:zuul:automatic-ci","change_message_id":"28f4e10abc29550f555a2a7611c69e62c1993988","unresolved":false,"context_lines":[{"line_number":296,"context_line":"                state\u003ddevice.device_state,"},{"line_number":297,"context_line":"            )"},{"line_number":298,"context_line":""},{"line_number":299,"context_line":"        self._allocate_attach_handle(context, deployable)"},{"line_number":300,"context_line":"        placement_reserved \u003d False"},{"line_number":301,"context_line":"        try:"},{"line_number":302,"context_line":"            device.device_state \u003d constants.DEVICE_STATE_ALLOCATED"}],"source_content_type":"text/x-python","patch_set":14,"id":"efb0575d_e7eb7e31","line":299,"updated":"2026-08-13 14:52:40.000000000","message":"The _allocate_attach_handle() call at line 299 sits outside the try/except block (lines 301-316), so if it partially succeeds (AttachHandle.allocate creates the DB row but MDEV creation fails), the exception propagates directly without invoking _rollback_bind(). The allocated attach handle is lea...\n\n**Severity**: HIGH | **Confidence**: 0.8\n\n**Risk**: If _allocate_attach_handle partially succeeds (AH allocated in DB, then MDEV step fails), the attach handle remains allocated in the database with no ARQ properly referencing it after the bind failure. Over time this can accumulate orphaned attach handles and cause resource accounting drift.\n\n**Priority**: Before merge\n**Why This Matters**: If _allocate_attach_handle partially succeeds (AH allocated in DB, then MDEV step fails), the attach handle remains allocated in the database with no ARQ properly referencing it after the bind failure. Over time this can accumulate orphaned attach handles and cause resource accounting drift.\n\n**Recommendation**:\nMove the _allocate_attach_handle() call inside the try block so that _rollback_bind() can deallocate the attach handle on failure. Ensure _rollback_bind handles the case where attach_handle_id is set but deployable_id and device state have not yet been modified.","commit_id":"9ee4eb82b7f716e7b143b044691f0311d4b89fe4"},{"author":{"_account_id":28006,"name":"teim-ci","display_name":"teim-ci","email":"ci@seanmooney.info","username":"ci-sean-mooney","status":"this is a third-party ci account run by sean-k-mooney on irc\nhosted at zuul.teim.app"},"tag":"autogenerated:zuul:automatic-ci","change_message_id":"28f4e10abc29550f555a2a7611c69e62c1993988","unresolved":false,"context_lines":[{"line_number":324,"context_line":"            self.attach_handle_id is not None"},{"line_number":325,"context_line":"        ):"},{"line_number":326,"context_line":"            try:"},{"line_number":327,"context_line":"                self._deallocate_attach_handle("},{"line_number":328,"context_line":"                    context, self.attach_handle_id, self.arq.hostname"},{"line_number":329,"context_line":"                )"},{"line_number":330,"context_line":"            except Exception:"}],"source_content_type":"text/x-python","patch_set":14,"id":"cce7bcc2_6cb94add","line":327,"updated":"2026-08-13 14:52:40.000000000","message":"During bind rollback, _rollback_bind calls _deallocate_attach_handle which internally calls update_check_state(context, ARQ_UNBIND_FAILED) on its own failure (line 397). This transitions the ARQ to UnbindFailed. Then bind()\u0027s except block calls update_check_state(ARQ_BIND_FAILED), but the transfo...\n\n**Severity**: HIGH | **Confidence**: 0.8\n\n**Risk**: If attach handle deallocation fails during rollback, the ARQ ends up in UnbindFailed state. The subsequent update_check_state(ARQ_BIND_FAILED) raises ARQBadState, masking the original bind error with a state machine exception and leaving the ARQ in an inconsistent state.\n\n**Priority**: Before merge\n**Why This Matters**: If attach handle deallocation fails during rollback, the ARQ ends up in UnbindFailed state. The subsequent update_check_state(ARQ_BIND_FAILED) raises ARQBadState, masking the original bind error with a state machine exception and leaving the ARQ in an inconsistent state.\n\n**Recommendation**:\nWrap the _deallocate_attach_handle call inside _rollback_bind so that the ARQ_UNBIND_FAILED state transition it triggers does not conflict with the subsequent ARQ_BIND_FAILED transition. Consider having _deallocate_attach_handle skip state transitions when called from a rollback context.","commit_id":"9ee4eb82b7f716e7b143b044691f0311d4b89fe4"},{"author":{"_account_id":28006,"name":"teim-ci","display_name":"teim-ci","email":"ci@seanmooney.info","username":"ci-sean-mooney","status":"this is a third-party ci account run by sean-k-mooney on irc\nhosted at zuul.teim.app"},"tag":"autogenerated:zuul:automatic-ci","change_message_id":"28f4e10abc29550f555a2a7611c69e62c1993988","unresolved":false,"context_lines":[{"line_number":332,"context_line":"                    \u0027Failed to rollback attach handle for ARQ %s\u0027,"},{"line_number":333,"context_line":"                    self.arq.uuid,"},{"line_number":334,"context_line":"                )"},{"line_number":335,"context_line":"            self.attach_handle_id \u003d None"},{"line_number":336,"context_line":""},{"line_number":337,"context_line":"        if self.obj_attr_is_set(\u0027deployable_id\u0027) and ("},{"line_number":338,"context_line":"            self.deployable_id is not None"}],"source_content_type":"text/x-python","patch_set":14,"id":"413dce10_ee5035c1","line":335,"updated":"2026-08-13 14:52:40.000000000","message":"In _rollback_bind, self.attach_handle_id is set to None at line 335 but is only persisted to the database if the deployable_id block (lines 337-347) executes and saves. If the exception occurred before self.deployable_id was set (line 307), the deployable_id block is skipped and attach_handle_id\u003d...\n\n**Severity**: HIGH | **Confidence**: 0.8\n\n**Risk**: After a rollback where deployable_id was not yet set, the ExtARQ\u0027s attach_handle_id in the DB still references the (now deallocated) attach handle. Subsequent unbind or delete operations on this ARQ may attempt to deallocate an already-freed handle or reference stale data.\n\n**Priority**: Before merge\n**Why This Matters**: After a rollback where deployable_id was not yet set, the ExtARQ\u0027s attach_handle_id in the DB still references the (now deallocated) attach handle. Subsequent unbind or delete operations on this ARQ may attempt to deallocate an already-freed handle or reference stale data.\n\n**Recommendation**:\nAfter setting self.attach_handle_id \u003d None at line 335, call self.save(context) to persist the change independently of the deployable_id block.","commit_id":"9ee4eb82b7f716e7b143b044691f0311d4b89fe4"},{"author":{"_account_id":28006,"name":"teim-ci","display_name":"teim-ci","email":"ci@seanmooney.info","username":"ci-sean-mooney","status":"this is a third-party ci account run by sean-k-mooney on irc\nhosted at zuul.teim.app"},"tag":"autogenerated:zuul:automatic-ci","change_message_id":"185588e997ef9c4f150634af7d46351face4a8bd","unresolved":false,"context_lines":[{"line_number":289,"context_line":""},{"line_number":290,"context_line":"    def bind(self, context, deployable):"},{"line_number":291,"context_line":"        device \u003d objects.Device.get_by_device_id(context, deployable.device_id)"},{"line_number":292,"context_line":"        if device.device_state !\u003d constants.DEVICE_STATE_AVAILABLE:"},{"line_number":293,"context_line":"            self.update_check_state(context, constants.ARQ_BIND_FAILED)"},{"line_number":294,"context_line":"            raise exception.DeviceNotAvailable("},{"line_number":295,"context_line":"                device\u003ddevice.uuid,"}],"source_content_type":"text/x-python","patch_set":15,"id":"f50aa4b0_38427faf","line":292,"updated":"2026-08-14 16:24:03.000000000","message":"The guard \u0027device.device_state !\u003d DEVICE_STATE_AVAILABLE\u0027 rejects devices whose device_state is NULL (old rows before the best-effort startup backfill, or newly reported devices before conductor\u0027s periodic upsert backfills AVAILABLE) and also rejects a second bind of a shared deployable (e.g. vGP...\n\n**Severity**: HIGH | **Confidence**: 0.8\n\n**Risk**: Upgraded clouds where device_state has not yet been backfilled reject all binds with DeviceNotAvailable. Device profiles allocating more than one ARQ against the same deployable (vGPU) fail on the second bind, breaking Nova requests for multiple vGPUs of one PGPU.\n\n**Priority**: Before merge\n**Why This Matters**: Upgraded clouds where device_state has not yet been backfilled reject all binds with DeviceNotAvailable. Device profiles allocating more than one ARQ against the same deployable (vGPU) fail on the second bind, breaking Nova requests for multiple vGPUs of one PGPU.\n\n**Recommendation**:\nTreat NULL device_state as available, and gate on attach-handle availability rather than device-level state for deployables with num_accelerators \u003e 1 (e.g. only reject cleaning/pending_cleaning/error states), or at minimum document the multi-ARQ limitation in the release note.","commit_id":"a27febdebfc2cfd835b251cd3848f3191655479c"},{"author":{"_account_id":28006,"name":"teim-ci","display_name":"teim-ci","email":"ci@seanmooney.info","username":"ci-sean-mooney","status":"this is a third-party ci account run by sean-k-mooney on irc\nhosted at zuul.teim.app"},"tag":"autogenerated:zuul:automatic-ci","change_message_id":"185588e997ef9c4f150634af7d46351face4a8bd","unresolved":false,"context_lines":[{"line_number":319,"context_line":"        # if (self.arq.state \u003d\u003d constants.ARQ_DELETING"},{"line_number":320,"context_line":"        #         or self.arq.state \u003d\u003d ARQ_UNBOUND):"},{"line_number":321,"context_line":""},{"line_number":322,"context_line":"    def _rollback_bind(self, context, deployable, device, placement_reserved):"},{"line_number":323,"context_line":"        if self.obj_attr_is_set(\u0027attach_handle_id\u0027) and ("},{"line_number":324,"context_line":"            self.attach_handle_id is not None"},{"line_number":325,"context_line":"        ):"}],"source_content_type":"text/x-python","patch_set":15,"id":"2d8fef66_51e6555e","line":322,"updated":"2026-08-14 16:24:03.000000000","message":"bind() calls _rollback_bind inside its except clause intending to always reach update_check_state(context, ARQ_BIND_FAILED). But _rollback_bind invokes _deallocate_attach_handle, which itself calls update_check_state(context, ARQ_UNBIND_FAILED) and re-raises on failure; if that nested state trans...\n\n**Severity**: HIGH | **Confidence**: 0.8\n\n**Risk**: A bind failure occurring after the ARQ reaches Bound can surface ARQBadState instead of the original error, skipping the BindFailed terminal state Nova expects, and leaving attach handle / device / Placement state inconsistently rolled back.\n\n**Priority**: Before merge\n**Why This Matters**: A bind failure occurring after the ARQ reaches Bound can surface ARQBadState instead of the original error, skipping the BindFailed terminal state Nova expects, and leaving attach handle / device / Placement state inconsistently rolled back.\n\n**Recommendation**:\nDo not call update_check_state(ARQ_UNBIND_FAILED) from the rollback path; use update_state directly or catch exception.ARQBadState there. Also wrap _rollback_bind so bind()\u0027s handler can always reach update_check_state(context, ARQ_BIND_FAILED) before re-raising the original exception.","commit_id":"a27febdebfc2cfd835b251cd3848f3191655479c"},{"author":{"_account_id":28006,"name":"teim-ci","display_name":"teim-ci","email":"ci@seanmooney.info","username":"ci-sean-mooney","status":"this is a third-party ci account run by sean-k-mooney on irc\nhosted at zuul.teim.app"},"tag":"autogenerated:zuul:automatic-ci","change_message_id":"086caf06cad3a1e20079dda408771e484ae6d831","unresolved":false,"context_lines":[{"line_number":288,"context_line":"        )"},{"line_number":289,"context_line":""},{"line_number":290,"context_line":"    def bind(self, context, deployable):"},{"line_number":291,"context_line":"        device \u003d objects.Device.get_by_device_id(context, deployable.device_id)"},{"line_number":292,"context_line":"        if device.device_state !\u003d constants.DEVICE_STATE_AVAILABLE:"},{"line_number":293,"context_line":"            self.update_check_state(context, constants.ARQ_BIND_FAILED)"},{"line_number":294,"context_line":"            raise exception.DeviceNotAvailable("}],"source_content_type":"text/x-python","patch_set":16,"id":"d1653107_1755a1bc","line":291,"updated":"2026-08-15 17:10:38.000000000","message":"bind() now raises DeviceNotAvailable unless device_state is exactly AVAILABLE, but device_state is set to ALLOCATED after the first successful bind and only returns to AVAILABLE after cleanup. A second ARQ (e.g. another vGPU instance) targeting the same physical device is rejected with a terminal...\n\n**Severity**: HIGH | **Confidence**: 0.9\n\n**Risk**: Any deployment where a device (particularly vGPU deployables with num_accelerators \u003e 1, or multiple ARQs from one instance) needs more than one concurrent attach handle now fails binding with ARQ_BIND_FAILED; Nova receives a terminal bind status. This is a functional regression for vGPU and multi...\n\n**Priority**: Before merge\n**Why This Matters**: Any deployment where a device (particularly vGPU deployables with num_accelerators \u003e 1, or multiple ARQs from one instance) needs more than one concurrent attach handle now fails binding with ARQ_BIND_FAILED; Nova receives a terminal bind status. This is a functional regression for vGPU and multi...\n\n**Recommendation**:\nScope the guard to the states it is meant to protect (reject PENDING_CLEANING/CLEANING/ERROR) while continuing to allow binding when the device is AVAILABLE or ALLOCATED, or track per-deployable free attach_handle/in_use counts instead of a single device-level state. At minimum document/limit the guard to single-tenant devices like NVMe and add a test for a second concurrent bind.","commit_id":"911a10a7ff7ea2d5f062199c37f1ef4633c926c7"},{"author":{"_account_id":28006,"name":"teim-ci","display_name":"teim-ci","email":"ci@seanmooney.info","username":"ci-sean-mooney","status":"this is a third-party ci account run by sean-k-mooney on irc\nhosted at zuul.teim.app"},"tag":"autogenerated:zuul:automatic-ci","change_message_id":"086caf06cad3a1e20079dda408771e484ae6d831","unresolved":false,"context_lines":[{"line_number":289,"context_line":""},{"line_number":290,"context_line":"    def bind(self, context, deployable):"},{"line_number":291,"context_line":"        device \u003d objects.Device.get_by_device_id(context, deployable.device_id)"},{"line_number":292,"context_line":"        if device.device_state !\u003d constants.DEVICE_STATE_AVAILABLE:"},{"line_number":293,"context_line":"            self.update_check_state(context, constants.ARQ_BIND_FAILED)"},{"line_number":294,"context_line":"            raise exception.DeviceNotAvailable("},{"line_number":295,"context_line":"                device\u003ddevice.uuid,"}],"source_content_type":"text/x-python","patch_set":16,"id":"9856bacb_3a515868","line":292,"updated":"2026-08-15 17:10:38.000000000","message":"On device-state rejection bind() first calls update_check_state(context, ARQ_BIND_FAILED) and then raises DeviceNotAvailable. ARQ_BIND_FAILED\u0027s transform scope is [BIND_STARTED, BOUND]; update_check_state itself raises ARQBadState when the transition is illegal, which would replace the intended D...\n\n**Severity**: WARNING | **Confidence**: 0.8\n\n**Impact**: When the guard fires while the ARQ is not in BIND_STARTED/BOUND (e.g. INITIAL/UNBOUND), Nova/conductor receives ARQBadState instead of DeviceNotAvailable, hiding the real reason; the new tests only cover the BIND_STARTED case so this is untested.\n\n**Suggestion**:\nWrap the update_check_state call defensively or set the failure state only when the current state is within the allowed scope, and ensure DeviceNotAvailable is the exception propagated. Add a unit test where the ARQ is in a non-transformable state.","commit_id":"911a10a7ff7ea2d5f062199c37f1ef4633c926c7"},{"author":{"_account_id":28006,"name":"teim-ci","display_name":"teim-ci","email":"ci@seanmooney.info","username":"ci-sean-mooney","status":"this is a third-party ci account run by sean-k-mooney on irc\nhosted at zuul.teim.app"},"tag":"autogenerated:zuul:automatic-ci","change_message_id":"086caf06cad3a1e20079dda408771e484ae6d831","unresolved":false,"context_lines":[{"line_number":319,"context_line":"        # if (self.arq.state \u003d\u003d constants.ARQ_DELETING"},{"line_number":320,"context_line":"        #         or self.arq.state \u003d\u003d ARQ_UNBOUND):"},{"line_number":321,"context_line":""},{"line_number":322,"context_line":"    def _rollback_bind(self, context, deployable, device, placement_reserved):"},{"line_number":323,"context_line":"        if self.obj_attr_is_set(\u0027attach_handle_id\u0027) and ("},{"line_number":324,"context_line":"            self.attach_handle_id is not None"},{"line_number":325,"context_line":"        ):"}],"source_content_type":"text/x-python","patch_set":16,"id":"eb14db6b_cd4057f6","line":322,"updated":"2026-08-15 17:10:38.000000000","message":"_rollback_bind clears attach_handle_id/deployable_id in memory but self.save() is only called when deployable_id was set; after an attach-handle allocation failure the saved ARQ row can still reference the old attach handle. When self.save() or device.save() themselves fail, the rollback swallows...\n\n**Severity**: HIGH | **Confidence**: 0.8\n\n**Risk**: A failure during the primary save path (the common trigger for rollback) leaves the device permanently ALLOCATED with Placement inventory reserved; combined with CF-001\u0027s guard, all future binds to that device are rejected until manual DB surgery. A stale attach_handle_id in the ARQ row can also...\n\n**Priority**: Before merge\n**Why This Matters**: A failure during the primary save path (the common trigger for rollback) leaves the device permanently ALLOCATED with Placement inventory reserved; combined with CF-001\u0027s guard, all future binds to that device are rejected until manual DB surgery. A stale attach_handle_id in the ARQ row can also...\n\n**Recommendation**:\nPersist the attach_handle_id reset in all cases (save after clearing both fields), and when device.save() fails during rollback, mark the device ERROR (fail closed) and log an actionable error instead of silently continuing, so the inconsistent state is at least visible and excluded from binding.","commit_id":"911a10a7ff7ea2d5f062199c37f1ef4633c926c7"},{"author":{"_account_id":28006,"name":"teim-ci","display_name":"teim-ci","email":"ci@seanmooney.info","username":"ci-sean-mooney","status":"this is a third-party ci account run by sean-k-mooney on irc\nhosted at zuul.teim.app"},"tag":"autogenerated:zuul:automatic-ci","change_message_id":"f21af27a0fa4cc660f97b95b3dff5380e0e99091","unresolved":false,"context_lines":[{"line_number":289,"context_line":""},{"line_number":290,"context_line":"    def bind(self, context, deployable):"},{"line_number":291,"context_line":"        device \u003d objects.Device.get_by_device_id(context, deployable.device_id)"},{"line_number":292,"context_line":"        if device.device_state !\u003d constants.DEVICE_STATE_AVAILABLE:"},{"line_number":293,"context_line":"            self.update_check_state(context, constants.ARQ_BIND_FAILED)"},{"line_number":294,"context_line":"            raise exception.DeviceNotAvailable("},{"line_number":295,"context_line":"                device\u003ddevice.uuid,"}],"source_content_type":"text/x-python","patch_set":17,"id":"161fc138_5a9976c8","line":292,"updated":"2026-08-17 08:39:23.000000000","message":"bind() now raises DeviceNotAvailable whenever device_state !\u003d \u0027available\u0027, but a successful bind sets device_state\u003d\u0027allocated\u0027 (line 302) and it is only reset on unbind/cleanup. Deployables legitimately carry multiple attach handles, so binding a second ARQ to the same device now fails permanentl...\n\n**Severity**: HIGH | **Confidence**: 0.8\n\n**Risk**: Requests for more than one accelerator of the same type/profile that land on the same device/deployable will fail at bind time with DeviceNotAvailable and leave the ARQ in BindFailed, regressing multi-AH (vGPU/multi-function) use cases and Nova allocations from a single RP. Also, FPGA subclass pr...\n\n**Priority**: Before merge\n**Why This Matters**: Requests for more than one accelerator of the same type/profile that land on the same device/deployable will fail at bind time with DeviceNotAvailable and leave the ARQ in BindFailed, regressing multi-AH (vGPU/multi-function) use cases and Nova allocations from a single RP. Also, FPGA subclass pr...\n\n**Recommendation**:\nTrack per-device in-use ARQ/AH count instead of a single device_state gate: allow bind when the device is \u0027available\u0027 or \u0027allocated\u0027 but free attach handles remain, or gate on attach handle capacity. Alternatively keep the guard only for cleaning/pending_cleaning/error states. Add a unit test binding two ARQs to the same deployable.","commit_id":"fc11b6660359304e8e52503b84feed015ec1b128"},{"author":{"_account_id":28006,"name":"teim-ci","display_name":"teim-ci","email":"ci@seanmooney.info","username":"ci-sean-mooney","status":"this is a third-party ci account run by sean-k-mooney on irc\nhosted at zuul.teim.app"},"tag":"autogenerated:zuul:automatic-ci","change_message_id":"352193e6789fbbbe210f4b5daad1d6c0373562ea","unresolved":false,"context_lines":[{"line_number":301,"context_line":"        try:"},{"line_number":302,"context_line":"            device.device_state \u003d constants.DEVICE_STATE_ALLOCATED"},{"line_number":303,"context_line":"            device.save(context)"},{"line_number":304,"context_line":"            _set_deployable_placement_reserved(context, deployable, True)"},{"line_number":305,"context_line":"            placement_reserved \u003d True"},{"line_number":306,"context_line":""},{"line_number":307,"context_line":"            self.deployable_id \u003d deployable.id"}],"source_content_type":"text/x-python","patch_set":19,"id":"a1ce9deb_889e3101","line":304,"updated":"2026-08-21 12:23:44.000000000","message":"bind() now calls _set_deployable_placement_reserved(context, deployable, True) unconditionally for every device type, setting reserved \u003d num_accelerators on the deployable\u0027s RP inventory. On unbind, ExtARQ.unbind() only calls _dispatch_device_cleanup(); conductor dispatch_cleanup() resets non-cleaning devices to AVAILABLE and returns without clearing the reservation. Only the NVMe agent cleanup path (_run_cleanup_async) clears reserved.\n\n**Severity**: HIGH | **Confidence**: 0.82\n\n**Impact**: After one bind/unbind cycle of any non-NVMe device (GPU, FPGA, QAT...), the deployable\u0027s Placement inventory stays fully reserved, so allocation_candidates excludes the RP and the device cannot be allocated to any instance until an operator manually fixes Placement inventory.\n\n**Priority**: Before merge\n**Recommendation**:\nOnly reserve on bind when the device requires exclusive cleanup (device.supports_cleaning), or make the unbind/dispatch_cleanup path for non-cleaning devices clear the reservation (reserved\u003dFalse update) before returning. Add a unit test asserting unbind of a supports_cleaning\u003dFalse device restores reserved\u003d0.","commit_id":"d2767238f4909b2dd88ce36590b780476e9c3889"},{"author":{"_account_id":28006,"name":"teim-ci","display_name":"teim-ci","email":"ci@seanmooney.info","username":"ci-sean-mooney","status":"this is a third-party ci account run by sean-k-mooney on irc\nhosted at zuul.teim.app"},"tag":"autogenerated:zuul:automatic-ci","change_message_id":"e59f04ba23d15b289ba6f27f907293430ad476b5","unresolved":false,"context_lines":[{"line_number":277,"context_line":""},{"line_number":278,"context_line":"    def bind(self, context, deployable):"},{"line_number":279,"context_line":"        device \u003d objects.Device.get_by_device_id(context, deployable.device_id)"},{"line_number":280,"context_line":"        if device.device_state !\u003d constants.DEVICE_STATE_AVAILABLE:"},{"line_number":281,"context_line":"            self.update_check_state(context, constants.ARQ_BIND_FAILED)"},{"line_number":282,"context_line":"            raise exception.DeviceNotAvailable("},{"line_number":283,"context_line":"                device\u003ddevice.uuid,"}],"source_content_type":"text/x-python","patch_set":28,"id":"2df2ff24_1b755c47","line":280,"updated":"2026-08-31 11:27:16.000000000","message":"bind() now rejects any device whose device_state is not AVAILABLE and marks the whole Device ALLOCATED after the first bind. Production drivers still report a single Device whose deployable advertises multiple consumable units: gpu/nvidia/sysinfo.py:122-140 sets num_accelerators \u003d num_available + num_created and appends one attach handle per vGPU, fpga/intel/sysinfo.py:290-293 sets num_accelerators \u003d len(fpga[\u0027regions\u0027]) and one AH per VF, and nic/qat VF paths do the same. The conductor reports Placement inventory from that same value (conductor/manager.py:550 \u0027total \u003d obj.num_accelerators\u0027), so Placement legitimately places several instances on one RP; previously each bind consumed a distinct free attach handle via AttachHandle.allocate(). Now the second instance\u0027s ARQ bind raises DeviceNotAvailable (HTTP 409) and the ARQ goes to BindFailed, failing instance spawn. The converse also breaks: conductor dispatch_cleanup (conductor/manager.py:92-94) resets a non-NVMe device to AVAILABLE when any one ARQ unbinds, even if another VM still holds an in-use attach handle on that device. Only the fake driver was split into 16 single-unit devices.\n\n**Severity**: HIGH | **Confidence**: 0.8\n\n**Impact**: Deployments with NVIDIA vGPU, Intel FPGA multi-region, QAT or NIC VF devices regress: Placement advertises N units per RP, Nova schedules N instances, but every bind after the first for that device fails with 409 DeviceNotAvailable and the instance build fails. Conversely, unbinding one instance frees the device for rebinding while another instance still holds it.\n\n**Priority**: Before merge\n**Recommendation**:\nMake the guard capacity-aware instead of whole-device: only refuse when the deployable has no free attach handle (or num_accelerators \u003d\u003d 1), and only mark the device ALLOCATED when the last unit is consumed (symmetrically restore AVAILABLE only when no AH remains in use). Alternatively, restrict the whole-device AVAILABLE/ALLOCATED fencing to devices where supports_cleaning is True (NVMe) and leave attach-handle allocation as the concurrency limit for other device types, or apply the same one-device-per-unit split to the affected drivers.","commit_id":"e78df01a78803123a1ffa985c0a6cbe67cee5130"},{"author":{"_account_id":28006,"name":"teim-ci","display_name":"teim-ci","email":"ci@seanmooney.info","username":"ci-sean-mooney","status":"this is a third-party ci account run by sean-k-mooney on irc\nhosted at zuul.teim.app"},"tag":"autogenerated:zuul:automatic-ci","change_message_id":"cdf9cfb89e55ab4a2a0a9989b2d6c7a56a5a0c26","unresolved":false,"context_lines":[{"line_number":277,"context_line":""},{"line_number":278,"context_line":"    def bind(self, context, deployable):"},{"line_number":279,"context_line":"        device \u003d objects.Device.get_by_device_id(context, deployable.device_id)"},{"line_number":280,"context_line":"        if device.device_state !\u003d constants.DEVICE_STATE_AVAILABLE:"},{"line_number":281,"context_line":"            self.update_check_state(context, constants.ARQ_BIND_FAILED)"},{"line_number":282,"context_line":"            raise exception.DeviceNotAvailable("},{"line_number":283,"context_line":"                device\u003ddevice.uuid,"}],"source_content_type":"text/x-python","patch_set":32,"id":"dbbe2e1b_b9fcb71a","line":280,"updated":"2026-09-04 07:15:15.000000000","message":"ext_arq.bind() now rejects any device whose device_state is not AVAILABLE and marks the entire device ALLOCATED on the first successful bind, for every device type. Cyborg\u0027s real drivers report shared devices as one Device with many consumables: NVIDIA vGPU deployables carry num_accelerators \u003d available vGPU instances with one MDEV attach handle each (cyborg/accelerator/drivers/gpu/nvidia/sysinfo.py:105-120), and Intel NIC SR-IOV reports one Device with a deployable per VF (cyborg/accelerator/drivers/nic/intel/sysinfo.py:225-242). After the first VM binds, device_state is ALLOCATED, so the next ARQ bind for the same physical device fails the guard, is marked ARQ_BIND_FAILED, and raises DeviceNotAvailable (Conflict -\u003e HTTP 409). Only the fake driver was split into per-consumer devices to compensate; real shared topologies cannot be split that way.\n\n**Severity**: HIGH | **Confidence**: 0.8\n\n**Impact**: For any deployment using shared accelerators (vGPU-enabled GPUs, SR-IOV NICs with VFs), only one VM at a time can bind to a physical device; subsequent binds fail with DeviceNotAvailable/BindFailed even though free attach handles remain, breaking a supported multi-tenant path until the first VM is deleted.\n\n**Priority**: Before merge\n**Recommendation**:\nScope the whole-device allocation guard to devices where whole-device allocation is the intended model (e.g. gate on device.supports_cleaning / DEVICE_NVME), or for shared devices gate on attach-handle/inventory availability instead of device_state. Add a release note describing the new sharing semantics either way.","commit_id":"9b92cd31098fdb5ca2a3dec610a133186fa88295"},{"author":{"_account_id":28006,"name":"teim-ci","display_name":"teim-ci","email":"ci@seanmooney.info","username":"ci-sean-mooney","status":"this is a third-party ci account run by sean-k-mooney on irc\nhosted at zuul.teim.app"},"tag":"autogenerated:zuul:automatic-ci","change_message_id":"087b9638a005931481f4299d154356bc3579801f","unresolved":false,"context_lines":[{"line_number":276,"context_line":"        )"},{"line_number":277,"context_line":""},{"line_number":278,"context_line":"    def bind(self, context, deployable):"},{"line_number":279,"context_line":"        device \u003d objects.Device.get_by_device_id(context, deployable.device_id)"},{"line_number":280,"context_line":"        if device.device_state !\u003d constants.DEVICE_STATE_AVAILABLE:"},{"line_number":281,"context_line":"            self.update_check_state(context, constants.ARQ_BIND_FAILED)"},{"line_number":282,"context_line":"            raise exception.DeviceNotAvailable("}],"source_content_type":"text/x-python","patch_set":33,"id":"585c67f6_f5f50a50","line":279,"updated":"2026-09-05 04:57:57.000000000","message":"ExtARQ.bind() now rejects any device whose device_state is not AVAILABLE and marks the entire Device ALLOCATED on the first successful bind. Only the fake driver was reworked to report one consumable unit per device; real drivers still report one Device with multiple consumable attach handles. For NVIDIA vGPU, cyborg/accelerator/drivers/gpu/nvidia/sysinfo.py (vGPU branch, ~lines 122-140) builds a single deployable with num_accelerators \u003d available+created instances and one attach handle per vGPU, and the conductor publishes that count as Placement RP inventory. After the first VM binds, the Device is ALLOCATED, so the second VM\u0027s bind raises DeviceNotAvailable (HTTP 409) even though Placement shows free inventory on the RP and Nova has already claimed allocations there, failing supported multi-tenant vGPU (and multi-attach-handle FPGA) server builds.\n\n**Severity**: HIGH | **Confidence**: 0.85\n\n**Impact**: Second and later VMs requesting accelerators from a multi-unit device (vGPU pGPU shared by N VMs, multi-function FPGAs) fail ARQ bind with HTTP 409 after Nova has already placed allocations, breaking server builds in supported GPU deployments.\n\n**Priority**: Before merge\n**Recommendation**:\nApply the whole-device allocation only where one device is one consumable unit (e.g. gate the ALLOCATED transition on the same supports_cleaning/NVMe condition used for Placement reserved, or on num_accelerators\u003d\u003d1), or derive the guard from remaining attach-handle capacity instead of device-level state, leaving multi-unit devices on the attach-handle accounting.","commit_id":"7a52beb77426bf39ae0931b4b75a28dda8ae2d66"}],"cyborg/tests/unit/agent/test_manager.py":[{"author":{"_account_id":28006,"name":"teim-ci","display_name":"teim-ci","email":"ci@seanmooney.info","username":"ci-sean-mooney","status":"this is a third-party ci account run by sean-k-mooney on irc\nhosted at zuul.teim.app"},"tag":"autogenerated:zuul:automatic-ci","change_message_id":"93de7e09f828c593325106444e1f3735463f901f","unresolved":false,"context_lines":[{"line_number":420,"context_line":"        mock_conf.host \u003d \u0027compute-0\u0027"},{"line_number":421,"context_line":"        am \u003d self._create_manager_with_mocks()"},{"line_number":422,"context_line":""},{"line_number":423,"context_line":"        device \u003d mock.Mock(device_state\u003d\u0027cleaning\u0027, uuid\u003d\u0027dev-1\u0027)"},{"line_number":424,"context_line":"        mock_get_devs.return_value \u003d [device]"},{"line_number":425,"context_line":""},{"line_number":426,"context_line":"        am._reconcile_device_states()"}],"source_content_type":"text/x-python","patch_set":11,"id":"9a72172d_7125798f","line":423,"updated":"2026-08-12 09:59:45.000000000","message":"The test relies on Mock\u0027s default truthy return for device.supports_cleaning instead of explicitly setting it to True. This makes the test ambiguous about which code path is exercised (NVMe vs non-NVMe) and fragile if the code later changes to use the value differently.\n\n**Severity**: SUGGESTION | **Confidence**: 0.8\n\n**Benefit**: The test passes accidentally rather than intentionally. A future change that uses supports_cleaning as a non-boolean would break the test in a confusing way. Reviewers cannot easily understand what device type scenario is being tested.\n\n**Recommendation**:\nExplicitly set supports_cleaning\u003dTrue on the mock device (e.g., mock.Mock(device_state\u003d\u0027cleaning\u0027, uuid\u003d\u0027dev-1\u0027, supports_cleaning\u003dTrue)) and rename the test to indicate it covers the NVMe (cleaning-capable) path, e.g. test_reconcile_cleaning_nvme_to_error_reserved.","commit_id":"cc9cb0b00ea1c5abb432862b2b25fa2b9231562e"}],"cyborg/tests/unit/objects/test_extarq.py":[{"author":{"_account_id":28006,"name":"teim-ci","display_name":"teim-ci","email":"ci@seanmooney.info","username":"ci-sean-mooney","status":"this is a third-party ci account run by sean-k-mooney on irc\nhosted at zuul.teim.app"},"tag":"autogenerated:zuul:automatic-ci","change_message_id":"06e27c3c94c934e718a9274621bd17b09420246c","unresolved":false,"context_lines":[{"line_number":687,"context_line":"        device \u003d mock.Mock(device_state\u003d\u0027available\u0027, uuid\u003d\u0027dev-uuid\u0027)"},{"line_number":688,"context_line":"        mock_get_dev.return_value \u003d device"},{"line_number":689,"context_line":""},{"line_number":690,"context_line":"        obj_extarq.bind(self.context, dep)"},{"line_number":691,"context_line":""},{"line_number":692,"context_line":"        self.assertEqual(\u0027allocated\u0027, device.device_state)"},{"line_number":693,"context_line":"        device.save.assert_called_once_with(self.context)"}],"source_content_type":"text/x-python","patch_set":6,"id":"8220758a_4dc4c84c","line":690,"updated":"2026-08-07 07:37:52.000000000","message":"The new bind tests verify rejection of cleaning/error devices and acceptance of available/null-state devices, but none test the scenario where a second ARQ attempts to bind to a device that is already ALLOCATED from a prior bind. This is the critical edge case introduced by setting device to ALLO...\n\n**Severity**: WARNING | **Confidence**: 0.8\n\n**Impact**: The multi-deployable bind regression described in CF-001 would not be caught by the current test suite, giving false confidence that the bind guard is safe.\n\n**Suggestion**:\nAdd a test that binds two ARQs to different deployables on the same device and verifies the second bind succeeds (or document the intended single-ARQ-per-device behavior explicitly if that is the design intent).","commit_id":"56a46f8cc76b7172f473bf27edec3d40fc5fc6f6"}]}