)]}'
{"/PATCHSET_LEVEL":[{"author":{"_account_id":19262,"name":"Liron Kuchlani","email":"lkuchlan@redhat.com","username":"lkuchlan"},"change_message_id":"a409af1e59d78ac00b84d7d30bc6b9771b00e81f","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":2,"id":"4f5a8620_0fa9c840","updated":"2025-06-22 07:50:19.000000000","message":"recheck","commit_id":"6eb829b5bcb68c95018dcfbe91df149cac0b4b8e"},{"author":{"_account_id":19262,"name":"Liron Kuchlani","email":"lkuchlan@redhat.com","username":"lkuchlan"},"change_message_id":"6f2f88c99bb1f84ad7f197647cc0dd711d45af7e","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":2,"id":"d4854785_6df32993","updated":"2025-06-19 09:12:10.000000000","message":"recheck","commit_id":"6eb829b5bcb68c95018dcfbe91df149cac0b4b8e"},{"author":{"_account_id":19262,"name":"Liron Kuchlani","email":"lkuchlan@redhat.com","username":"lkuchlan"},"change_message_id":"e028d076d728c89521b0bb16cb15c6f4c9c5dcdd","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":12,"id":"0df6512c_e6f986d4","updated":"2025-06-25 09:38:34.000000000","message":"recheck","commit_id":"b3e91162ceb710e4548854b28f4da23f936f6e2b"},{"author":{"_account_id":9236,"name":"Jon Bernard","email":"jobernar@redhat.com","username":"jbernard"},"change_message_id":"d22c45dabdd09feb29f50d12faf1048002c693f7","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":16,"id":"e2319216_984cbd27","updated":"2025-07-31 15:40:12.000000000","message":"recheck cinder-tempest-plugin-cbak-ceph failed but no artifacts or logs are available to diagnose the problem.","commit_id":"ba29c20578e138e8e1d27647b1f64c162bb95a89"},{"author":{"_account_id":5314,"name":"Brian Rosmaita","email":"rosmaita.fossdev@gmail.com","username":"brian-rosmaita"},"change_message_id":"7fccb47699bdd626d8ae98ed8c057ef952984b68","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":17,"id":"49360090_8368c2b9","updated":"2025-08-20 13:21:10.000000000","message":"recheck cinder-tempest-plugin-cbak-ceph - OOM killed ceph-mgr, which kind of puts a damper on the ceph tests","commit_id":"5739915422b53d3c5c963c617be6762328537bcd"},{"author":{"_account_id":27615,"name":"Rajat Dhasmana","email":"rajatdhasmana@gmail.com","username":"whoami-rajat"},"change_message_id":"524bfba316aacc7407bec085aae3efd03ab9bfdd","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":19,"id":"52edd981_7b138090","updated":"2026-06-15 15:48:32.000000000","message":"Hi Liron, sorry for being late on this and thanks for working.\nI\u0027ve some small concerns and a big concern with the validation strategy we are using since the timestamp seems to be:\n\n1. not a valid check for external attached volumes\n2. for BFV, with disk I/O errors, it still returns the timestamp","commit_id":"5b68fc72c1e40d100c6fc5593dc7d5f425a2f878"},{"author":{"_account_id":19262,"name":"Liron Kuchlani","email":"lkuchlan@redhat.com","username":"lkuchlan"},"change_message_id":"fdc1d44bbdbff03788a7fc68d46191eafe78fa60","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":20,"id":"9f73e189_d731d85a","updated":"2026-06-16 12:42:21.000000000","message":"Thanks Rajat for you comments\nI\u0027ve uncommented the problematict tests so I guess there will failures","commit_id":"f7129e8ca49505f96482275cf07bd75c7ae4be6e"},{"author":{"_account_id":27615,"name":"Rajat Dhasmana","email":"rajatdhasmana@gmail.com","username":"whoami-rajat"},"change_message_id":"f4a007b56ebbe28e6e6e1bc7a88972d3f01f8d00","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":22,"id":"54de7a2f_38f38fb6","updated":"2026-06-16 21:55:39.000000000","message":"recheck\n\nThanks Liron for the updates. I\u0027ve verified the scenario tests manually and they work correctly.\nmulti-ceph job is passing so LGTM.","commit_id":"9395c4bec3054bd9c31c6aed66e78d1e164b9181"},{"author":{"_account_id":19262,"name":"Liron Kuchlani","email":"lkuchlan@redhat.com","username":"lkuchlan"},"change_message_id":"cd768a4e8a8d88b0eaa58e24003200422c092647","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":23,"id":"408e0409_9d9df26f","updated":"2026-06-17 11:10:27.000000000","message":"recheck","commit_id":"7353abcf3a36faa2a952245c33f59dfeb0e66888"},{"author":{"_account_id":27615,"name":"Rajat Dhasmana","email":"rajatdhasmana@gmail.com","username":"whoami-rajat"},"change_message_id":"3292d309009979edf678084922dc99017559065b","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":23,"id":"23b458a3_536f71f3","updated":"2026-06-18 07:47:40.000000000","message":"still LGTM, thanks Liron!","commit_id":"7353abcf3a36faa2a952245c33f59dfeb0e66888"}],"cinder_tempest_plugin/api/volume/admin/test_volume_replication.py":[{"author":{"_account_id":27615,"name":"Rajat Dhasmana","email":"rajatdhasmana@gmail.com","username":"whoami-rajat"},"change_message_id":"524bfba316aacc7407bec085aae3efd03ab9bfdd","unresolved":true,"context_lines":[{"line_number":143,"context_line":"    #"},{"line_number":144,"context_line":"    #     # Use helper to failover and register cleanup to failback"},{"line_number":145,"context_line":"    #     self._failover_host(volume)"},{"line_number":146,"context_line":"    #"},{"line_number":147,"context_line":"    #     # Verify replication_status is \u0027failed-over\u0027"},{"line_number":148,"context_line":"    #     volume_details \u003d self.volumes_client.show_volume("},{"line_number":149,"context_line":"    #         volume[\u0027id\u0027])[\u0027volume\u0027]"}],"source_content_type":"text/x-python","patch_set":19,"id":"e324a696_49de10b0","line":146,"updated":"2026-06-15 15:48:32.000000000","message":"We should wait for the failover to complete before verifying replication_status","commit_id":"5b68fc72c1e40d100c6fc5593dc7d5f425a2f878"},{"author":{"_account_id":19262,"name":"Liron Kuchlani","email":"lkuchlan@redhat.com","username":"lkuchlan"},"change_message_id":"fdc1d44bbdbff03788a7fc68d46191eafe78fa60","unresolved":true,"context_lines":[{"line_number":143,"context_line":"    #"},{"line_number":144,"context_line":"    #     # Use helper to failover and register cleanup to failback"},{"line_number":145,"context_line":"    #     self._failover_host(volume)"},{"line_number":146,"context_line":"    #"},{"line_number":147,"context_line":"    #     # Verify replication_status is \u0027failed-over\u0027"},{"line_number":148,"context_line":"    #     volume_details \u003d self.volumes_client.show_volume("},{"line_number":149,"context_line":"    #         volume[\u0027id\u0027])[\u0027volume\u0027]"}],"source_content_type":"text/x-python","patch_set":19,"id":"25cb7590_0b6d6c5a","line":146,"in_reply_to":"e324a696_49de10b0","updated":"2026-06-16 12:42:21.000000000","message":"There is already a waiter in the _failover_host method.\nAs far as I remember, we discussed this before, and we should investigate why that waiter is not sufficient.\n\nThis is the reason I commented out these tests","commit_id":"5b68fc72c1e40d100c6fc5593dc7d5f425a2f878"},{"author":{"_account_id":27615,"name":"Rajat Dhasmana","email":"rajatdhasmana@gmail.com","username":"whoami-rajat"},"change_message_id":"524bfba316aacc7407bec085aae3efd03ab9bfdd","unresolved":true,"context_lines":[{"line_number":164,"context_line":"    #         host\u003dself.replication_host, backend_id\u003d\u0027default\u0027)"},{"line_number":165,"context_line":"    #     waiters.wait_for_volume_replication_status("},{"line_number":166,"context_line":"    #         self.volumes_client, volume[\u0027id\u0027], \u0027enabled\u0027)"},{"line_number":167,"context_line":"    #"},{"line_number":168,"context_line":"    #     # Verify replication_status is \u0027enabled\u0027"},{"line_number":169,"context_line":"    #     volume_details \u003d self.volumes_client.show_volume("},{"line_number":170,"context_line":"    #         volume[\u0027id\u0027])[\u0027volume\u0027]"}],"source_content_type":"text/x-python","patch_set":19,"id":"2f98a8f0_91a90f7b","line":167,"updated":"2026-06-15 15:48:32.000000000","message":"same as above, needs a wait","commit_id":"5b68fc72c1e40d100c6fc5593dc7d5f425a2f878"},{"author":{"_account_id":19262,"name":"Liron Kuchlani","email":"lkuchlan@redhat.com","username":"lkuchlan"},"change_message_id":"fdc1d44bbdbff03788a7fc68d46191eafe78fa60","unresolved":true,"context_lines":[{"line_number":164,"context_line":"    #         host\u003dself.replication_host, backend_id\u003d\u0027default\u0027)"},{"line_number":165,"context_line":"    #     waiters.wait_for_volume_replication_status("},{"line_number":166,"context_line":"    #         self.volumes_client, volume[\u0027id\u0027], \u0027enabled\u0027)"},{"line_number":167,"context_line":"    #"},{"line_number":168,"context_line":"    #     # Verify replication_status is \u0027enabled\u0027"},{"line_number":169,"context_line":"    #     volume_details \u003d self.volumes_client.show_volume("},{"line_number":170,"context_line":"    #         volume[\u0027id\u0027])[\u0027volume\u0027]"}],"source_content_type":"text/x-python","patch_set":19,"id":"928de960_5c84adbd","line":167,"in_reply_to":"2f98a8f0_91a90f7b","updated":"2026-06-16 12:42:21.000000000","message":"same","commit_id":"5b68fc72c1e40d100c6fc5593dc7d5f425a2f878"},{"author":{"_account_id":27615,"name":"Rajat Dhasmana","email":"rajatdhasmana@gmail.com","username":"whoami-rajat"},"change_message_id":"524bfba316aacc7407bec085aae3efd03ab9bfdd","unresolved":true,"context_lines":[{"line_number":189,"context_line":"            snapshot[\u0027id\u0027])[\u0027snapshot\u0027]"},{"line_number":190,"context_line":"        self.assertEqual(volume[\u0027id\u0027], snapshot_details[\u0027volume_id\u0027])"},{"line_number":191,"context_line":""},{"line_number":192,"context_line":"    # @decorators.idempotent_id(\u00276a079f96-51ee-408a-aced-e90694804a72\u0027)"},{"line_number":193,"context_line":"    # def test_snapshot_after_failover(self):"},{"line_number":194,"context_line":"    #     \"\"\"Test creating snapshot on secondary backend after failover.\"\"\""},{"line_number":195,"context_line":"    #     volume \u003d self._create_replicated_volume()"},{"line_number":196,"context_line":"    #     self.admin_volume_services_client.failover_host(host\u003dself.replication_host)"},{"line_number":197,"context_line":"    #     waiters.wait_for_volume_replication_status("},{"line_number":198,"context_line":"    #         self.volumes_client, volume[\u0027id\u0027], \u0027failed-over\u0027)"},{"line_number":199,"context_line":"    #     self._failover_host(volume)"},{"line_number":200,"context_line":"    #"},{"line_number":201,"context_line":"    #     snapshot \u003d self.create_snapshot(volume[\u0027id\u0027])"},{"line_number":202,"context_line":"    #     snap_details \u003d self.snapshots_client.show_snapshot("},{"line_number":203,"context_line":"    #         snapshot[\u0027id\u0027])[\u0027snapshot\u0027]"},{"line_number":204,"context_line":"    #     self.assertEqual(snap_details[\u0027volume_id\u0027], volume[\u0027id\u0027])"},{"line_number":205,"context_line":"    #"},{"line_number":206,"context_line":"    # @decorators.idempotent_id(\u0027e0c5dd9a-2b20-4231-bf96-aa19929b282e\u0027)"},{"line_number":207,"context_line":"    # def test_clone_after_failover(self):"},{"line_number":208,"context_line":"    #     \"\"\"Test cloning volume on secondary backend after failover.\"\"\""},{"line_number":209,"context_line":"    #     volume \u003d self._create_replicated_volume()"},{"line_number":210,"context_line":"    #     self._failover_host(volume)"},{"line_number":211,"context_line":"    #"},{"line_number":212,"context_line":"    #     cloned_volume \u003d self.create_volume(source_volid\u003dvolume[\u0027id\u0027])"},{"line_number":213,"context_line":"    #     cloned_vol_details \u003d self.volumes_client.show_volume("},{"line_number":214,"context_line":"    #         cloned_volume[\u0027id\u0027])[\u0027volume\u0027]"},{"line_number":215,"context_line":"    #"},{"line_number":216,"context_line":"    #     self.assertEqual(cloned_vol_details[\u0027source_volid\u0027], volume[\u0027id\u0027])"},{"line_number":217,"context_line":"    #     self.assertEqual(cloned_vol_details[\u0027replication_status\u0027],"},{"line_number":218,"context_line":"    #                      \u0027failed-over\u0027)"},{"line_number":219,"context_line":"    #"},{"line_number":220,"context_line":"    # @decorators.idempotent_id(\u00274fbfe590-2026-4ea6-9495-60a32a88a856\u0027)"},{"line_number":221,"context_line":"    # def test_retype_after_failover(self):"},{"line_number":222,"context_line":"    #     \"\"\"Test retyping volume on secondary backend after failover.\"\"\""},{"line_number":223,"context_line":"    #     volume \u003d self._create_replicated_volume()"},{"line_number":224,"context_line":"    #     self._failover_host(volume)"},{"line_number":225,"context_line":"    #"},{"line_number":226,"context_line":"    #     # Create a non-replicated volume type"},{"line_number":227,"context_line":"    #     non_replicated_type \u003d self.create_volume_type("},{"line_number":228,"context_line":"    #         extra_specs\u003dself.non_replication_specs)"},{"line_number":229,"context_line":"    #"},{"line_number":230,"context_line":"    #     # Retype from replicated to non-replicated"},{"line_number":231,"context_line":"    #     self.volumes_client.retype_volume(volume[\u0027id\u0027],"},{"line_number":232,"context_line":"    #                                       new_type\u003dnon_replicated_type[\u0027id\u0027],"},{"line_number":233,"context_line":"    #                                       migration_policy\u003d\u0027on-demand\u0027)"},{"line_number":234,"context_line":"    #"},{"line_number":235,"context_line":"    #     waiters.wait_for_volume_replication_status("},{"line_number":236,"context_line":"    #         self.volumes_client, volume[\u0027id\u0027], \u0027disabled\u0027)"}],"source_content_type":"text/x-python","patch_set":19,"id":"5864a72e_edae4e98","line":236,"range":{"start_line":192,"start_character":0,"end_line":236,"end_character":60},"updated":"2026-06-15 15:48:32.000000000","message":"why are these tests commented?","commit_id":"5b68fc72c1e40d100c6fc5593dc7d5f425a2f878"},{"author":{"_account_id":19262,"name":"Liron Kuchlani","email":"lkuchlan@redhat.com","username":"lkuchlan"},"change_message_id":"fdc1d44bbdbff03788a7fc68d46191eafe78fa60","unresolved":true,"context_lines":[{"line_number":189,"context_line":"            snapshot[\u0027id\u0027])[\u0027snapshot\u0027]"},{"line_number":190,"context_line":"        self.assertEqual(volume[\u0027id\u0027], snapshot_details[\u0027volume_id\u0027])"},{"line_number":191,"context_line":""},{"line_number":192,"context_line":"    # @decorators.idempotent_id(\u00276a079f96-51ee-408a-aced-e90694804a72\u0027)"},{"line_number":193,"context_line":"    # def test_snapshot_after_failover(self):"},{"line_number":194,"context_line":"    #     \"\"\"Test creating snapshot on secondary backend after failover.\"\"\""},{"line_number":195,"context_line":"    #     volume \u003d self._create_replicated_volume()"},{"line_number":196,"context_line":"    #     self.admin_volume_services_client.failover_host(host\u003dself.replication_host)"},{"line_number":197,"context_line":"    #     waiters.wait_for_volume_replication_status("},{"line_number":198,"context_line":"    #         self.volumes_client, volume[\u0027id\u0027], \u0027failed-over\u0027)"},{"line_number":199,"context_line":"    #     self._failover_host(volume)"},{"line_number":200,"context_line":"    #"},{"line_number":201,"context_line":"    #     snapshot \u003d self.create_snapshot(volume[\u0027id\u0027])"},{"line_number":202,"context_line":"    #     snap_details \u003d self.snapshots_client.show_snapshot("},{"line_number":203,"context_line":"    #         snapshot[\u0027id\u0027])[\u0027snapshot\u0027]"},{"line_number":204,"context_line":"    #     self.assertEqual(snap_details[\u0027volume_id\u0027], volume[\u0027id\u0027])"},{"line_number":205,"context_line":"    #"},{"line_number":206,"context_line":"    # @decorators.idempotent_id(\u0027e0c5dd9a-2b20-4231-bf96-aa19929b282e\u0027)"},{"line_number":207,"context_line":"    # def test_clone_after_failover(self):"},{"line_number":208,"context_line":"    #     \"\"\"Test cloning volume on secondary backend after failover.\"\"\""},{"line_number":209,"context_line":"    #     volume \u003d self._create_replicated_volume()"},{"line_number":210,"context_line":"    #     self._failover_host(volume)"},{"line_number":211,"context_line":"    #"},{"line_number":212,"context_line":"    #     cloned_volume \u003d self.create_volume(source_volid\u003dvolume[\u0027id\u0027])"},{"line_number":213,"context_line":"    #     cloned_vol_details \u003d self.volumes_client.show_volume("},{"line_number":214,"context_line":"    #         cloned_volume[\u0027id\u0027])[\u0027volume\u0027]"},{"line_number":215,"context_line":"    #"},{"line_number":216,"context_line":"    #     self.assertEqual(cloned_vol_details[\u0027source_volid\u0027], volume[\u0027id\u0027])"},{"line_number":217,"context_line":"    #     self.assertEqual(cloned_vol_details[\u0027replication_status\u0027],"},{"line_number":218,"context_line":"    #                      \u0027failed-over\u0027)"},{"line_number":219,"context_line":"    #"},{"line_number":220,"context_line":"    # @decorators.idempotent_id(\u00274fbfe590-2026-4ea6-9495-60a32a88a856\u0027)"},{"line_number":221,"context_line":"    # def test_retype_after_failover(self):"},{"line_number":222,"context_line":"    #     \"\"\"Test retyping volume on secondary backend after failover.\"\"\""},{"line_number":223,"context_line":"    #     volume \u003d self._create_replicated_volume()"},{"line_number":224,"context_line":"    #     self._failover_host(volume)"},{"line_number":225,"context_line":"    #"},{"line_number":226,"context_line":"    #     # Create a non-replicated volume type"},{"line_number":227,"context_line":"    #     non_replicated_type \u003d self.create_volume_type("},{"line_number":228,"context_line":"    #         extra_specs\u003dself.non_replication_specs)"},{"line_number":229,"context_line":"    #"},{"line_number":230,"context_line":"    #     # Retype from replicated to non-replicated"},{"line_number":231,"context_line":"    #     self.volumes_client.retype_volume(volume[\u0027id\u0027],"},{"line_number":232,"context_line":"    #                                       new_type\u003dnon_replicated_type[\u0027id\u0027],"},{"line_number":233,"context_line":"    #                                       migration_policy\u003d\u0027on-demand\u0027)"},{"line_number":234,"context_line":"    #"},{"line_number":235,"context_line":"    #     waiters.wait_for_volume_replication_status("},{"line_number":236,"context_line":"    #         self.volumes_client, volume[\u0027id\u0027], \u0027disabled\u0027)"}],"source_content_type":"text/x-python","patch_set":19,"id":"1c03326a_b7f6e748","line":236,"range":{"start_line":192,"start_character":0,"end_line":236,"end_character":60},"in_reply_to":"5864a72e_edae4e98","updated":"2026-06-16 12:42:21.000000000","message":"same as above","commit_id":"5b68fc72c1e40d100c6fc5593dc7d5f425a2f878"},{"author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"tag":"autogenerated:zuul:check","change_message_id":"031a2109be51898c7c66b82cb0a0b828235f887a","unresolved":false,"context_lines":[{"line_number":193,"context_line":"    def test_snapshot_after_failover(self):"},{"line_number":194,"context_line":"        \"\"\"Test creating snapshot on secondary backend after failover.\"\"\""},{"line_number":195,"context_line":"        volume \u003d self._create_replicated_volume()"},{"line_number":196,"context_line":"        self.admin_volume_services_client.failover_host(host\u003dself.replication_host)"},{"line_number":197,"context_line":"        waiters.wait_for_volume_replication_status("},{"line_number":198,"context_line":"            self.volumes_client, volume[\u0027id\u0027], \u0027failed-over\u0027)"},{"line_number":199,"context_line":"        self._failover_host(volume)"}],"source_content_type":"text/x-python","patch_set":20,"id":"d419e662_c454a672","line":196,"updated":"2026-06-16 17:11:43.000000000","message":"pep8: E501 line too long (83 \u003e 79 characters)","commit_id":"f7129e8ca49505f96482275cf07bd75c7ae4be6e"},{"author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"tag":"autogenerated:zuul:check","change_message_id":"f78c04936a57a904bc57cfa61eea37d7b49eba21","unresolved":false,"context_lines":[{"line_number":122,"context_line":"        snapshot_details \u003d self.snapshots_client.show_snapshot("},{"line_number":123,"context_line":"            snapshot[\u0027id\u0027])[\u0027snapshot\u0027]"},{"line_number":124,"context_line":"        self.assertEqual(volume[\u0027id\u0027], snapshot_details[\u0027volume_id\u0027])"},{"line_number":125,"context_line":""}],"source_content_type":"text/x-python","patch_set":22,"id":"da3efa16_4e10f352","line":125,"updated":"2026-06-17 00:29:26.000000000","message":"pep8: W391 blank line at end of file","commit_id":"9395c4bec3054bd9c31c6aed66e78d1e164b9181"},{"author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"tag":"autogenerated:zuul:check","change_message_id":"988903b701a3a95c2e402f91498a7f41b46021c4","unresolved":false,"context_lines":[{"line_number":122,"context_line":"        snapshot_details \u003d self.snapshots_client.show_snapshot("},{"line_number":123,"context_line":"            snapshot[\u0027id\u0027])[\u0027snapshot\u0027]"},{"line_number":124,"context_line":"        self.assertEqual(volume[\u0027id\u0027], snapshot_details[\u0027volume_id\u0027])"},{"line_number":125,"context_line":""}],"source_content_type":"text/x-python","patch_set":22,"id":"f70a1367_82e3880b","line":125,"updated":"2026-06-16 21:27:55.000000000","message":"pep8: W391 blank line at end of file","commit_id":"9395c4bec3054bd9c31c6aed66e78d1e164b9181"}],"cinder_tempest_plugin/scenario/test_volume_replication.py":[{"author":{"_account_id":27615,"name":"Rajat Dhasmana","email":"rajatdhasmana@gmail.com","username":"whoami-rajat"},"change_message_id":"933393d038a1f819135b2647544db4271db7a133","unresolved":true,"context_lines":[{"line_number":156,"context_line":"        waiters.wait_for_volume_replication_status("},{"line_number":157,"context_line":"            self.volumes_client, volume[\u0027id\u0027], \u0027failed-over\u0027)"},{"line_number":158,"context_line":""},{"line_number":159,"context_line":"        # Validate data still accessible after failover"},{"line_number":160,"context_line":"        timestamp_after_failover \u003d self.get_timestamp("},{"line_number":161,"context_line":"            server_ip, private_key\u003dkeypair[\u0027private_key\u0027], server\u003dserver)"},{"line_number":162,"context_line":"        self.assertEqual(timestamp_before_failover, timestamp_after_failover)"},{"line_number":163,"context_line":""},{"line_number":164,"context_line":"        # Write new data after failover"},{"line_number":165,"context_line":"        timestamp_before_failback \u003d self.create_timestamp("},{"line_number":166,"context_line":"            server_ip, private_key\u003dkeypair[\u0027private_key\u0027], server\u003dserver)"}],"source_content_type":"text/x-python","patch_set":14,"id":"78dd2678_7311d4e4","line":163,"range":{"start_line":159,"start_character":7,"end_line":163,"end_character":0},"updated":"2025-06-26 09:52:07.000000000","message":"This ideally shouldn\u0027t work since after failover is done, we need to detach/attach the volume (external data vol) or shelve/unshelve the VM (BFV) to access the replica volume on secondary cluster.","commit_id":"2709819a02d36c719e19f679b6b2aa193035fa8d"},{"author":{"_account_id":19262,"name":"Liron Kuchlani","email":"lkuchlan@redhat.com","username":"lkuchlan"},"change_message_id":"fdc1d44bbdbff03788a7fc68d46191eafe78fa60","unresolved":false,"context_lines":[{"line_number":156,"context_line":"        waiters.wait_for_volume_replication_status("},{"line_number":157,"context_line":"            self.volumes_client, volume[\u0027id\u0027], \u0027failed-over\u0027)"},{"line_number":158,"context_line":""},{"line_number":159,"context_line":"        # Validate data still accessible after failover"},{"line_number":160,"context_line":"        timestamp_after_failover \u003d self.get_timestamp("},{"line_number":161,"context_line":"            server_ip, private_key\u003dkeypair[\u0027private_key\u0027], server\u003dserver)"},{"line_number":162,"context_line":"        self.assertEqual(timestamp_before_failover, timestamp_after_failover)"},{"line_number":163,"context_line":""},{"line_number":164,"context_line":"        # Write new data after failover"},{"line_number":165,"context_line":"        timestamp_before_failback \u003d self.create_timestamp("},{"line_number":166,"context_line":"            server_ip, private_key\u003dkeypair[\u0027private_key\u0027], server\u003dserver)"}],"source_content_type":"text/x-python","patch_set":14,"id":"8fb14cd6_8317c00e","line":163,"range":{"start_line":159,"start_character":7,"end_line":163,"end_character":0},"in_reply_to":"06e14c55_77864cd9","updated":"2026-06-16 12:42:21.000000000","message":"Done","commit_id":"2709819a02d36c719e19f679b6b2aa193035fa8d"},{"author":{"_account_id":19262,"name":"Liron Kuchlani","email":"lkuchlan@redhat.com","username":"lkuchlan"},"change_message_id":"c9d83881f3947d22fc5b0269f7f6a3c6575e8510","unresolved":true,"context_lines":[{"line_number":156,"context_line":"        waiters.wait_for_volume_replication_status("},{"line_number":157,"context_line":"            self.volumes_client, volume[\u0027id\u0027], \u0027failed-over\u0027)"},{"line_number":158,"context_line":""},{"line_number":159,"context_line":"        # Validate data still accessible after failover"},{"line_number":160,"context_line":"        timestamp_after_failover \u003d self.get_timestamp("},{"line_number":161,"context_line":"            server_ip, private_key\u003dkeypair[\u0027private_key\u0027], server\u003dserver)"},{"line_number":162,"context_line":"        self.assertEqual(timestamp_before_failover, timestamp_after_failover)"},{"line_number":163,"context_line":""},{"line_number":164,"context_line":"        # Write new data after failover"},{"line_number":165,"context_line":"        timestamp_before_failback \u003d self.create_timestamp("},{"line_number":166,"context_line":"            server_ip, private_key\u003dkeypair[\u0027private_key\u0027], server\u003dserver)"}],"source_content_type":"text/x-python","patch_set":14,"id":"e8dbaa2b_28be5c10","line":163,"range":{"start_line":159,"start_character":7,"end_line":163,"end_character":0},"in_reply_to":"06fba8c5_cd6ca2bc","updated":"2025-06-26 10:53:49.000000000","message":"I suspect this issue is related to the strange behavior we observed: after creating a replicated volume and performing a failover, the volume ends up in an error state despite its replication_status showing as “enabled”.\nIf I add an extra waiter after creating the replicated volume, it works fine.","commit_id":"2709819a02d36c719e19f679b6b2aa193035fa8d"},{"author":{"_account_id":19262,"name":"Liron Kuchlani","email":"lkuchlan@redhat.com","username":"lkuchlan"},"change_message_id":"44324f0481a31b108f17d35084450b19776894a6","unresolved":true,"context_lines":[{"line_number":156,"context_line":"        waiters.wait_for_volume_replication_status("},{"line_number":157,"context_line":"            self.volumes_client, volume[\u0027id\u0027], \u0027failed-over\u0027)"},{"line_number":158,"context_line":""},{"line_number":159,"context_line":"        # Validate data still accessible after failover"},{"line_number":160,"context_line":"        timestamp_after_failover \u003d self.get_timestamp("},{"line_number":161,"context_line":"            server_ip, private_key\u003dkeypair[\u0027private_key\u0027], server\u003dserver)"},{"line_number":162,"context_line":"        self.assertEqual(timestamp_before_failover, timestamp_after_failover)"},{"line_number":163,"context_line":""},{"line_number":164,"context_line":"        # Write new data after failover"},{"line_number":165,"context_line":"        timestamp_before_failback \u003d self.create_timestamp("},{"line_number":166,"context_line":"            server_ip, private_key\u003dkeypair[\u0027private_key\u0027], server\u003dserver)"}],"source_content_type":"text/x-python","patch_set":14,"id":"06fba8c5_cd6ca2bc","line":163,"range":{"start_line":159,"start_character":7,"end_line":163,"end_character":0},"in_reply_to":"78dd2678_7311d4e4","updated":"2025-06-26 10:02:31.000000000","message":"It’s odd because the test passes just fine on the gate.","commit_id":"2709819a02d36c719e19f679b6b2aa193035fa8d"},{"author":{"_account_id":27615,"name":"Rajat Dhasmana","email":"rajatdhasmana@gmail.com","username":"whoami-rajat"},"change_message_id":"524bfba316aacc7407bec085aae3efd03ab9bfdd","unresolved":true,"context_lines":[{"line_number":156,"context_line":"        waiters.wait_for_volume_replication_status("},{"line_number":157,"context_line":"            self.volumes_client, volume[\u0027id\u0027], \u0027failed-over\u0027)"},{"line_number":158,"context_line":""},{"line_number":159,"context_line":"        # Validate data still accessible after failover"},{"line_number":160,"context_line":"        timestamp_after_failover \u003d self.get_timestamp("},{"line_number":161,"context_line":"            server_ip, private_key\u003dkeypair[\u0027private_key\u0027], server\u003dserver)"},{"line_number":162,"context_line":"        self.assertEqual(timestamp_before_failover, timestamp_after_failover)"},{"line_number":163,"context_line":""},{"line_number":164,"context_line":"        # Write new data after failover"},{"line_number":165,"context_line":"        timestamp_before_failback \u003d self.create_timestamp("},{"line_number":166,"context_line":"            server_ip, private_key\u003dkeypair[\u0027private_key\u0027], server\u003dserver)"}],"source_content_type":"text/x-python","patch_set":14,"id":"06e14c55_77864cd9","line":163,"range":{"start_line":159,"start_character":7,"end_line":163,"end_character":0},"in_reply_to":"af71b5d5_57fa6ae1","updated":"2026-06-15 15:48:32.000000000","message":"This is primarily being tested with RBD supporting async replication so sync is out of question here.","commit_id":"2709819a02d36c719e19f679b6b2aa193035fa8d"},{"author":{"_account_id":13425,"name":"Simon Dodsley","email":"simon@everpuredata.com","username":"sdodsley"},"change_message_id":"8eb1b6750c1402346fbab4f3ef544d867f7474e2","unresolved":true,"context_lines":[{"line_number":156,"context_line":"        waiters.wait_for_volume_replication_status("},{"line_number":157,"context_line":"            self.volumes_client, volume[\u0027id\u0027], \u0027failed-over\u0027)"},{"line_number":158,"context_line":""},{"line_number":159,"context_line":"        # Validate data still accessible after failover"},{"line_number":160,"context_line":"        timestamp_after_failover \u003d self.get_timestamp("},{"line_number":161,"context_line":"            server_ip, private_key\u003dkeypair[\u0027private_key\u0027], server\u003dserver)"},{"line_number":162,"context_line":"        self.assertEqual(timestamp_before_failover, timestamp_after_failover)"},{"line_number":163,"context_line":""},{"line_number":164,"context_line":"        # Write new data after failover"},{"line_number":165,"context_line":"        timestamp_before_failback \u003d self.create_timestamp("},{"line_number":166,"context_line":"            server_ip, private_key\u003dkeypair[\u0027private_key\u0027], server\u003dserver)"}],"source_content_type":"text/x-python","patch_set":14,"id":"af71b5d5_57fa6ae1","line":163,"range":{"start_line":159,"start_character":7,"end_line":163,"end_character":0},"in_reply_to":"e8dbaa2b_28be5c10","updated":"2025-08-01 14:37:37.000000000","message":"Depends on the backend as well. a sync replicated volume that connectivity to the VM from both arrays will still show as accessible.","commit_id":"2709819a02d36c719e19f679b6b2aa193035fa8d"},{"author":{"_account_id":27615,"name":"Rajat Dhasmana","email":"rajatdhasmana@gmail.com","username":"whoami-rajat"},"change_message_id":"a9f0610c039a1ab0110d52234902257eadc5a87c","unresolved":true,"context_lines":[{"line_number":165,"context_line":"        timestamp_before_failback \u003d self.create_timestamp("},{"line_number":166,"context_line":"            server_ip, private_key\u003dkeypair[\u0027private_key\u0027], server\u003dserver)"},{"line_number":167,"context_line":""},{"line_number":168,"context_line":"        # Failback"},{"line_number":169,"context_line":"        self.admin_volume_services_client.failover_host("},{"line_number":170,"context_line":"            host\u003dself.replication_host, backend_id\u003d\u0027default\u0027)"},{"line_number":171,"context_line":"        waiters.wait_for_volume_replication_status("},{"line_number":172,"context_line":"            self.volumes_client, volume[\u0027id\u0027], \u0027enabled\u0027)"},{"line_number":173,"context_line":""}],"source_content_type":"text/x-python","patch_set":14,"id":"d691a0b7_02433d67","line":170,"range":{"start_line":168,"start_character":8,"end_line":170,"end_character":61},"updated":"2025-06-26 09:52:56.000000000","message":"I feel failback should be a different test since it\u0027s a different operation altogether","commit_id":"2709819a02d36c719e19f679b6b2aa193035fa8d"},{"author":{"_account_id":19262,"name":"Liron Kuchlani","email":"lkuchlan@redhat.com","username":"lkuchlan"},"change_message_id":"7f0432e0b9210b5836a45f6284c14e2521e7d27e","unresolved":true,"context_lines":[{"line_number":165,"context_line":"        timestamp_before_failback \u003d self.create_timestamp("},{"line_number":166,"context_line":"            server_ip, private_key\u003dkeypair[\u0027private_key\u0027], server\u003dserver)"},{"line_number":167,"context_line":""},{"line_number":168,"context_line":"        # Failback"},{"line_number":169,"context_line":"        self.admin_volume_services_client.failover_host("},{"line_number":170,"context_line":"            host\u003dself.replication_host, backend_id\u003d\u0027default\u0027)"},{"line_number":171,"context_line":"        waiters.wait_for_volume_replication_status("},{"line_number":172,"context_line":"            self.volumes_client, volume[\u0027id\u0027], \u0027enabled\u0027)"},{"line_number":173,"context_line":""}],"source_content_type":"text/x-python","patch_set":14,"id":"7727d5bf_82d28560","line":170,"range":{"start_line":168,"start_character":8,"end_line":170,"end_character":61},"in_reply_to":"7181d49f_106aa32f","updated":"2025-06-26 10:06:15.000000000","message":"I\u0027ll add tests specific to each operation in the API directory.","commit_id":"2709819a02d36c719e19f679b6b2aa193035fa8d"},{"author":{"_account_id":19262,"name":"Liron Kuchlani","email":"lkuchlan@redhat.com","username":"lkuchlan"},"change_message_id":"44324f0481a31b108f17d35084450b19776894a6","unresolved":true,"context_lines":[{"line_number":165,"context_line":"        timestamp_before_failback \u003d self.create_timestamp("},{"line_number":166,"context_line":"            server_ip, private_key\u003dkeypair[\u0027private_key\u0027], server\u003dserver)"},{"line_number":167,"context_line":""},{"line_number":168,"context_line":"        # Failback"},{"line_number":169,"context_line":"        self.admin_volume_services_client.failover_host("},{"line_number":170,"context_line":"            host\u003dself.replication_host, backend_id\u003d\u0027default\u0027)"},{"line_number":171,"context_line":"        waiters.wait_for_volume_replication_status("},{"line_number":172,"context_line":"            self.volumes_client, volume[\u0027id\u0027], \u0027enabled\u0027)"},{"line_number":173,"context_line":""}],"source_content_type":"text/x-python","patch_set":14,"id":"7181d49f_106aa32f","line":170,"range":{"start_line":168,"start_character":8,"end_line":170,"end_character":61},"in_reply_to":"d691a0b7_02433d67","updated":"2025-06-26 10:02:31.000000000","message":"That\u0027s why this test is placed under the scenario category—we\u0027re validating the entire workflow, not just the failback operation specifically.","commit_id":"2709819a02d36c719e19f679b6b2aa193035fa8d"},{"author":{"_account_id":27615,"name":"Rajat Dhasmana","email":"rajatdhasmana@gmail.com","username":"whoami-rajat"},"change_message_id":"524bfba316aacc7407bec085aae3efd03ab9bfdd","unresolved":true,"context_lines":[{"line_number":105,"context_line":"                keypair\u003dkeypair,"},{"line_number":106,"context_line":"                security_group\u003dsecurity_group"},{"line_number":107,"context_line":"            )"},{"line_number":108,"context_line":"            return server, volume"},{"line_number":109,"context_line":"        else:"},{"line_number":110,"context_line":"            server \u003d self.create_server("},{"line_number":111,"context_line":"                key_name\u003dkeypair[\u0027name\u0027],"}],"source_content_type":"text/x-python","patch_set":19,"id":"ba0343d2_d10b81e6","line":108,"range":{"start_line":108,"start_character":0,"end_line":108,"end_character":33},"updated":"2026-06-15 15:48:32.000000000","message":"nit: just assigning volume\u003dNone on L#87 and returning server, volume on L#114 (without the indent in else case) is more readable","commit_id":"5b68fc72c1e40d100c6fc5593dc7d5f425a2f878"},{"author":{"_account_id":19262,"name":"Liron Kuchlani","email":"lkuchlan@redhat.com","username":"lkuchlan"},"change_message_id":"fdc1d44bbdbff03788a7fc68d46191eafe78fa60","unresolved":false,"context_lines":[{"line_number":105,"context_line":"                keypair\u003dkeypair,"},{"line_number":106,"context_line":"                security_group\u003dsecurity_group"},{"line_number":107,"context_line":"            )"},{"line_number":108,"context_line":"            return server, volume"},{"line_number":109,"context_line":"        else:"},{"line_number":110,"context_line":"            server \u003d self.create_server("},{"line_number":111,"context_line":"                key_name\u003dkeypair[\u0027name\u0027],"}],"source_content_type":"text/x-python","patch_set":19,"id":"d30e6b54_2afa0c49","line":108,"range":{"start_line":108,"start_character":0,"end_line":108,"end_character":33},"in_reply_to":"ba0343d2_d10b81e6","updated":"2026-06-16 12:42:21.000000000","message":"Done","commit_id":"5b68fc72c1e40d100c6fc5593dc7d5f425a2f878"},{"author":{"_account_id":27615,"name":"Rajat Dhasmana","email":"rajatdhasmana@gmail.com","username":"whoami-rajat"},"change_message_id":"524bfba316aacc7407bec085aae3efd03ab9bfdd","unresolved":true,"context_lines":[{"line_number":146,"context_line":"            waiters.wait_for_volume_resource_status("},{"line_number":147,"context_line":"                self.volumes_client, volume[\u0027id\u0027], \u0027in-use\u0027)"},{"line_number":148,"context_line":""},{"line_number":149,"context_line":"        # Write data before failover"},{"line_number":150,"context_line":"        timestamp_before_failover \u003d self.create_timestamp("},{"line_number":151,"context_line":"            server_ip, private_key\u003dkeypair[\u0027private_key\u0027], server\u003dserver)"},{"line_number":152,"context_line":""},{"line_number":153,"context_line":"        # Perform failover"},{"line_number":154,"context_line":"        self.admin_volume_services_client.failover_host("},{"line_number":155,"context_line":"            host\u003dself.replication_host)"},{"line_number":156,"context_line":"        waiters.wait_for_volume_replication_status("},{"line_number":157,"context_line":"            self.volumes_client, volume[\u0027id\u0027], \u0027failed-over\u0027)"},{"line_number":158,"context_line":""},{"line_number":159,"context_line":"        # Validate data still accessible after failover"},{"line_number":160,"context_line":"        timestamp_after_failover \u003d self.get_timestamp("}],"source_content_type":"text/x-python","patch_set":19,"id":"618e86db_a00765f1","line":157,"range":{"start_line":149,"start_character":0,"end_line":157,"end_character":61},"updated":"2026-06-15 15:48:32.000000000","message":"I tested this locally to understand what\u0027s really happening and here are my findings\n\n1. The directory used to create the timestamp file is configured via target_dir defaulting to /tmp so the external attach volume case isn\u0027t valid for this scenario\n2. Even after failover I can see we can still fetch the timestamp for BFV with some I/O errors so we need a better mechanism to check this\n\nBEFORE FAILOVER\n---------------\n\n**TEMPEST**\n\ncode::\n\n    (venv) stack@rbd-repl-v3:~/cinder-tempest-plugin$ python3 -m testtools.run cinder_tempest_plugin.scenario.test_volume_replication.VolumeReplicationTest.test_failover_bfv_server\n    Tests running...\n    \u003e /opt/stack/cinder-tempest-plugin/cinder_tempest_plugin/scenario/test_volume_replication.py(155)_test_failover_scenario()\n    -\u003e self.admin_volume_services_client.failover_host(\n    (Pdb) \n\n\n**VOLUME**\n\ncode::\n\n\t(venv) stack@rbd-repl-v3:~/glance_store$ openstack volume show ba5ffbd0-35dc-46a7-8ea6-de0d0bf361a0 -c name -c status -c replication_status\n\t+--------------------+-------------------------------------------------------+\n\t| Field              | Value                                                 |\n\t+--------------------+-------------------------------------------------------+\n\t| name               | tempest-VolumeReplicationTest-volume-origin-757199014 |\n\t| replication_status | enabled                                               |\n\t| status             | in-use                                                |\n\t+--------------------+-------------------------------------------------------+\n\n\n**VM**\n\n\t$ lsblk\n\tNAME    MAJ:MIN RM  SIZE RO TYPE MOUNTPOINTS\n\tvda     252:0    0    1G  0 disk \n\t|-vda1  252:1    0 1015M  0 part /\n\t`-vda15 252:15   0    8M  0 part \n\n\t$ sudo cat /tmp/timestamp\n\tMon Jun 15 12:47:09 UTC 2026\n\n    # WRITING TO A FILE WORKS\n\t$ sudo dd if\u003d/dev/urandom of\u003d/home/cirros/test-file bs\u003d1M count\u003d10\n\t10+0 records in\n\t10+0 records out\n\nAFTER FAILOVER\n--------------\n\n**VOLUME**\n\n\t(venv) stack@rbd-repl-v3:~/glance_store$ openstack volume show ba5ffbd0-35dc-46a7-8ea6-de0d0bf361a0 -c name -c status -c replication_status\n\t+--------------------+-------------------------------------------------------+\n\t| Field              | Value                                                 |\n\t+--------------------+-------------------------------------------------------+\n\t| name               | tempest-VolumeReplicationTest-volume-origin-757199014 |\n\t| replication_status | failed-over                                           |\n\t| status             | in-use                                                |\n\t+--------------------+-------------------------------------------------------+\n\n**VM**\n\n\t$ [  267.270875] blk_update_request: I/O error, dev vda, sector 18440 op 0x1:(WRITE) flags 0x103000 phys_seg 1 prio class 0\n\t[  267.274695] Buffer I/O error on dev vda1, logical block 1, lost async page write\n\t[  267.319876] blk_update_request: I/O error, dev vda, sector 18616 op 0x1:(WRITE) flags 0x103000 phys_seg 1 prio class 0\n\t[  267.323864] Buffer I/O error on dev vda1, logical block 23, lost async page write\n\t[  267.361256] blk_update_request: I/O error, dev vda, sector 18496 op 0x1:(WRITE) flags 0x103000 phys_seg 2 prio class 0\n\n\n    # WRITING TO FILE FAILS\n\n\t$ sudo dd if\u003d/dev/urandom of\u003d/home/cirros/test-file-failed-over bs\u003d1M count\u003d10\n\t[  348.304752] EXT4-fs (vda1): Remounting filesystem read-only\n\tJun 15 12:52:32 cirros authpriv.notice sudo:   cirros : TTY\u003dttyS0 ; PWD\u003d/home/cirros ; USER\u003droot ; COMMAND\u003d/bin/dd if\u003d/dev/urandom of\u003d/home/cirros/test-file-failed-over bs\u003d1M count\u003d10\n\tdd: can\u0027t open \u0027/home/cirros/test-file-failed-over\u0027: Read-only file system\t\n\t\n    # TIMESTAMP STILL ACCESSIBLE\n\t$ sudo cat /tmp/timestamp \n\tJun 15 12:53:15 cirros authpriv.notice sudo:   cirros : TTY\u003dttyS0 ; PWD\u003d/home/cirros ; USER\u003droot ; COMMAND\u003d/bin/cat /tmp/timestamp\n\tMon Jun 15 12:47:09 UTC 2026\n\nAFTER FAILBACK\n--------------\n\n**VOLUME**\n\n\t(venv) stack@rbd-repl-v3:~/glance_store$ openstack volume show ba5ffbd0-35dc-46a7-8ea6-de0d0bf361a0 -c name -c status -c replication_status\n\t+--------------------+-------------------------------------------------------+\n\t| Field              | Value                                                 |\n\t+--------------------+-------------------------------------------------------+\n\t| name               | tempest-VolumeReplicationTest-volume-origin-757199014 |\n\t| replication_status | enabled                                               |\n\t| status             | in-use                                                |\n\t+--------------------+-------------------------------------------------------+\n\n**VM**\n\n    # WRITING TO FILE FAILS\n\n\t$ sudo dd if\u003d/dev/urandom of\u003d/home/cirros/test-file-failed-back bs\u003d1M count\u003d10\n\tJun 15 12:57:35 cirros authpriv.notice sudo:   cirros : TTY\u003dttyS0 ; PWD\u003d/home/cirros ; USER\u003droot ; COMMAND\u003d/bin/dd if\u003d/dev/urandom of\u003d/home/cirros/test-file-failed-back bs\u003d1M count\u003d10\n\tdd: can\u0027t open \u0027/home/cirros/test-file-failed-back\u0027: Read-only file system\n\n\n    # TIMESTAMP STILL ACCESSIBLE\n\n\t$ sudo cat /tmp/timestamp \n\tJun 15 12:57:20 cirros authpriv.notice sudo:   cirros : TTY\u003dttyS0 ; PWD\u003d/home/cirros ; USER\u003droot ; COMMAND\u003d/bin/cat /tmp/timestamp\n\tMon Jun 15 12:47:09 UTC 2026\n\nhttps://opendev.org/openstack/tempest/src/branch/master/tempest/config.py#L1276","commit_id":"5b68fc72c1e40d100c6fc5593dc7d5f425a2f878"},{"author":{"_account_id":19262,"name":"Liron Kuchlani","email":"lkuchlan@redhat.com","username":"lkuchlan"},"change_message_id":"fdc1d44bbdbff03788a7fc68d46191eafe78fa60","unresolved":false,"context_lines":[{"line_number":146,"context_line":"            waiters.wait_for_volume_resource_status("},{"line_number":147,"context_line":"                self.volumes_client, volume[\u0027id\u0027], \u0027in-use\u0027)"},{"line_number":148,"context_line":""},{"line_number":149,"context_line":"        # Write data before failover"},{"line_number":150,"context_line":"        timestamp_before_failover \u003d self.create_timestamp("},{"line_number":151,"context_line":"            server_ip, private_key\u003dkeypair[\u0027private_key\u0027], server\u003dserver)"},{"line_number":152,"context_line":""},{"line_number":153,"context_line":"        # Perform failover"},{"line_number":154,"context_line":"        self.admin_volume_services_client.failover_host("},{"line_number":155,"context_line":"            host\u003dself.replication_host)"},{"line_number":156,"context_line":"        waiters.wait_for_volume_replication_status("},{"line_number":157,"context_line":"            self.volumes_client, volume[\u0027id\u0027], \u0027failed-over\u0027)"},{"line_number":158,"context_line":""},{"line_number":159,"context_line":"        # Validate data still accessible after failover"},{"line_number":160,"context_line":"        timestamp_after_failover \u003d self.get_timestamp("}],"source_content_type":"text/x-python","patch_set":19,"id":"41a423a1_46f84dd6","line":157,"range":{"start_line":149,"start_character":0,"end_line":157,"end_character":61},"in_reply_to":"618e86db_a00765f1","updated":"2026-06-16 12:42:21.000000000","message":"Acknowledged","commit_id":"5b68fc72c1e40d100c6fc5593dc7d5f425a2f878"},{"author":{"_account_id":27615,"name":"Rajat Dhasmana","email":"rajatdhasmana@gmail.com","username":"whoami-rajat"},"change_message_id":"524bfba316aacc7407bec085aae3efd03ab9bfdd","unresolved":true,"context_lines":[{"line_number":187,"context_line":"    def test_failover_attached_volume(self):"},{"line_number":188,"context_line":"        \"\"\"Test failover with volume attached after booting from image.\"\"\""},{"line_number":189,"context_line":"        self._test_failover_scenario("},{"line_number":190,"context_line":"            boot_from_volume\u003dFalse, attach_after_boot\u003dTrue)"}],"source_content_type":"text/x-python","patch_set":19,"id":"3a24c253_1954157f","line":190,"range":{"start_line":190,"start_character":12,"end_line":190,"end_character":34},"updated":"2026-06-15 15:48:32.000000000","message":"nit: default value is already false so we don\u0027t need to pass this","commit_id":"5b68fc72c1e40d100c6fc5593dc7d5f425a2f878"},{"author":{"_account_id":19262,"name":"Liron Kuchlani","email":"lkuchlan@redhat.com","username":"lkuchlan"},"change_message_id":"fdc1d44bbdbff03788a7fc68d46191eafe78fa60","unresolved":false,"context_lines":[{"line_number":187,"context_line":"    def test_failover_attached_volume(self):"},{"line_number":188,"context_line":"        \"\"\"Test failover with volume attached after booting from image.\"\"\""},{"line_number":189,"context_line":"        self._test_failover_scenario("},{"line_number":190,"context_line":"            boot_from_volume\u003dFalse, attach_after_boot\u003dTrue)"}],"source_content_type":"text/x-python","patch_set":19,"id":"99739bdf_2e7562b2","line":190,"range":{"start_line":190,"start_character":12,"end_line":190,"end_character":34},"in_reply_to":"3a24c253_1954157f","updated":"2026-06-16 12:42:21.000000000","message":"Done","commit_id":"5b68fc72c1e40d100c6fc5593dc7d5f425a2f878"},{"author":{"_account_id":27615,"name":"Rajat Dhasmana","email":"rajatdhasmana@gmail.com","username":"whoami-rajat"},"change_message_id":"f4a007b56ebbe28e6e6e1bc7a88972d3f01f8d00","unresolved":false,"context_lines":[{"line_number":103,"context_line":"            self.volumes_client, volume[\u0027id\u0027], \u0027available\u0027)"},{"line_number":104,"context_line":"        waiters.wait_for_volume_attachment_remove_from_server("},{"line_number":105,"context_line":"            self.servers_client, server[\u0027id\u0027], volume[\u0027id\u0027])"},{"line_number":106,"context_line":"        self.attach_volume(server, volume)"},{"line_number":107,"context_line":""},{"line_number":108,"context_line":"    def _shelve_then_unshelve_server(self, server):"},{"line_number":109,"context_line":"        compute.shelve_server(self.servers_client, server[\u0027id\u0027],"}],"source_content_type":"text/x-python","patch_set":20,"id":"463dfb0b_0ba10810","line":106,"range":{"start_line":106,"start_character":0,"end_line":106,"end_character":42},"updated":"2026-06-16 21:55:39.000000000","message":"Note for reviewers: I can see we wait for the in-use status so this should be fine\n\nhttps://opendev.org/openstack/cinder-tempest-plugin/src/branch/master/cinder_tempest_plugin/scenario/manager.py#L225-L226","commit_id":"f7129e8ca49505f96482275cf07bd75c7ae4be6e"},{"author":{"_account_id":27615,"name":"Rajat Dhasmana","email":"rajatdhasmana@gmail.com","username":"whoami-rajat"},"change_message_id":"f4a007b56ebbe28e6e6e1bc7a88972d3f01f8d00","unresolved":false,"context_lines":[{"line_number":106,"context_line":"        self.attach_volume(server, volume)"},{"line_number":107,"context_line":""},{"line_number":108,"context_line":"    def _shelve_then_unshelve_server(self, server):"},{"line_number":109,"context_line":"        compute.shelve_server(self.servers_client, server[\u0027id\u0027],"},{"line_number":110,"context_line":"                              force_shelve_offload\u003dTrue)"},{"line_number":111,"context_line":"        self.servers_client.unshelve_server(server[\u0027id\u0027])"},{"line_number":112,"context_line":"        waiters.wait_for_server_status("}],"source_content_type":"text/x-python","patch_set":20,"id":"0f8e4d6d_24b65fc4","line":109,"range":{"start_line":109,"start_character":0,"end_line":109,"end_character":64},"updated":"2026-06-16 21:55:39.000000000","message":"this also has a waiter so looks good\n\nhttps://opendev.org/openstack/tempest/src/branch/master/tempest/common/compute.py#L389-L400","commit_id":"f7129e8ca49505f96482275cf07bd75c7ae4be6e"},{"author":{"_account_id":27615,"name":"Rajat Dhasmana","email":"rajatdhasmana@gmail.com","username":"whoami-rajat"},"change_message_id":"f4a007b56ebbe28e6e6e1bc7a88972d3f01f8d00","unresolved":false,"context_lines":[{"line_number":174,"context_line":""},{"line_number":175,"context_line":"        # Verify secondary cluster and data integrity after failover"},{"line_number":176,"context_line":"        secondary_cluster_name \u003d self._get_attachment_cluster_name(volume)"},{"line_number":177,"context_line":"        self.assertNotEqual(primary_cluster_name, secondary_cluster_name)"},{"line_number":178,"context_line":"        timestamp_after_failover \u003d self.get_timestamp("},{"line_number":179,"context_line":"            server_ip, dev_name\u003dvolume_device_name,"},{"line_number":180,"context_line":"            private_key\u003dkeypair[\u0027private_key\u0027], server\u003dserver)"}],"source_content_type":"text/x-python","patch_set":20,"id":"f1860ffe_e6bace45","line":177,"range":{"start_line":177,"start_character":0,"end_line":177,"end_character":73},"updated":"2026-06-16 21:55:39.000000000","message":"this confirms that the cluster name changed before/after failover","commit_id":"f7129e8ca49505f96482275cf07bd75c7ae4be6e"},{"author":{"_account_id":27615,"name":"Rajat Dhasmana","email":"rajatdhasmana@gmail.com","username":"whoami-rajat"},"change_message_id":"f4a007b56ebbe28e6e6e1bc7a88972d3f01f8d00","unresolved":true,"context_lines":[{"line_number":181,"context_line":"        self.assertEqual(timestamp_on_primary, timestamp_after_failover)"},{"line_number":182,"context_line":""},{"line_number":183,"context_line":"        # Skip the secondary write for the attached-volume case:"},{"line_number":184,"context_line":"        # detach/reattach takes ~10s, which is too short for Ceph\u0027s async"},{"line_number":185,"context_line":"        # rbd-mirror journal replay (ceph2 → ceph) to complete after failback."},{"line_number":186,"context_line":"        # Shelve/unshelve used for BFV takes ~60-120s, giving rbd-mirror"},{"line_number":187,"context_line":"        # enough time to finish."},{"line_number":188,"context_line":"        timestamp_on_secondary \u003d None"}],"source_content_type":"text/x-python","patch_set":20,"id":"3092be51_baf4ab34","line":185,"range":{"start_line":184,"start_character":8,"end_line":185,"end_character":62},"updated":"2026-06-16 21:55:39.000000000","message":"what happens if we execute sync command before the failback? does that push data to disk and trigger journal write to primary rbd image?","commit_id":"f7129e8ca49505f96482275cf07bd75c7ae4be6e"},{"author":{"_account_id":19262,"name":"Liron Kuchlani","email":"lkuchlan@redhat.com","username":"lkuchlan"},"change_message_id":"82c33300ae133d8f2b79f3aa2f86bbfc2da9759d","unresolved":true,"context_lines":[{"line_number":181,"context_line":"        self.assertEqual(timestamp_on_primary, timestamp_after_failover)"},{"line_number":182,"context_line":""},{"line_number":183,"context_line":"        # Skip the secondary write for the attached-volume case:"},{"line_number":184,"context_line":"        # detach/reattach takes ~10s, which is too short for Ceph\u0027s async"},{"line_number":185,"context_line":"        # rbd-mirror journal replay (ceph2 → ceph) to complete after failback."},{"line_number":186,"context_line":"        # Shelve/unshelve used for BFV takes ~60-120s, giving rbd-mirror"},{"line_number":187,"context_line":"        # enough time to finish."},{"line_number":188,"context_line":"        timestamp_on_secondary \u003d None"}],"source_content_type":"text/x-python","patch_set":20,"id":"fc96793f_83e9c6ca","line":185,"range":{"start_line":184,"start_character":8,"end_line":185,"end_character":62},"in_reply_to":"3092be51_baf4ab34","updated":"2026-06-17 06:59:40.000000000","message":"sync flushes the page cache to ceph2\u0027s RBD journal, but rbd-mirror still replays that journal to the primary asynchronously — failover_host doesn\u0027t wait for it.\nthe race remains","commit_id":"f7129e8ca49505f96482275cf07bd75c7ae4be6e"},{"author":{"_account_id":27615,"name":"Rajat Dhasmana","email":"rajatdhasmana@gmail.com","username":"whoami-rajat"},"change_message_id":"3292d309009979edf678084922dc99017559065b","unresolved":false,"context_lines":[{"line_number":181,"context_line":"        self.assertEqual(timestamp_on_primary, timestamp_after_failover)"},{"line_number":182,"context_line":""},{"line_number":183,"context_line":"        # Skip the secondary write for the attached-volume case:"},{"line_number":184,"context_line":"        # detach/reattach takes ~10s, which is too short for Ceph\u0027s async"},{"line_number":185,"context_line":"        # rbd-mirror journal replay (ceph2 → ceph) to complete after failback."},{"line_number":186,"context_line":"        # Shelve/unshelve used for BFV takes ~60-120s, giving rbd-mirror"},{"line_number":187,"context_line":"        # enough time to finish."},{"line_number":188,"context_line":"        timestamp_on_secondary \u003d None"}],"source_content_type":"text/x-python","patch_set":20,"id":"a817d421_5f52f99b","line":185,"range":{"start_line":184,"start_character":8,"end_line":185,"end_character":62},"in_reply_to":"fc96793f_83e9c6ca","updated":"2026-06-18 07:47:40.000000000","message":"Acknowledged","commit_id":"f7129e8ca49505f96482275cf07bd75c7ae4be6e"},{"author":{"_account_id":27615,"name":"Rajat Dhasmana","email":"rajatdhasmana@gmail.com","username":"whoami-rajat"},"change_message_id":"f4a007b56ebbe28e6e6e1bc7a88972d3f01f8d00","unresolved":true,"context_lines":[{"line_number":156,"context_line":"        # Record primary cluster and write timestamp before failover"},{"line_number":157,"context_line":"        primary_cluster_name \u003d self._get_attachment_cluster_name(volume)"},{"line_number":158,"context_line":"        timestamp_on_primary \u003d self.create_timestamp("},{"line_number":159,"context_line":"            server_ip, dev_name\u003dvolume_device_name,"},{"line_number":160,"context_line":"            private_key\u003dkeypair[\u0027private_key\u0027], server\u003dserver)"},{"line_number":161,"context_line":""},{"line_number":162,"context_line":"        # Failover to secondary"}],"source_content_type":"text/x-python","patch_set":21,"id":"b7f75954_061cd1c7","line":159,"range":{"start_line":159,"start_character":23,"end_line":159,"end_character":50},"updated":"2026-06-16 21:55:39.000000000","message":"So this was really confusing for me at first, since i saw nothing on the mountpath /mnt/vdb\n\n    # ls -alh /mnt/vdb/\n    total 8K     \n    drwx------    2 root     root        4.0K Jun 16 19:48 .\n    drwxr-xr-x    3 root     root        4.0K Jun 16 19:48 ..\n\nAlso lsblk didn\u0027t show any mount\n\n    # lsblk --fs\n    NAME FSTYPE FSVER LABEL UUID                                 FSAVAIL FSUSE% MOUNTPOINTS\n    vda\n    |-vda1\n    |    ext3         cirros-rootfs\n    |                       30b4ceb1-96d7-4458-aee6-c4153e79e96e  885.2M     5% /\n    `-vda15\n         vfat               AFB7-FBE3\n    vdb  vfat               6A31-9A64\n\nBut later i realized that the mounting/unmounting happens inside the create_timestamp method so it is only visible only for a particular duration\n\n    # lsblk --fs\n    NAME FSTYPE FSVER LABEL UUID                                 FSAVAIL FSUSE% MOUNTPOINTS\n    vda\n    |-vda1\n    |    ext3         cirros-rootfs\n    |                       30b4ceb1-96d7-4458-aee6-c4153e79e96e  885.2M     5% /\n    `-vda15\n         vfat               AFB7-FBE3\n    vdb  vfat               6A31-9A64                              1022M     0% /mnt/vdb\n\n    # cat /mnt/vdb/timestamp \n    Tue Jun 16 19:48:04 UTC 2026","commit_id":"c63a0d5919c91c22cd2fafec577972b4a61cb36d"},{"author":{"_account_id":27615,"name":"Rajat Dhasmana","email":"rajatdhasmana@gmail.com","username":"whoami-rajat"},"change_message_id":"3292d309009979edf678084922dc99017559065b","unresolved":false,"context_lines":[{"line_number":156,"context_line":"        # Record primary cluster and write timestamp before failover"},{"line_number":157,"context_line":"        primary_cluster_name \u003d self._get_attachment_cluster_name(volume)"},{"line_number":158,"context_line":"        timestamp_on_primary \u003d self.create_timestamp("},{"line_number":159,"context_line":"            server_ip, dev_name\u003dvolume_device_name,"},{"line_number":160,"context_line":"            private_key\u003dkeypair[\u0027private_key\u0027], server\u003dserver)"},{"line_number":161,"context_line":""},{"line_number":162,"context_line":"        # Failover to secondary"}],"source_content_type":"text/x-python","patch_set":21,"id":"0b77d64e_fd9ace00","line":159,"range":{"start_line":159,"start_character":23,"end_line":159,"end_character":50},"in_reply_to":"b7f75954_061cd1c7","updated":"2026-06-18 07:47:40.000000000","message":"Done","commit_id":"c63a0d5919c91c22cd2fafec577972b4a61cb36d"}]}