)]}'
{"/PATCHSET_LEVEL":[{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"862081812a675b5293aa89bd1093289f4d62a7ac","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":2,"id":"ccf7e04d_8ac30cb7","updated":"2025-09-19 17:18:40.000000000","message":"(just the beginning of a review, I ran out of time for more today)\n\nI think change to the diskfile behaviour is necesssary, but not sufficient to prevent sets fo durable frags with conflicting etags. That\u0027s because it is still possible for a non-durable frag to be written, and if there are other durables with the same timestamp then ssync will mark the non-durable as durable.\n\nSo this patch is an improvement in that we won\u0027t return a 201 to a client unless a durable *and consistent* frag set is written. But we\u0027ll need more work to ensure that non-durable frags are guaranteed to be or become consistent.","commit_id":"ef79b71af3da708757d905b79040184d77b2950b"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a96ec59e76f209bd9b8f0a0fed8096ac0ba8573e","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":2,"id":"aa997bd8_2899670b","updated":"2025-09-19 22:23:04.000000000","message":"we should squash this","commit_id":"ef79b71af3da708757d905b79040184d77b2950b"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a96ec59e76f209bd9b8f0a0fed8096ac0ba8573e","unresolved":true,"context_lines":[],"source_content_type":"","patch_set":2,"id":"af478d90_dc81f724","in_reply_to":"ccf7e04d_8ac30cb7","updated":"2025-09-19 22:23:04.000000000","message":"that\u0027s really nice prose - i\u0027m going to steal it for the commit message!","commit_id":"ef79b71af3da708757d905b79040184d77b2950b"}],"test/probe/test_timestamp_collision.py":[{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"862081812a675b5293aa89bd1093289f4d62a7ac","unresolved":true,"context_lines":[{"line_number":108,"context_line":"        self.container_name \u003d \u0027badtest\u0027"},{"line_number":109,"context_line":"        self.object_name \u003d \u0027badnews\u0027"},{"line_number":110,"context_line":"        self.swift \u003d internal_client.InternalClient("},{"line_number":111,"context_line":"            \u0027/etc/swift/internal-client.conf\u0027, \u0027probe-test\u0027, 3)"},{"line_number":112,"context_line":"        self.swift.create_container("},{"line_number":113,"context_line":"            self.account, self.container_name,"},{"line_number":114,"context_line":"            headers\u003d{\u0027x-storage-policy\u0027: self.policy.name})"}],"source_content_type":"text/x-python","patch_set":2,"id":"f9855e35_035713b5","line":111,"updated":"2025-09-19 17:18:40.000000000","message":"re. my comment in earlier patch, this should be 1 not 3 - now we\u0027re having 500s come back from object server the FragZipper is blowing up during the retries because there\u0027s more than 2 unique requests but FragZipper can only handle 2:\n\n```\nvagrant@saio:~$ cat /var/log/syslog|grep \u0027swift:\u0027 | grep \u0027badnews\u0027\nSep 19 14:31:09 saio swift: ERROR 500 b\u0027\u0027 Trying to PUT /v1/AUTH_test/badtest/badnews From Object Server 127.0.0.3:6030/sdb7 (txn: txd0cae4a942864bebbe750-0068cd692b)\nSep 19 14:31:09 saio swift: ERROR 500 b\u0027\u0027 Trying to PUT /v1/AUTH_test/badtest/badnews From Object Server 127.0.0.1:6010/sdb1 (txn: txd0cae4a942864bebbe750-0068cd692b)\nSep 19 14:31:09 saio swift: ERROR 500 b\u0027\u0027 Trying to PUT /v1/AUTH_test/badtest/badnews From Object Server 127.0.0.1:6010/sdb5 (txn: txd0cae4a942864bebbe750-0068cd692b)\nSep 19 14:31:09 saio swift: - - 19/Sep/2025/14/31/09 PUT /v1/AUTH_test/badtest/badnews HTTP/1.0 503 - probe-test - 2845728 118 - txd0cae4a942864bebbe750-0068cd692b - 1.9391 - - 1758292267.170501947 1758292269.109625816 1 -\nSep 19 14:31:09 saio swift: ERROR 500 b\u0027\u0027 Trying to PUT /v1/AUTH_test/badtest/badnews From Object Server 127.0.0.2:6020/sdb2 (txn: txb27fa4e066a0405681de6-0068cd692b)\nSep 19 14:31:09 saio swift: ERROR 500 b\u0027\u0027 Trying to PUT /v1/AUTH_test/badtest/badnews From Object Server 127.0.0.2:6020/sdb6 (txn: txb27fa4e066a0405681de6-0068cd692b)\nSep 19 14:31:09 saio swift: ERROR 500 b\u0027\u0027 Trying to PUT /v1/AUTH_test/badtest/badnews From Object Server 127.0.0.4:6040/sdb4 (txn: txb27fa4e066a0405681de6-0068cd692b)\nSep 19 14:31:09 saio swift: - - 19/Sep/2025/14/31/09 PUT /v1/AUTH_test/badtest/badnews HTTP/1.0 503 - probe-test - 2845728 118 - txb27fa4e066a0405681de6-0068cd692b - 2.5432 - - 1758292267.169957638 1758292269.713178158 1 -\nSep 19 14:31:11 saio swift: ERROR Exception transferring data to object servers {\u0027path\u0027: \u0027/v1/AUTH_test/badtest/badnews\u0027}: #012Traceback (most recent call last):#012  File \"/vagrant/swift/swift/proxy/controllers/obj.py\", line 3200, in _transfer_data#012    putter.end_of_object_data(footer_metadata\u003dtrail_md)#012  File \"/vagrant/swift/test/probe/test_timestamp_collision.py\", line 270, in patched_end_of_object_data#012    zipper.wait(putter.__tracking_id, fi)#012  File \"/vagrant/swift/test/probe/test_timestamp_collision.py\", line 77, in wait#012    self.sent[my_id].add(fi)#012IndexError: list index out of range (txn: txe127e7aa1bf94b44ba294-0068cd692f)\nSep 19 14:31:11 saio swift: - - 19/Sep/2025/14/31/11 PUT /v1/AUTH_test/badtest/badnews HTTP/1.0 500 - probe-test - 2845728 125 - txe127e7aa1bf94b44ba294-0068cd692f - 0.4518 - - 1758292271.115623236 1758292271.567421436 1 -\nSep 19 14:31:12 saio swift: ERROR Exception transferring data to object servers {\u0027path\u0027: \u0027/v1/AUTH_test/badtest/badnews\u0027}: #012Traceback (most recent call last):#012  File \"/vagrant/swift/swift/proxy/controllers/obj.py\", line 3200, in _transfer_data#012    putter.end_of_object_data(footer_metadata\u003dtrail_md)#012  File \"/vagrant/swift/test/probe/test_timestamp_collision.py\", line 270, in patched_end_of_object_data#012    zipper.wait(putter.__tracking_id, fi)#012  File \"/vagrant/swift/test/probe/test_timestamp_collision.py\", line 77, in wait#012    self.sent[my_id].add(fi)#012IndexError: list index out of range (txn: txca74c6dafa7548e988a20-0068cd692f)\nSep 19 14:31:12 saio swift: - - 19/Sep/2025/14/31/12 PUT /v1/AUTH_test/badtest/badnews HTTP/1.0 500 - probe-test - 2845728 125 - txca74c6dafa7548e988a20-0068cd692f - 0.3839 - - 1758292271.715099335 1758292272.099041939 1 -\nSep 19 14:31:15 saio swift: ERROR Exception transferring data to object servers {\u0027path\u0027: \u0027/v1/AUTH_test/badtest/badnews\u0027}: #012Traceback (most recent call last):#012  File \"/vagrant/swift/swift/proxy/controllers/obj.py\", line 3200, in _transfer_data#012    putter.end_of_object_data(footer_metadata\u003dtrail_md)#012  File \"/vagrant/swift/test/probe/test_timestamp_collision.py\", line 270, in patched_end_of_object_data#012    zipper.wait(putter.__tracking_id, fi)#012  File \"/vagrant/swift/test/probe/test_timestamp_collision.py\", line 77, in wait#012    self.sent[my_id].add(fi)#012IndexError: list index out of range (txn: txea65bdce4b7d49178db85-0068cd6933)\nSep 19 14:31:15 saio swift: - - 19/Sep/2025/14/31/15 PUT /v1/AUTH_test/badtest/badnews HTTP/1.0 500 - probe-test - 2845728 125 - txea65bdce4b7d49178db85-0068cd6933 - 0.3988 - - 1758292275.574983597 1758292275.973746777 1 -\nSep 19 14:31:16 saio swift: ERROR Exception transferring data to object servers {\u0027path\u0027: \u0027/v1/AUTH_test/badtest/badnews\u0027}: #012Traceback (most recent call last):#012  File \"/vagrant/swift/swift/proxy/controllers/obj.py\", line 3200, in _transfer_data#012    putter.end_of_object_data(footer_metadata\u003dtrail_md)#012  File \"/vagrant/swift/test/probe/test_timestamp_collision.py\", line 270, in patched_end_of_object_data#012    zipper.wait(putter.__tracking_id, fi)#012  File \"/vagrant/swift/test/probe/test_timestamp_collision.py\", line 77, in wait#012    self.sent[my_id].add(fi)#012IndexError: list index out of range (txn: txdd722ca78731496b9a7c8-0068cd6934)\nSep 19 14:31:16 saio swift: - - 19/Sep/2025/14/31/16 PUT /v1/AUTH_test/badtest/badnews HTTP/1.0 500 - probe-test - 2845728 125 - txdd722ca78731496b9a7c8-0068cd6934 - 0.3922 - - 1758292276.102040768 1758292276.494206190 1 -\n\n```","commit_id":"ef79b71af3da708757d905b79040184d77b2950b"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a96ec59e76f209bd9b8f0a0fed8096ac0ba8573e","unresolved":false,"context_lines":[{"line_number":108,"context_line":"        self.container_name \u003d \u0027badtest\u0027"},{"line_number":109,"context_line":"        self.object_name \u003d \u0027badnews\u0027"},{"line_number":110,"context_line":"        self.swift \u003d internal_client.InternalClient("},{"line_number":111,"context_line":"            \u0027/etc/swift/internal-client.conf\u0027, \u0027probe-test\u0027, 3)"},{"line_number":112,"context_line":"        self.swift.create_container("},{"line_number":113,"context_line":"            self.account, self.container_name,"},{"line_number":114,"context_line":"            headers\u003d{\u0027x-storage-policy\u0027: self.policy.name})"}],"source_content_type":"text/x-python","patch_set":2,"id":"579b87ea_9d3d6e29","line":111,"in_reply_to":"f9855e35_035713b5","updated":"2025-09-19 22:23:04.000000000","message":"Acknowledged","commit_id":"ef79b71af3da708757d905b79040184d77b2950b"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"862081812a675b5293aa89bd1093289f4d62a7ac","unresolved":true,"context_lines":[{"line_number":212,"context_line":"                self.do_upload(contents, headers)"},{"line_number":213,"context_line":"            self.assertEqual(503, ctx.exception.resp.status_int)"},{"line_number":214,"context_line":"            df2node \u003d self.map_data_file_to_node()"},{"line_number":215,"context_line":"            # nothing is durable yet"},{"line_number":216,"context_line":"            self.assertEqual({\u0027#%d.data\u0027 % i for i in range(6)},"},{"line_number":217,"context_line":"                             {k[-7:] for k in df2node})"},{"line_number":218,"context_line":"            # all the original metadata"}],"source_content_type":"text/x-python","patch_set":2,"id":"0db9e331_dc42248b","line":215,"updated":"2025-09-19 17:18:40.000000000","message":"I suggested some helpers for the all/none durable assertions in sq: test_timestamp_collision fixups  https://review.opendev.org/c/openstack/swift/+/961802 which is based on the previous patch","commit_id":"ef79b71af3da708757d905b79040184d77b2950b"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a96ec59e76f209bd9b8f0a0fed8096ac0ba8573e","unresolved":false,"context_lines":[{"line_number":212,"context_line":"                self.do_upload(contents, headers)"},{"line_number":213,"context_line":"            self.assertEqual(503, ctx.exception.resp.status_int)"},{"line_number":214,"context_line":"            df2node \u003d self.map_data_file_to_node()"},{"line_number":215,"context_line":"            # nothing is durable yet"},{"line_number":216,"context_line":"            self.assertEqual({\u0027#%d.data\u0027 % i for i in range(6)},"},{"line_number":217,"context_line":"                             {k[-7:] for k in df2node})"},{"line_number":218,"context_line":"            # all the original metadata"}],"source_content_type":"text/x-python","patch_set":2,"id":"e3262bea_fa8322ae","line":215,"in_reply_to":"0db9e331_dc42248b","updated":"2025-09-19 22:23:04.000000000","message":"Acknowledged","commit_id":"ef79b71af3da708757d905b79040184d77b2950b"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"862081812a675b5293aa89bd1093289f4d62a7ac","unresolved":true,"context_lines":[{"line_number":313,"context_line":"        self.assertTrue(any(key in all_found_keys for key in bar_keys),"},{"line_number":314,"context_line":"                        \"No metadata found for \u0027bar\u0027, expected one of %r, \""},{"line_number":315,"context_line":"                        \"got %r\" % (bar_keys, all_found_keys))"},{"line_number":316,"context_line":"        # nothing is durable"},{"line_number":317,"context_line":"        self.assertEqual("},{"line_number":318,"context_line":"            {\u0027#0.data\u0027, \u0027#1.data\u0027, \u0027#2.data\u0027, \u0027#3.data\u0027, \u0027#4.data\u0027, \u0027#5.data\u0027},"},{"line_number":319,"context_line":"            {k[-7:] for k in metadata.keys()}"}],"source_content_type":"text/x-python","patch_set":2,"id":"a47f7d16_000ae37a","line":316,"range":{"start_line":316,"start_character":8,"end_line":316,"end_character":28},"updated":"2025-09-19 17:18:40.000000000","message":"it would be really interesting to have a variation of this test where one request gets a quorum and the other doesn\u0027t, so we end up with 5 durables and one non-durable\n\n(perhaps FragZipper can be parameterised so it is biased to one request??)\n\nI have a horrible feeling that, when the reconstructor runs, the \u0027bad\u0027 non-durable won\u0027t be *replaced* by a good durable frag, but instead the bad non-durable will be *made durable* by ssync, NQA 😭 see https://github.com/openstack/swift/blob/1dc3307eaf0457e59337ca9a153c12aaade6696b/swift/obj/ssync_receiver.py#L389-L406","commit_id":"ef79b71af3da708757d905b79040184d77b2950b"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a96ec59e76f209bd9b8f0a0fed8096ac0ba8573e","unresolved":true,"context_lines":[{"line_number":313,"context_line":"        self.assertTrue(any(key in all_found_keys for key in bar_keys),"},{"line_number":314,"context_line":"                        \"No metadata found for \u0027bar\u0027, expected one of %r, \""},{"line_number":315,"context_line":"                        \"got %r\" % (bar_keys, all_found_keys))"},{"line_number":316,"context_line":"        # nothing is durable"},{"line_number":317,"context_line":"        self.assertEqual("},{"line_number":318,"context_line":"            {\u0027#0.data\u0027, \u0027#1.data\u0027, \u0027#2.data\u0027, \u0027#3.data\u0027, \u0027#4.data\u0027, \u0027#5.data\u0027},"},{"line_number":319,"context_line":"            {k[-7:] for k in metadata.keys()}"}],"source_content_type":"text/x-python","patch_set":2,"id":"5793a344_037c0d9e","line":316,"range":{"start_line":316,"start_character":8,"end_line":316,"end_character":28},"in_reply_to":"a47f7d16_000ae37a","updated":"2025-09-19 22:23:04.000000000","message":"I 100% agree - if we can make the FragZipper more flexible we should explicitly test [3, 3], [4, 2] AND [5, 1] collisions\n\n... the last one is interesting b/c as you said the *initially* non-durable frags in the collision set will become durable after the reconsturctor runs!  And the ndata+1 \"quorum\" of successful link_at is *super scary* b/c we\u0027d have returned 200 to one of those requests and it\u0027s possible the client won\u0027t retry the other failed upload that hit the conflict!","commit_id":"ef79b71af3da708757d905b79040184d77b2950b"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"862081812a675b5293aa89bd1093289f4d62a7ac","unresolved":true,"context_lines":[{"line_number":319,"context_line":"            {k[-7:] for k in metadata.keys()}"},{"line_number":320,"context_line":"        )"},{"line_number":321,"context_line":"        # both responses were errors"},{"line_number":322,"context_line":"        self.assertEqual([500, 500], [r.status_int for r in results])"},{"line_number":323,"context_line":""},{"line_number":324,"context_line":"    def test_overlap_writes_to_handoffs(self):"},{"line_number":325,"context_line":"        contents \u003d b\u0027a\u0027 * 97"}],"source_content_type":"text/x-python","patch_set":2,"id":"f997b1c6_4931bc8c","line":322,"range":{"start_line":322,"start_character":26,"end_line":322,"end_character":29},"updated":"2025-09-19 17:18:40.000000000","message":"these should be 503 but are 500 because the FragZipper is blowing up, see comment at line 111","commit_id":"ef79b71af3da708757d905b79040184d77b2950b"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a96ec59e76f209bd9b8f0a0fed8096ac0ba8573e","unresolved":false,"context_lines":[{"line_number":319,"context_line":"            {k[-7:] for k in metadata.keys()}"},{"line_number":320,"context_line":"        )"},{"line_number":321,"context_line":"        # both responses were errors"},{"line_number":322,"context_line":"        self.assertEqual([500, 500], [r.status_int for r in results])"},{"line_number":323,"context_line":""},{"line_number":324,"context_line":"    def test_overlap_writes_to_handoffs(self):"},{"line_number":325,"context_line":"        contents \u003d b\u0027a\u0027 * 97"}],"source_content_type":"text/x-python","patch_set":2,"id":"276d1236_4313ba5d","line":322,"range":{"start_line":322,"start_character":26,"end_line":322,"end_character":29},"in_reply_to":"f997b1c6_4931bc8c","updated":"2025-09-19 22:23:04.000000000","message":"good catch!","commit_id":"ef79b71af3da708757d905b79040184d77b2950b"}]}