)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":17245,"name":"Christopher Bartz","email":"bartz@dkrz.de","username":"cbartz"},"change_message_id":"a47f8fbb50fa5b5cc0ae3d6b70556b79ed9b1e1e","unresolved":false,"context_lines":[{"line_number":21,"context_line":"max_put_failures)."},{"line_number":22,"context_line":""},{"line_number":23,"context_line":"Example: given an EC 12+5 policy and 3 container replicas, you can"},{"line_number":24,"context_line":"lose up to 4 connections and still succeed. Since you need to have 2"},{"line_number":25,"context_line":"container updates happen for durability, you need 6 connections to"},{"line_number":26,"context_line":"have X-Container headers. That way, you can lose 4 and still have 2"},{"line_number":27,"context_line":"left. The current code would put X-Container headers on 14 of the"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":1,"id":"9f91af0f_5e53d23f","line":24,"range":{"start_line":24,"start_character":11,"end_line":24,"end_character":12},"updated":"2018-01-08 17:10:36.000000000","message":"5 ? At least with rs_vand","commit_id":"d8120bdfc5f055b555dc87bd9ca4230ff9a7ff0f"},{"author":{"_account_id":2622,"name":"Samuel Merritt","email":"spam+launchpad@andcheese.org","username":"torgomatic"},"change_message_id":"609f0a6a190e1985de5f720981b5d20d710f94f3","unresolved":false,"context_lines":[{"line_number":21,"context_line":"max_put_failures)."},{"line_number":22,"context_line":""},{"line_number":23,"context_line":"Example: given an EC 12+5 policy and 3 container replicas, you can"},{"line_number":24,"context_line":"lose up to 4 connections and still succeed. Since you need to have 2"},{"line_number":25,"context_line":"container updates happen for durability, you need 6 connections to"},{"line_number":26,"context_line":"have X-Container headers. That way, you can lose 4 and still have 2"},{"line_number":27,"context_line":"left. The current code would put X-Container headers on 14 of the"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":1,"id":"9f91af0f_d47583cc","line":24,"range":{"start_line":24,"start_character":11,"end_line":24,"end_character":12},"in_reply_to":"9f91af0f_5e53d23f","updated":"2018-01-08 18:21:27.000000000","message":"The proxy won\u0027t count it as success unless the object can withstand a single-disk failure. It\u0027s true that one could lose all 5 of the parity fragments and still get the object back, but Swift will return a 503 to the client once it loses the 5th.","commit_id":"d8120bdfc5f055b555dc87bd9ca4230ff9a7ff0f"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"3592e406251991afde83928d6dd37e8010b438de","unresolved":false,"context_lines":[{"line_number":21,"context_line":"max_put_failures)."},{"line_number":22,"context_line":""},{"line_number":23,"context_line":"Example: given an EC 12+5 policy and 3 container replicas, you can"},{"line_number":24,"context_line":"lose up to 4 connections and still succeed. Since you need to have 2"},{"line_number":25,"context_line":"container updates happen for durability, you need 6 connections to"},{"line_number":26,"context_line":"have X-Container headers. That way, you can lose 4 and still have 2"},{"line_number":27,"context_line":"left. The current code would put X-Container headers on 14 of the"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":1,"id":"9f91af0f_e701da6b","line":24,"range":{"start_line":24,"start_character":11,"end_line":24,"end_character":12},"in_reply_to":"9f91af0f_87d24e27","updated":"2018-01-09 19:41:01.000000000","message":"...and that\u0027s all coming out of the definition of quorum [1], _ec_quorum_size [2], and pyeclib\u0027s behavior for min_parity_fragments_needed [3]. It\u0027s all a horrible miscalculation (*especially* when we\u0027re doing duplicated EC IMO), but it\u0027s better to address that as a separate issue. Even when we *do*, we\u0027ll probably still come down on the side of not declaring the object durable unless we can withstand at least one drive failure.\n\n[1] https://github.com/openstack/swift/blob/2.16.0/swift/common/storage_policy.py#L579-L599\n\n[2] https://github.com/openstack/swift/blob/2.16.0/swift/common/storage_policy.py#L481-L483\n\n[3] https://github.com/openstack/pyeclib/blob/9a20711/pyeclib/core.py#L121-L123","commit_id":"d8120bdfc5f055b555dc87bd9ca4230ff9a7ff0f"},{"author":{"_account_id":17245,"name":"Christopher Bartz","email":"bartz@dkrz.de","username":"cbartz"},"change_message_id":"e24559a81b0d2d49040217c65b6cf75267d2f4d3","unresolved":false,"context_lines":[{"line_number":21,"context_line":"max_put_failures)."},{"line_number":22,"context_line":""},{"line_number":23,"context_line":"Example: given an EC 12+5 policy and 3 container replicas, you can"},{"line_number":24,"context_line":"lose up to 4 connections and still succeed. Since you need to have 2"},{"line_number":25,"context_line":"container updates happen for durability, you need 6 connections to"},{"line_number":26,"context_line":"have X-Container headers. That way, you can lose 4 and still have 2"},{"line_number":27,"context_line":"left. The current code would put X-Container headers on 14 of the"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":1,"id":"9f91af0f_f97bce24","line":24,"range":{"start_line":24,"start_character":11,"end_line":24,"end_character":12},"in_reply_to":"9f91af0f_d47583cc","updated":"2018-01-09 09:07:15.000000000","message":"I tried isa_l_rs_vand with 2+1 scheme and a total storage node count of 3. I shut down one server, but I was still able to PUT an object (so without any parity). Can you explain me, why this should be different for EC 12+5 ?","commit_id":"d8120bdfc5f055b555dc87bd9ca4230ff9a7ff0f"},{"author":{"_account_id":2622,"name":"Samuel Merritt","email":"spam+launchpad@andcheese.org","username":"torgomatic"},"change_message_id":"cb4be0114af7dec7a08daeb807648e147c3e5751","unresolved":false,"context_lines":[{"line_number":21,"context_line":"max_put_failures)."},{"line_number":22,"context_line":""},{"line_number":23,"context_line":"Example: given an EC 12+5 policy and 3 container replicas, you can"},{"line_number":24,"context_line":"lose up to 4 connections and still succeed. Since you need to have 2"},{"line_number":25,"context_line":"container updates happen for durability, you need 6 connections to"},{"line_number":26,"context_line":"have X-Container headers. That way, you can lose 4 and still have 2"},{"line_number":27,"context_line":"left. The current code would put X-Container headers on 14 of the"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":1,"id":"9f91af0f_87d24e27","line":24,"range":{"start_line":24,"start_character":11,"end_line":24,"end_character":12},"in_reply_to":"9f91af0f_f97bce24","updated":"2018-01-09 19:12:55.000000000","message":"If a storage node is down, the proxy will select another disk in a functioning storage node to use as a handoff, and it will store the data there.\n\nWere you to reduce your available drives down to 2, then PUTs would fail.","commit_id":"d8120bdfc5f055b555dc87bd9ca4230ff9a7ff0f"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"c3e248372ec54a27d9a465ec74b3033f348bb342","unresolved":false,"context_lines":[{"line_number":43,"context_line":"EC 15+4, 5x container     |    17     |      6        |"},{"line_number":44,"context_line":"EC 4+8, 3x container      |    6      |      9        |"},{"line_number":45,"context_line":"7x object, 3x container   |    5      |      5        |"},{"line_number":46,"context_line":"6x object, 3x container   |    4      |      5        |"},{"line_number":47,"context_line":""},{"line_number":48,"context_line":"Change-Id: I34efd48655b890340912810ab111bb63445e5c8b"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":2,"id":"9f91af0f_8bd91958","line":46,"updated":"2018-01-08 21:13:16.000000000","message":"I would *love* to have a test framework to demonstrate all of these.\n\nOther interesting cases:\n\n 2x object, 3x container\n 1x object, 3x container","commit_id":"a809425da5bf3e174e2e25aa68576e68674a8c16"}],"swift/proxy/controllers/obj.py":[{"author":{"_account_id":17245,"name":"Christopher Bartz","email":"bartz@dkrz.de","username":"cbartz"},"change_message_id":"a47f8fbb50fa5b5cc0ae3d6b70556b79ed9b1e1e","unresolved":false,"context_lines":[{"line_number":300,"context_line":"        # nodes. To always be durable, we have to send enough container"},{"line_number":301,"context_line":"        # updates so that, if only Qo object PUTs succeed, and all the"},{"line_number":302,"context_line":"        # failed object PUTs had container updates, at least Qc updates"},{"line_number":303,"context_line":"        # remain. Since (Ro - Qo) object PUTs failed, we must have at least"},{"line_number":304,"context_line":"        # Qc + Ro - Qo container updates to ensure that Qc of them remain."},{"line_number":305,"context_line":"        #"},{"line_number":306,"context_line":"        # Also, each container replica is named in at least one object PUT"}],"source_content_type":"text/x-python","patch_set":1,"id":"9f91af0f_60f71961","line":303,"range":{"start_line":303,"start_character":46,"end_line":303,"end_character":52},"updated":"2018-01-08 17:10:36.000000000","message":"may fail","commit_id":"d8120bdfc5f055b555dc87bd9ca4230ff9a7ff0f"},{"author":{"_account_id":2622,"name":"Samuel Merritt","email":"spam+launchpad@andcheese.org","username":"torgomatic"},"change_message_id":"609f0a6a190e1985de5f720981b5d20d710f94f3","unresolved":false,"context_lines":[{"line_number":300,"context_line":"        # nodes. To always be durable, we have to send enough container"},{"line_number":301,"context_line":"        # updates so that, if only Qo object PUTs succeed, and all the"},{"line_number":302,"context_line":"        # failed object PUTs had container updates, at least Qc updates"},{"line_number":303,"context_line":"        # remain. Since (Ro - Qo) object PUTs failed, we must have at least"},{"line_number":304,"context_line":"        # Qc + Ro - Qo container updates to ensure that Qc of them remain."},{"line_number":305,"context_line":"        #"},{"line_number":306,"context_line":"        # Also, each container replica is named in at least one object PUT"}],"source_content_type":"text/x-python","patch_set":1,"id":"9f91af0f_d42343bc","line":303,"range":{"start_line":303,"start_character":46,"end_line":303,"end_character":52},"in_reply_to":"9f91af0f_60f71961","updated":"2018-01-08 18:21:27.000000000","message":"Done","commit_id":"d8120bdfc5f055b555dc87bd9ca4230ff9a7ff0f"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"37a275dcfe07b64d641a98a11a28b69dcb0ffa07","unresolved":false,"context_lines":[{"line_number":308,"context_line":"        # request so that, when all requests succeed, no work is generated"},{"line_number":309,"context_line":"        # for the container replicator. Thus, at least Rc updates are"},{"line_number":310,"context_line":"        # necessary."},{"line_number":311,"context_line":"        n_updates_needed \u003d max("},{"line_number":312,"context_line":"            # Qc + Ro - Qo"},{"line_number":313,"context_line":"            quorum_size(len(containers)) + n_outgoing - policy.quorum,"},{"line_number":314,"context_line":"            # Rc"}],"source_content_type":"text/x-python","patch_set":2,"id":"9f91af0f_714da498","line":311,"updated":"2018-01-10 23:21:17.000000000","message":"I can see three reasons to pull this out to a function\n\n1) anchor point for the doc string\n2) testability\n3) keep sam busy","commit_id":"a809425da5bf3e174e2e25aa68576e68674a8c16"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"c3e248372ec54a27d9a465ec74b3033f348bb342","unresolved":false,"context_lines":[{"line_number":312,"context_line":"            # Qc + Ro - Qo"},{"line_number":313,"context_line":"            quorum_size(len(containers)) + n_outgoing - policy.quorum,"},{"line_number":314,"context_line":"            # Rc"},{"line_number":315,"context_line":"            len(containers))"},{"line_number":316,"context_line":""},{"line_number":317,"context_line":"        container_iter \u003d itertools.cycle(containers)"},{"line_number":318,"context_line":"        existing_updates \u003d 0"}],"source_content_type":"text/x-python","patch_set":2,"id":"9f91af0f_ebb22594","line":315,"updated":"2018-01-08 21:13:16.000000000","message":"So in a reduced-redundancy policy... n_updates_needed comes out to len(containers)... the modulo below keeps us limited to the total number of outgoing requests... and some requests become responsible for updating more than one container server. OK, I think that makes sense.","commit_id":"a809425da5bf3e174e2e25aa68576e68674a8c16"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"c3e248372ec54a27d9a465ec74b3033f348bb342","unresolved":false,"context_lines":[{"line_number":319,"context_line":"        while existing_updates \u003c n_updates_needed:"},{"line_number":320,"context_line":"            set_container_update(existing_updates % n_outgoing,"},{"line_number":321,"context_line":"                                 next(container_iter))"},{"line_number":322,"context_line":"            existing_updates +\u003d 1"},{"line_number":323,"context_line":""},{"line_number":324,"context_line":"        for i, node in enumerate(delete_at_nodes or []):"},{"line_number":325,"context_line":"            i \u003d i % len(headers)"}],"source_content_type":"text/x-python","patch_set":2,"id":"9f91af0f_eb4b0528","line":322,"updated":"2018-01-08 21:13:16.000000000","message":"nit: Maybe better as\n\n for i, cont in zip(range(n_updates_needed), container_iter):\n     set_container_update(i % n_outgoing, cont)\n\n?","commit_id":"a809425da5bf3e174e2e25aa68576e68674a8c16"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"c3e248372ec54a27d9a465ec74b3033f348bb342","unresolved":false,"context_lines":[{"line_number":321,"context_line":"                                 next(container_iter))"},{"line_number":322,"context_line":"            existing_updates +\u003d 1"},{"line_number":323,"context_line":""},{"line_number":324,"context_line":"        for i, node in enumerate(delete_at_nodes or []):"},{"line_number":325,"context_line":"            i \u003d i % len(headers)"},{"line_number":326,"context_line":""},{"line_number":327,"context_line":"            headers[i][\u0027X-Delete-At-Container\u0027] \u003d delete_at_container"}],"source_content_type":"text/x-python","patch_set":2,"id":"9f91af0f_ae9bcb70","line":324,"updated":"2018-01-08 21:13:16.000000000","message":"Should we be doing some similar logic for x-delete-at? Maybe as a follow-on?\n\nOh, I see -- https://review.openstack.org/#/c/524548/","commit_id":"a809425da5bf3e174e2e25aa68576e68674a8c16"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"3592e406251991afde83928d6dd37e8010b438de","unresolved":false,"context_lines":[{"line_number":84,"context_line":"    return None"},{"line_number":85,"context_line":""},{"line_number":86,"context_line":""},{"line_number":87,"context_line":"def num_container_updates(container_replicas, container_quorum,"},{"line_number":88,"context_line":"                          object_replicas, object_quorum):"},{"line_number":89,"context_line":"    \"\"\""},{"line_number":90,"context_line":"    We need to send container updates via enough object servers such"}],"source_content_type":"text/x-python","patch_set":3,"id":"9f91af0f_279b4218","line":87,"range":{"start_line":87,"start_character":46,"end_line":87,"end_character":62},"updated":"2018-01-09 19:41:01.000000000","message":"nit: We could just derive this from container_replicas locally.","commit_id":"c8d0f6858ad1989b9fd199c71b741ff796c1b719"},{"author":{"_account_id":2622,"name":"Samuel Merritt","email":"spam+launchpad@andcheese.org","username":"torgomatic"},"change_message_id":"10ca20eacb2a7a2d8a08276d746e2ee0c72e4f12","unresolved":false,"context_lines":[{"line_number":84,"context_line":"    return None"},{"line_number":85,"context_line":""},{"line_number":86,"context_line":""},{"line_number":87,"context_line":"def num_container_updates(container_replicas, container_quorum,"},{"line_number":88,"context_line":"                          object_replicas, object_quorum):"},{"line_number":89,"context_line":"    \"\"\""},{"line_number":90,"context_line":"    We need to send container updates via enough object servers such"}],"source_content_type":"text/x-python","patch_set":3,"id":"9f91af0f_db810bc5","line":87,"range":{"start_line":87,"start_character":46,"end_line":87,"end_character":62},"in_reply_to":"9f91af0f_279b4218","updated":"2018-01-09 22:52:38.000000000","message":"We could, but I like the symmetry. :)","commit_id":"c8d0f6858ad1989b9fd199c71b741ff796c1b719"}],"test/unit/proxy/controllers/test_obj.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"3592e406251991afde83928d6dd37e8010b438de","unresolved":false,"context_lines":[{"line_number":4529,"context_line":"            (3, 5, 3, 4),    # 5 object replicas"},{"line_number":4530,"context_line":"            (3, 6, 3, 5),    # 6 object replicas"},{"line_number":4531,"context_line":"            (3, 7, 4, 5),    # 7 object replicas"},{"line_number":4532,"context_line":"        ]"},{"line_number":4533,"context_line":""},{"line_number":4534,"context_line":"        for c_replica, o_replica, o_quorum, exp in test_cases:"},{"line_number":4535,"context_line":"            c_quorum \u003d utils.quorum_size(c_replica)"}],"source_content_type":"text/x-python","patch_set":3,"id":"9f91af0f_aa10774f","line":4532,"updated":"2018-01-09 19:41:01.000000000","message":"Thanks, this is beautiful :-)\n\nMaybe add\n\n (3, 28, 22, 8),  # EC 10+4x2\n\nto test what we reference at https://docs.openstack.org/swift/latest/overview_erasure_code.html#ec-duplication ? That\u0027s just *loads* better than the 23 (!) updates that used to require. Those poor, poor container servers...","commit_id":"c8d0f6858ad1989b9fd199c71b741ff796c1b719"},{"author":{"_account_id":2622,"name":"Samuel Merritt","email":"spam+launchpad@andcheese.org","username":"torgomatic"},"change_message_id":"64688972ccb43046a513d3cb530e382dc01b2203","unresolved":false,"context_lines":[{"line_number":4529,"context_line":"            (3, 5, 3, 4),    # 5 object replicas"},{"line_number":4530,"context_line":"            (3, 6, 3, 5),    # 6 object replicas"},{"line_number":4531,"context_line":"            (3, 7, 4, 5),    # 7 object replicas"},{"line_number":4532,"context_line":"        ]"},{"line_number":4533,"context_line":""},{"line_number":4534,"context_line":"        for c_replica, o_replica, o_quorum, exp in test_cases:"},{"line_number":4535,"context_line":"            c_quorum \u003d utils.quorum_size(c_replica)"}],"source_content_type":"text/x-python","patch_set":3,"id":"9f91af0f_c1acbeee","line":4532,"in_reply_to":"9f91af0f_aa10774f","updated":"2018-01-10 00:34:50.000000000","message":"Done","commit_id":"c8d0f6858ad1989b9fd199c71b741ff796c1b719"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"37a275dcfe07b64d641a98a11a28b69dcb0ffa07","unresolved":false,"context_lines":[{"line_number":4540,"context_line":"            self.assertEqual("},{"line_number":4541,"context_line":"                exp, got,"},{"line_number":4542,"context_line":"                \"Failed for c_replica\u003d%d, o_replica\u003d%d, o_quorum\u003d%d\" % ("},{"line_number":4543,"context_line":"                    c_replica, o_replica, o_quorum))"},{"line_number":4544,"context_line":""},{"line_number":4545,"context_line":""},{"line_number":4546,"context_line":"if __name__ \u003d\u003d \u0027__main__\u0027:"}],"source_content_type":"text/x-python","patch_set":4,"id":"7f96bb07_fecfd654","line":4543,"updated":"2018-01-10 23:21:17.000000000","message":"this is a handy chart... sometimes I collect the failures for these kind of tests...\n\n    diff --git a/test/unit/proxy/controllers/test_obj.py b/test/unit/proxy/controllers/test_obj.py\n    index c3a1d5a..3e26972 100644\n    --- a/test/unit/proxy/controllers/test_obj.py\n    +++ b/test/unit/proxy/controllers/test_obj.py\n    @@ -4533,14 +4533,21 @@ class TestNumContainerUpdates(unittest.TestCase):\n                 (3, 7, 4, 5),    # 7 object replicas\n             ]\n     \n    +        failures \u003d []\n             for c_replica, o_replica, o_quorum, exp in test_cases:\n                 c_quorum \u003d utils.quorum_size(c_replica)\n                 got \u003d obj.num_container_updates(c_replica, c_quorum,\n                                                 o_replica, o_quorum)\n    -            self.assertEqual(\n    -                exp, got,\n    -                \"Failed for c_replica\u003d%d, o_replica\u003d%d, o_quorum\u003d%d\" % (\n    -                    c_replica, o_replica, o_quorum))\n    +            try:\n    +                self.assertEqual(\n    +                    exp, got,\n    +                    \"Failed for c_replica\u003d%d, o_replica\u003d%d, o_quorum\u003d%d\" % (\n    +                        c_replica, o_replica, o_quorum))\n    +            except AssertionError as e:\n    +                failures.append(e)\n    +        if failures:\n    +            self.fail(\u0027Some scenarios failed:\\n\u0027 + \u0027\\n\u0027.join(\n    +                str(f) for f in failures))\n     \n     \n     if __name__ \u003d\u003d \u0027__main__\u0027:\n\n\n\ntim always tells me I want sub test:\n\nhttps://docs.python.org/3/library/unittest.html#distinguishing-test-iterations-using-subtests","commit_id":"a41c458c90b12c52688dd8b2b8a818b79b4e9693"}],"test/unit/proxy/test_server.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"c3e248372ec54a27d9a465ec74b3033f348bb342","unresolved":false,"context_lines":[{"line_number":5345,"context_line":"        seen_headers \u003d self._gather_x_container_headers("},{"line_number":5346,"context_line":"            controller.PUT, req,"},{"line_number":5347,"context_line":"            # HEAD HEAD PUT PUT PUT PUT PUT PUT PUT PUT PUT PUT PUT"},{"line_number":5348,"context_line":"            200, 200, 201, 201, 201, 201, 201, 201, 201, 201, 201, 201, 201)"},{"line_number":5349,"context_line":""},{"line_number":5350,"context_line":"        self.assertEqual("},{"line_number":5351,"context_line":"            sorted(seen_headers), sorted(["}],"source_content_type":"text/x-python","patch_set":2,"id":"9f91af0f_4b2a312c","line":5348,"updated":"2018-01-08 21:13:16.000000000","message":"Better as\n\n *([200] * 2 + [201] * 11)\n\n?","commit_id":"a809425da5bf3e174e2e25aa68576e68674a8c16"}]}