)]}'
{"/PATCHSET_LEVEL":[{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"7e9ba76b150f75f2d8d6f16204105504ff242837","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":2,"id":"96da08db_6d4d4735","updated":"2023-01-27 13:42:57.000000000","message":"I\u0027m working on some before/after timing figures","commit_id":"eef67f5319da794b7786ed0613cdbc77889551af"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"cc218538c0cb87c2bba48a008cd604e255850a9c","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":2,"id":"fe5616d2_de064aad","updated":"2023-02-17 05:13:45.000000000","message":"LGTM","commit_id":"eef67f5319da794b7786ed0613cdbc77889551af"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"93c719df7eb8fdcdb8b78858c0f26db2a2027d19","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":2,"id":"4a729687_a2f01112","updated":"2023-01-27 15:10:36.000000000","message":"benchmark shows yield_objects is ~1.8x faster with this patch\n\nhttps://gist.github.com/alistairncoles/c3baa5c7b26584aa7ded3c0eccc4216b","commit_id":"eef67f5319da794b7786ed0613cdbc77889551af"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"d55e4a0be04b4f1e421a74e08467c6893c71ab69","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":2,"id":"ee9126c5_d6d4b253","updated":"2023-01-27 17:34:42.000000000","message":"sorry, I didn\u0027t post these","commit_id":"eef67f5319da794b7786ed0613cdbc77889551af"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"bfd67aef540c710568c7c3cf67917edf6547017b","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":3,"id":"c718afba_c842bcb5","updated":"2023-02-21 19:23:11.000000000","message":"I think I\u0027m content to carry this. Still a little nervous about what happens if we can\u0027t find any dest ranges, though.","commit_id":"06166ea4dc6e7dd9ce31db17fb8ed6c5ae85e090"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"0b9241e7c762c7db021655804d0e41b6d5c555cb","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":3,"id":"f473bfa2_8615a6bc","updated":"2023-02-17 21:38:02.000000000","message":"LOTs of object 503 PUT failures. Need \"recheck\" at a different time.\nE           swiftclient.exceptions.ClientException: Container PUT failed: http://127.0.0.1:8080/v1/AUTH_test/dest-container-3715cf16-172f-4074-ba90-25740dad7610 503 Service Unavailable  [first 60 chars of response] b\u0027\u003chtml\u003e\u003ch1\u003eService Unavailable\u003c/h1\u003e\u003cp\u003eThe server is currently\u0027 (txn: txa9ef443edded4742ac03c-0063efe18a)\nE           swift.common.internal_client.UnexpectedResponse: Unexpected response: 503 Service Unavailable (b\u0027\u003chtml\u003e\u003ch1\u003eService Unavailable\u003c/h1\u003e\u003cp\u003eThe server is currently unavailable. Please try again at a later time.\u003c/p\u003e\u003c/html\u003e\u0027)\n","commit_id":"06166ea4dc6e7dd9ce31db17fb8ed6c5ae85e090"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"4f190f99a2cf28e03f453d98c8ca6c4476135f94","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":3,"id":"970a35ec_841e1095","updated":"2023-02-18 00:28:17.000000000","message":"recheck\n\nAgreed; the failures smell like a bunch of timeouts from a slow VM.","commit_id":"06166ea4dc6e7dd9ce31db17fb8ed6c5ae85e090"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"9e274bca8dd25304648b225214d008d17497a760","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":3,"id":"7f6bae72_1f9279e6","updated":"2023-02-17 19:58:14.000000000","message":"recheck\n\"socket.timeout: timed out\" error in Grenade test suite, not related to this patch.","commit_id":"06166ea4dc6e7dd9ce31db17fb8ed6c5ae85e090"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"63698463b38ad2335e92719c6610b4fa4e7828df","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":4,"id":"b0938701_36bebd82","updated":"2023-02-22 14:44:29.000000000","message":"This is OK as it is, but there\u0027s a follow-up that I\u0027d value opinions on https://review.opendev.org/c/openstack/swift/+/874781","commit_id":"1654da33b4de0093a5a29b29f1b480aa70ef6e82"}],"swift/container/sharder.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"d55e4a0be04b4f1e421a74e08467c6893c71ab69","unresolved":true,"context_lines":[{"line_number":1587,"context_line":"                except StopIteration:"},{"line_number":1588,"context_line":"                    return None"},{"line_number":1589,"context_line":""},{"line_number":1590,"context_line":"            if dest_shard_range_iter is None:"},{"line_number":1591,"context_line":"                dest_shard_range_iter \u003d iter(dest_shard_ranges())"},{"line_number":1592,"context_line":"                dest_shard_range \u003d next_or_none(dest_shard_range_iter)"},{"line_number":1593,"context_line":""}],"source_content_type":"text/x-python","patch_set":1,"id":"e8088e11_610b510b","side":"PARENT","line":1590,"updated":"2023-01-27 17:34:42.000000000","message":"when would it not be None?!","commit_id":"5de745c2bc3aa82049424b9c3e8744cda53a4c74"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"67092f34747a88a9613ed804fb6fe1a86ef37eef","unresolved":false,"context_lines":[{"line_number":1587,"context_line":"                except StopIteration:"},{"line_number":1588,"context_line":"                    return None"},{"line_number":1589,"context_line":""},{"line_number":1590,"context_line":"            if dest_shard_range_iter is None:"},{"line_number":1591,"context_line":"                dest_shard_range_iter \u003d iter(dest_shard_ranges())"},{"line_number":1592,"context_line":"                dest_shard_range \u003d next_or_none(dest_shard_range_iter)"},{"line_number":1593,"context_line":""}],"source_content_type":"text/x-python","patch_set":1,"id":"4d492c0c_ec800efc","side":"PARENT","line":1590,"in_reply_to":"db4ab594_8ef40d25","updated":"2023-02-22 13:45:42.000000000","message":"Done","commit_id":"5de745c2bc3aa82049424b9c3e8744cda53a4c74"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a32be7111a084d2b64972b5624cef7a9dd8e5fb6","unresolved":true,"context_lines":[{"line_number":1587,"context_line":"                except StopIteration:"},{"line_number":1588,"context_line":"                    return None"},{"line_number":1589,"context_line":""},{"line_number":1590,"context_line":"            if dest_shard_range_iter is None:"},{"line_number":1591,"context_line":"                dest_shard_range_iter \u003d iter(dest_shard_ranges())"},{"line_number":1592,"context_line":"                dest_shard_range \u003d next_or_none(dest_shard_range_iter)"},{"line_number":1593,"context_line":""}],"source_content_type":"text/x-python","patch_set":1,"id":"db4ab594_8ef40d25","side":"PARENT","line":1590,"in_reply_to":"e8088e11_610b510b","updated":"2023-01-27 18:34:16.000000000","message":"first iteration of the loop it is still None, because we wanted to have the \n\n  if not objs:\n       break\n       \nbefore calling dest_shard_ranges","commit_id":"5de745c2bc3aa82049424b9c3e8744cda53a4c74"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"d55e4a0be04b4f1e421a74e08467c6893c71ab69","unresolved":true,"context_lines":[{"line_number":1530,"context_line":"        \"\"\""},{"line_number":1531,"context_line":"        Iterates through all objects in ``src_shard_range`` in name order"},{"line_number":1532,"context_line":"        yielding them in lists of up to ``batch_size`` in length. All batches"},{"line_number":1533,"context_line":"        of undeleted objects are yielded before all batches of undeleted"},{"line_number":1534,"context_line":"        objects."},{"line_number":1535,"context_line":""},{"line_number":1536,"context_line":"        :param broker: A :class:`~swift.container.backend.ContainerBroker`."}],"source_content_type":"text/x-python","patch_set":1,"id":"33f1ed00_8bcae649","line":1533,"updated":"2023-01-27 17:34:42.000000000","message":"looks like deleted objects are yielded before undeleted","commit_id":"0e70777360cb668a46249e7c2d74ade2d85e88db"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a32be7111a084d2b64972b5624cef7a9dd8e5fb6","unresolved":true,"context_lines":[{"line_number":1530,"context_line":"        \"\"\""},{"line_number":1531,"context_line":"        Iterates through all objects in ``src_shard_range`` in name order"},{"line_number":1532,"context_line":"        yielding them in lists of up to ``batch_size`` in length. All batches"},{"line_number":1533,"context_line":"        of undeleted objects are yielded before all batches of undeleted"},{"line_number":1534,"context_line":"        objects."},{"line_number":1535,"context_line":""},{"line_number":1536,"context_line":"        :param broker: A :class:`~swift.container.backend.ContainerBroker`."}],"source_content_type":"text/x-python","patch_set":1,"id":"a55476c6_a61e4e1a","line":1533,"in_reply_to":"33f1ed00_8bcae649","updated":"2023-01-27 18:34:16.000000000","message":"added some tests in patchset 2 that illustrate, but it\u0027s undeleted first:\n\n  for include_deleted in (False, True)","commit_id":"0e70777360cb668a46249e7c2d74ade2d85e88db"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"67092f34747a88a9613ed804fb6fe1a86ef37eef","unresolved":false,"context_lines":[{"line_number":1530,"context_line":"        \"\"\""},{"line_number":1531,"context_line":"        Iterates through all objects in ``src_shard_range`` in name order"},{"line_number":1532,"context_line":"        yielding them in lists of up to ``batch_size`` in length. All batches"},{"line_number":1533,"context_line":"        of undeleted objects are yielded before all batches of undeleted"},{"line_number":1534,"context_line":"        objects."},{"line_number":1535,"context_line":""},{"line_number":1536,"context_line":"        :param broker: A :class:`~swift.container.backend.ContainerBroker`."}],"source_content_type":"text/x-python","patch_set":1,"id":"d5765c6d_1af24e1e","line":1533,"in_reply_to":"a55476c6_a61e4e1a","updated":"2023-02-22 13:45:42.000000000","message":"Done","commit_id":"0e70777360cb668a46249e7c2d74ade2d85e88db"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"d55e4a0be04b4f1e421a74e08467c6893c71ab69","unresolved":true,"context_lines":[{"line_number":1565,"context_line":"                    yield objects, info"},{"line_number":1566,"context_line":""},{"line_number":1567,"context_line":"                if len(objects) \u003c self.cleave_row_batch_size:"},{"line_number":1568,"context_line":"                    break"},{"line_number":1569,"context_line":"                marker \u003d objects[-1][\u0027name\u0027]"},{"line_number":1570,"context_line":""},{"line_number":1571,"context_line":"    def yield_objects_to_shard_range(self, broker, src_shard_range,"}],"source_content_type":"text/x-python","patch_set":1,"id":"f75735d7_1c402482","line":1568,"updated":"2023-01-27 17:34:42.000000000","message":"i don\u0027t get this break when we don\u0027t have enough ...\n\nshould it compare to batch_size?  maybe just an if objects: else?","commit_id":"0e70777360cb668a46249e7c2d74ade2d85e88db"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"67092f34747a88a9613ed804fb6fe1a86ef37eef","unresolved":false,"context_lines":[{"line_number":1565,"context_line":"                    yield objects, info"},{"line_number":1566,"context_line":""},{"line_number":1567,"context_line":"                if len(objects) \u003c self.cleave_row_batch_size:"},{"line_number":1568,"context_line":"                    break"},{"line_number":1569,"context_line":"                marker \u003d objects[-1][\u0027name\u0027]"},{"line_number":1570,"context_line":""},{"line_number":1571,"context_line":"    def yield_objects_to_shard_range(self, broker, src_shard_range,"}],"source_content_type":"text/x-python","patch_set":1,"id":"e20f2074_2ad14baf","line":1568,"in_reply_to":"39ab2e40_510372c2","updated":"2023-02-22 13:45:42.000000000","message":"Done","commit_id":"0e70777360cb668a46249e7c2d74ade2d85e88db"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a32be7111a084d2b64972b5624cef7a9dd8e5fb6","unresolved":true,"context_lines":[{"line_number":1565,"context_line":"                    yield objects, info"},{"line_number":1566,"context_line":""},{"line_number":1567,"context_line":"                if len(objects) \u003c self.cleave_row_batch_size:"},{"line_number":1568,"context_line":"                    break"},{"line_number":1569,"context_line":"                marker \u003d objects[-1][\u0027name\u0027]"},{"line_number":1570,"context_line":""},{"line_number":1571,"context_line":"    def yield_objects_to_shard_range(self, broker, src_shard_range,"}],"source_content_type":"text/x-python","patch_set":1,"id":"39ab2e40_510372c2","line":1568,"in_reply_to":"f75735d7_1c402482","updated":"2023-01-27 18:34:16.000000000","message":"maybe your comment was before I fixed this? \n\nI think if objects ... else will result in extra wasted lookups for no objects when the last lookup gave us less than batch_size","commit_id":"0e70777360cb668a46249e7c2d74ade2d85e88db"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"d55e4a0be04b4f1e421a74e08467c6893c71ab69","unresolved":true,"context_lines":[{"line_number":1572,"context_line":"                                     dest_shard_ranges):"},{"line_number":1573,"context_line":"        \"\"\""},{"line_number":1574,"context_line":"        Iterates through all objects in ``src_shard_range`` to place them in"},{"line_number":1575,"context_line":"        destination shard ranges provided by the ``next_shard_range`` function."},{"line_number":1576,"context_line":"        Yields tuples of (object list, destination shard range in which those"},{"line_number":1577,"context_line":"        objects belong). Note that the same destination shard range may be"},{"line_number":1578,"context_line":"        referenced in more than one yielded tuple."}],"source_content_type":"text/x-python","patch_set":1,"id":"c53760b6_9354242c","line":1575,"updated":"2023-01-27 17:34:42.000000000","message":"s/next_shard_range/dest_shard_range","commit_id":"0e70777360cb668a46249e7c2d74ade2d85e88db"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"67092f34747a88a9613ed804fb6fe1a86ef37eef","unresolved":false,"context_lines":[{"line_number":1572,"context_line":"                                     dest_shard_ranges):"},{"line_number":1573,"context_line":"        \"\"\""},{"line_number":1574,"context_line":"        Iterates through all objects in ``src_shard_range`` to place them in"},{"line_number":1575,"context_line":"        destination shard ranges provided by the ``next_shard_range`` function."},{"line_number":1576,"context_line":"        Yields tuples of (object list, destination shard range in which those"},{"line_number":1577,"context_line":"        objects belong). Note that the same destination shard range may be"},{"line_number":1578,"context_line":"        referenced in more than one yielded tuple."}],"source_content_type":"text/x-python","patch_set":1,"id":"93002c1d_437f2be0","line":1575,"in_reply_to":"c53760b6_9354242c","updated":"2023-02-22 13:45:42.000000000","message":"Done","commit_id":"0e70777360cb668a46249e7c2d74ade2d85e88db"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"d55e4a0be04b4f1e421a74e08467c6893c71ab69","unresolved":true,"context_lines":[{"line_number":1591,"context_line":"        for _ in self.yield_objects(broker, src_shard_range, batch_size\u003d1):"},{"line_number":1592,"context_line":"            break"},{"line_number":1593,"context_line":"        else:"},{"line_number":1594,"context_line":"            return"},{"line_number":1595,"context_line":""},{"line_number":1596,"context_line":"        dest_shard_range_iter \u003d iter(dest_shard_ranges())"},{"line_number":1597,"context_line":"        dest_shard_range \u003d None"}],"source_content_type":"text/x-python","patch_set":1,"id":"d754bea2_5fa9d660","line":1594,"updated":"2023-01-27 17:34:42.000000000","message":"seems a little weird, but the existing signature didn\u0027t change\n\nand i guess why pass a callable instead of the result-value-itself unless you\u0027re trying to defer/avoid calling it","commit_id":"0e70777360cb668a46249e7c2d74ade2d85e88db"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"67092f34747a88a9613ed804fb6fe1a86ef37eef","unresolved":false,"context_lines":[{"line_number":1591,"context_line":"        for _ in self.yield_objects(broker, src_shard_range, batch_size\u003d1):"},{"line_number":1592,"context_line":"            break"},{"line_number":1593,"context_line":"        else:"},{"line_number":1594,"context_line":"            return"},{"line_number":1595,"context_line":""},{"line_number":1596,"context_line":"        dest_shard_range_iter \u003d iter(dest_shard_ranges())"},{"line_number":1597,"context_line":"        dest_shard_range \u003d None"}],"source_content_type":"text/x-python","patch_set":1,"id":"dbc95374_f1b4e8d1","line":1594,"in_reply_to":"32c681e9_a6a34733","updated":"2023-02-22 13:45:42.000000000","message":"Done","commit_id":"0e70777360cb668a46249e7c2d74ade2d85e88db"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a32be7111a084d2b64972b5624cef7a9dd8e5fb6","unresolved":true,"context_lines":[{"line_number":1591,"context_line":"        for _ in self.yield_objects(broker, src_shard_range, batch_size\u003d1):"},{"line_number":1592,"context_line":"            break"},{"line_number":1593,"context_line":"        else:"},{"line_number":1594,"context_line":"            return"},{"line_number":1595,"context_line":""},{"line_number":1596,"context_line":"        dest_shard_range_iter \u003d iter(dest_shard_ranges())"},{"line_number":1597,"context_line":"        dest_shard_range \u003d None"}],"source_content_type":"text/x-python","patch_set":1,"id":"32c681e9_a6a34733","line":1594,"in_reply_to":"d754bea2_5fa9d660","updated":"2023-01-27 18:34:16.000000000","message":"we are trying to defer calling dest_shard_ranges\n\nThe current implementation of yield_objects_to_shard_range iters over the source shard range in an outer loop, and inside it iters over dest_shard_ranges, and so dest_shard_ranges is never called if the source has no objects, BUT that implementation is what caused the original bug whereby we might not go back for deleted objects.\n\nHere, I turn it on its head and iter over dest_shard_ranges, and for each of them iter over the relevant part of the src until we have deleted and deleted, then move on to the next dest. But, that means I need a pre-flight check to make sure it is worth calling dest_shard_ranges to start the outer iteration.","commit_id":"0e70777360cb668a46249e7c2d74ade2d85e88db"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"d55e4a0be04b4f1e421a74e08467c6893c71ab69","unresolved":true,"context_lines":[{"line_number":1596,"context_line":"        dest_shard_range_iter \u003d iter(dest_shard_ranges())"},{"line_number":1597,"context_line":"        dest_shard_range \u003d None"},{"line_number":1598,"context_line":"        src_shard_range_marker \u003d src_shard_range.lower"},{"line_number":1599,"context_line":"        for dest_shard_range in dest_shard_range_iter:"},{"line_number":1600,"context_line":"            if dest_shard_range.upper \u003c\u003d src_shard_range.lower:"},{"line_number":1601,"context_line":"                continue"},{"line_number":1602,"context_line":""}],"source_content_type":"text/x-python","patch_set":1,"id":"2303bb16_fb7fef20","line":1599,"updated":"2023-01-27 17:34:42.000000000","message":"i do like the explict iteratoin of the dest shard ranges - where we yield ALL objects/rows that belong to each.","commit_id":"0e70777360cb668a46249e7c2d74ade2d85e88db"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"cc218538c0cb87c2bba48a008cd604e255850a9c","unresolved":true,"context_lines":[{"line_number":1596,"context_line":"        dest_shard_range_iter \u003d iter(dest_shard_ranges())"},{"line_number":1597,"context_line":"        dest_shard_range \u003d None"},{"line_number":1598,"context_line":"        src_shard_range_marker \u003d src_shard_range.lower"},{"line_number":1599,"context_line":"        for dest_shard_range in dest_shard_range_iter:"},{"line_number":1600,"context_line":"            if dest_shard_range.upper \u003c\u003d src_shard_range.lower:"},{"line_number":1601,"context_line":"                continue"},{"line_number":1602,"context_line":""}],"source_content_type":"text/x-python","patch_set":1,"id":"3cd8cb42_a07bcce8","line":1599,"in_reply_to":"2303bb16_fb7fef20","updated":"2023-02-17 05:13:45.000000000","message":"+1 here. The implementation is cleaner. I do have a question on the input \"dest_shard_range\" which is sorted by \"name\", why do we have them sorted by name instead of by \"lower\"?","commit_id":"0e70777360cb668a46249e7c2d74ade2d85e88db"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"ca43ab8cd471a003dff324c28016f0bfa74ab417","unresolved":false,"context_lines":[{"line_number":1596,"context_line":"        dest_shard_range_iter \u003d iter(dest_shard_ranges())"},{"line_number":1597,"context_line":"        dest_shard_range \u003d None"},{"line_number":1598,"context_line":"        src_shard_range_marker \u003d src_shard_range.lower"},{"line_number":1599,"context_line":"        for dest_shard_range in dest_shard_range_iter:"},{"line_number":1600,"context_line":"            if dest_shard_range.upper \u003c\u003d src_shard_range.lower:"},{"line_number":1601,"context_line":"                continue"},{"line_number":1602,"context_line":""}],"source_content_type":"text/x-python","patch_set":1,"id":"78e2cfca_ca1443d2","line":1599,"in_reply_to":"3cd8cb42_a07bcce8","updated":"2023-02-17 20:28:54.000000000","message":"Done","commit_id":"0e70777360cb668a46249e7c2d74ade2d85e88db"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"67092f34747a88a9613ed804fb6fe1a86ef37eef","unresolved":false,"context_lines":[{"line_number":1596,"context_line":"        dest_shard_range_iter \u003d iter(dest_shard_ranges())"},{"line_number":1597,"context_line":"        dest_shard_range \u003d None"},{"line_number":1598,"context_line":"        src_shard_range_marker \u003d src_shard_range.lower"},{"line_number":1599,"context_line":"        for dest_shard_range in dest_shard_range_iter:"},{"line_number":1600,"context_line":"            if dest_shard_range.upper \u003c\u003d src_shard_range.lower:"},{"line_number":1601,"context_line":"                continue"},{"line_number":1602,"context_line":""}],"source_content_type":"text/x-python","patch_set":1,"id":"5ad93e9d_78cbbdbf","line":1599,"in_reply_to":"3cd8cb42_a07bcce8","updated":"2023-02-22 13:45:42.000000000","message":"dest_shard_ranges doesn\u0027t in fact iterate by name order (so the doc string is wrong). The order is defined by the usual ContainerBroker get_shard_ranges method (see _make_shard_range_fetcher), so it is sorted by ShardRange.sort_key (upper, state, lower, name).","commit_id":"0e70777360cb668a46249e7c2d74ade2d85e88db"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"d55e4a0be04b4f1e421a74e08467c6893c71ab69","unresolved":true,"context_lines":[{"line_number":1605,"context_line":"                sub_src_range \u003d src_shard_range.copy("},{"line_number":1606,"context_line":"                    lower\u003dsrc_shard_range_marker, upper\u003ddest_shard_range.lower)"},{"line_number":1607,"context_line":"                for objs, info in self.yield_objects(broker, sub_src_range):"},{"line_number":1608,"context_line":"                    yield objs, None, info"},{"line_number":1609,"context_line":"            src_shard_range_marker \u003d dest_shard_range.upper"},{"line_number":1610,"context_line":""},{"line_number":1611,"context_line":"            sub_src_range \u003d src_shard_range.copy("}],"source_content_type":"text/x-python","patch_set":1,"id":"256842ac_e80f870a","line":1608,"updated":"2023-01-27 17:34:42.000000000","message":"the significance of returning None instead of the synthasized sub-src_range is not obvious from this diff","commit_id":"0e70777360cb668a46249e7c2d74ade2d85e88db"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a32be7111a084d2b64972b5624cef7a9dd8e5fb6","unresolved":true,"context_lines":[{"line_number":1605,"context_line":"                sub_src_range \u003d src_shard_range.copy("},{"line_number":1606,"context_line":"                    lower\u003dsrc_shard_range_marker, upper\u003ddest_shard_range.lower)"},{"line_number":1607,"context_line":"                for objs, info in self.yield_objects(broker, sub_src_range):"},{"line_number":1608,"context_line":"                    yield objs, None, info"},{"line_number":1609,"context_line":"            src_shard_range_marker \u003d dest_shard_range.upper"},{"line_number":1610,"context_line":""},{"line_number":1611,"context_line":"            sub_src_range \u003d src_shard_range.copy("}],"source_content_type":"text/x-python","patch_set":1,"id":"43b65d63_31dbb331","line":1608,"in_reply_to":"256842ac_e80f870a","updated":"2023-01-27 18:34:16.000000000","message":"we never yield anything to do with the source range\n\nYielding None indicates that there is no dest in which to place objects found in the source. Based on getting a None back, the caller records how many unplaced objects there are and marks the attempt to move misplaced objects as a failure.\n\nWe could not bother even yielding objects when there is no dest shard range - they\u0027re not going anywhere - but currently we do so that we can count and report unplaced.","commit_id":"0e70777360cb668a46249e7c2d74ade2d85e88db"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"67092f34747a88a9613ed804fb6fe1a86ef37eef","unresolved":false,"context_lines":[{"line_number":1605,"context_line":"                sub_src_range \u003d src_shard_range.copy("},{"line_number":1606,"context_line":"                    lower\u003dsrc_shard_range_marker, upper\u003ddest_shard_range.lower)"},{"line_number":1607,"context_line":"                for objs, info in self.yield_objects(broker, sub_src_range):"},{"line_number":1608,"context_line":"                    yield objs, None, info"},{"line_number":1609,"context_line":"            src_shard_range_marker \u003d dest_shard_range.upper"},{"line_number":1610,"context_line":""},{"line_number":1611,"context_line":"            sub_src_range \u003d src_shard_range.copy("}],"source_content_type":"text/x-python","patch_set":1,"id":"6a9b0ed5_d3d0d9a6","line":1608,"in_reply_to":"43b65d63_31dbb331","updated":"2023-02-22 13:45:42.000000000","message":"Done","commit_id":"0e70777360cb668a46249e7c2d74ade2d85e88db"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"d55e4a0be04b4f1e421a74e08467c6893c71ab69","unresolved":true,"context_lines":[{"line_number":1624,"context_line":"                sub_src_range \u003d src_shard_range.copy("},{"line_number":1625,"context_line":"                    lower\u003dmax(dest_shard_range.upper, src_shard_range.lower))"},{"line_number":1626,"context_line":"                for objs, info in self.yield_objects(broker, sub_src_range):"},{"line_number":1627,"context_line":"                    yield objs, None, info"},{"line_number":1628,"context_line":""},{"line_number":1629,"context_line":"    def _post_replicate_hook(self, broker, info, responses):"},{"line_number":1630,"context_line":"        # override superclass behaviour"}],"source_content_type":"text/x-python","patch_set":1,"id":"0d657419_5893e98b","line":1627,"updated":"2023-01-27 17:34:42.000000000","message":"this seems like it\u0027s been totally re-written... did we \"fix\" anything else besides the slow query?\n\nI think I\u0027d need a picture of src/dest range boundary alignment to understand why we have 3 different yield_object calls in this function - i assume it has something to do with gaps or duplicate ranges in dest and tests would explain it if I tried removing some of these blocks?","commit_id":"0e70777360cb668a46249e7c2d74ade2d85e88db"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a32be7111a084d2b64972b5624cef7a9dd8e5fb6","unresolved":true,"context_lines":[{"line_number":1624,"context_line":"                sub_src_range \u003d src_shard_range.copy("},{"line_number":1625,"context_line":"                    lower\u003dmax(dest_shard_range.upper, src_shard_range.lower))"},{"line_number":1626,"context_line":"                for objs, info in self.yield_objects(broker, sub_src_range):"},{"line_number":1627,"context_line":"                    yield objs, None, info"},{"line_number":1628,"context_line":""},{"line_number":1629,"context_line":"    def _post_replicate_hook(self, broker, info, responses):"},{"line_number":1630,"context_line":"        # override superclass behaviour"}],"source_content_type":"text/x-python","patch_set":1,"id":"ffab1f46_99910335","line":1627,"in_reply_to":"0d657419_5893e98b","updated":"2023-01-27 18:34:16.000000000","message":"I reverted yield_objects to pretty much how it was pre Ie8404f0c7e84d3916f0e0fa62afc54f1f43a4d06 \n\nI\u0027ve then re-written yield_objects_to_shard_range to fix the bug that Ie8404f0c7e84d3916f0e0fa62afc54f1f43a4d06 fixed (which was that tombstones were often not yielded)\n\nI added test_yield_objects_to_shard_range in patchset 2 which may help illustrate","commit_id":"0e70777360cb668a46249e7c2d74ade2d85e88db"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"67092f34747a88a9613ed804fb6fe1a86ef37eef","unresolved":false,"context_lines":[{"line_number":1624,"context_line":"                sub_src_range \u003d src_shard_range.copy("},{"line_number":1625,"context_line":"                    lower\u003dmax(dest_shard_range.upper, src_shard_range.lower))"},{"line_number":1626,"context_line":"                for objs, info in self.yield_objects(broker, sub_src_range):"},{"line_number":1627,"context_line":"                    yield objs, None, info"},{"line_number":1628,"context_line":""},{"line_number":1629,"context_line":"    def _post_replicate_hook(self, broker, info, responses):"},{"line_number":1630,"context_line":"        # override superclass behaviour"}],"source_content_type":"text/x-python","patch_set":1,"id":"1a4c3177_c38b3357","line":1627,"in_reply_to":"ffab1f46_99910335","updated":"2023-02-22 13:45:42.000000000","message":"Done","commit_id":"0e70777360cb668a46249e7c2d74ade2d85e88db"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"cc218538c0cb87c2bba48a008cd604e255850a9c","unresolved":true,"context_lines":[{"line_number":1530,"context_line":"        \"\"\""},{"line_number":1531,"context_line":"        Iterates through all objects in ``src_shard_range`` in name order"},{"line_number":1532,"context_line":"        yielding them in lists of up to ``batch_size`` in length. All batches"},{"line_number":1533,"context_line":"        of undeleted objects are yielded before all batches of undeleted"},{"line_number":1534,"context_line":"        objects."},{"line_number":1535,"context_line":""},{"line_number":1536,"context_line":"        :param broker: A :class:`~swift.container.backend.ContainerBroker`."}],"source_content_type":"text/x-python","patch_set":2,"id":"4f8a9d71_51c47523","line":1533,"updated":"2023-02-17 05:13:45.000000000","message":"\"before all batches of undeleted\" -\u003e \"before all batches of deleted\"","commit_id":"eef67f5319da794b7786ed0613cdbc77889551af"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"67092f34747a88a9613ed804fb6fe1a86ef37eef","unresolved":false,"context_lines":[{"line_number":1530,"context_line":"        \"\"\""},{"line_number":1531,"context_line":"        Iterates through all objects in ``src_shard_range`` in name order"},{"line_number":1532,"context_line":"        yielding them in lists of up to ``batch_size`` in length. All batches"},{"line_number":1533,"context_line":"        of undeleted objects are yielded before all batches of undeleted"},{"line_number":1534,"context_line":"        objects."},{"line_number":1535,"context_line":""},{"line_number":1536,"context_line":"        :param broker: A :class:`~swift.container.backend.ContainerBroker`."}],"source_content_type":"text/x-python","patch_set":2,"id":"064261b4_339f5f49","line":1533,"in_reply_to":"4f8a9d71_51c47523","updated":"2023-02-22 13:45:42.000000000","message":"Done","commit_id":"eef67f5319da794b7786ed0613cdbc77889551af"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"ca43ab8cd471a003dff324c28016f0bfa74ab417","unresolved":false,"context_lines":[{"line_number":1530,"context_line":"        \"\"\""},{"line_number":1531,"context_line":"        Iterates through all objects in ``src_shard_range`` in name order"},{"line_number":1532,"context_line":"        yielding them in lists of up to ``batch_size`` in length. All batches"},{"line_number":1533,"context_line":"        of undeleted objects are yielded before all batches of undeleted"},{"line_number":1534,"context_line":"        objects."},{"line_number":1535,"context_line":""},{"line_number":1536,"context_line":"        :param broker: A :class:`~swift.container.backend.ContainerBroker`."}],"source_content_type":"text/x-python","patch_set":2,"id":"c4147a3c_99993555","line":1533,"in_reply_to":"4f8a9d71_51c47523","updated":"2023-02-17 20:28:54.000000000","message":"Done","commit_id":"eef67f5319da794b7786ed0613cdbc77889551af"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"cc218538c0cb87c2bba48a008cd604e255850a9c","unresolved":true,"context_lines":[{"line_number":1547,"context_line":"            return"},{"line_number":1548,"context_line":""},{"line_number":1549,"context_line":"        batch_size \u003d batch_size or self.cleave_row_batch_size"},{"line_number":1550,"context_line":"        for include_deleted in (False, True):"},{"line_number":1551,"context_line":"            marker \u003d src_shard_range.lower_str"},{"line_number":1552,"context_line":"            while True:"},{"line_number":1553,"context_line":"                info \u003d broker.get_info()"}],"source_content_type":"text/x-python","patch_set":2,"id":"262b0504_f1a12fbb","line":1550,"updated":"2023-02-17 05:13:45.000000000","message":"this change is also in the path of \"_cleave_shard_range\". I understand previously \"_cleave_shard_range\" will also yield both deleted and undeleted objects but without order. I am thinking in future maybe we can ignore \"deleted\" for cleaving? if yes, I am thinking to leave a TODO here.","commit_id":"eef67f5319da794b7786ed0613cdbc77889551af"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"67092f34747a88a9613ed804fb6fe1a86ef37eef","unresolved":false,"context_lines":[{"line_number":1547,"context_line":"            return"},{"line_number":1548,"context_line":""},{"line_number":1549,"context_line":"        batch_size \u003d batch_size or self.cleave_row_batch_size"},{"line_number":1550,"context_line":"        for include_deleted in (False, True):"},{"line_number":1551,"context_line":"            marker \u003d src_shard_range.lower_str"},{"line_number":1552,"context_line":"            while True:"},{"line_number":1553,"context_line":"                info \u003d broker.get_info()"}],"source_content_type":"text/x-python","patch_set":2,"id":"fd1c900b_c7ca0132","line":1550,"in_reply_to":"262b0504_f1a12fbb","updated":"2023-02-22 13:45:42.000000000","message":"Cleaving must also yield both deleted and undeleted object rows. Deleted rows are preserved for a reclaim_age so that \u0027stale\u0027 object updates (e.g. a delayed async pending) and replication with other container replicas does not re-create an object row after an object has been deleted. Deleted rows must therefore be cleaved to shards.","commit_id":"eef67f5319da794b7786ed0613cdbc77889551af"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"cc218538c0cb87c2bba48a008cd604e255850a9c","unresolved":true,"context_lines":[{"line_number":1560,"context_line":"                    include_deleted\u003dinclude_deleted,"},{"line_number":1561,"context_line":"                    since_row\u003dsince_row)"},{"line_number":1562,"context_line":"                if objects:"},{"line_number":1563,"context_line":"                    self.logger.debug(\u0027got %s objects from %s in %ss\u0027,"},{"line_number":1564,"context_line":"                                      len(objects), broker.db_file,"},{"line_number":1565,"context_line":"                                      time.time() - start)"},{"line_number":1566,"context_line":"                    yield objects, info"}],"source_content_type":"text/x-python","patch_set":2,"id":"34c54457_954a75ec","line":1563,"updated":"2023-02-17 05:13:45.000000000","message":"nit: add flag of \"include_deleted\"?","commit_id":"eef67f5319da794b7786ed0613cdbc77889551af"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"ca43ab8cd471a003dff324c28016f0bfa74ab417","unresolved":false,"context_lines":[{"line_number":1560,"context_line":"                    include_deleted\u003dinclude_deleted,"},{"line_number":1561,"context_line":"                    since_row\u003dsince_row)"},{"line_number":1562,"context_line":"                if objects:"},{"line_number":1563,"context_line":"                    self.logger.debug(\u0027got %s objects from %s in %ss\u0027,"},{"line_number":1564,"context_line":"                                      len(objects), broker.db_file,"},{"line_number":1565,"context_line":"                                      time.time() - start)"},{"line_number":1566,"context_line":"                    yield objects, info"}],"source_content_type":"text/x-python","patch_set":2,"id":"3f37b8bd_3d2f55fe","line":1563,"in_reply_to":"34c54457_954a75ec","updated":"2023-02-17 20:28:54.000000000","message":"Done","commit_id":"eef67f5319da794b7786ed0613cdbc77889551af"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"67092f34747a88a9613ed804fb6fe1a86ef37eef","unresolved":false,"context_lines":[{"line_number":1560,"context_line":"                    include_deleted\u003dinclude_deleted,"},{"line_number":1561,"context_line":"                    since_row\u003dsince_row)"},{"line_number":1562,"context_line":"                if objects:"},{"line_number":1563,"context_line":"                    self.logger.debug(\u0027got %s objects from %s in %ss\u0027,"},{"line_number":1564,"context_line":"                                      len(objects), broker.db_file,"},{"line_number":1565,"context_line":"                                      time.time() - start)"},{"line_number":1566,"context_line":"                    yield objects, info"}],"source_content_type":"text/x-python","patch_set":2,"id":"89f3c1bc_c67e43e4","line":1563,"in_reply_to":"34c54457_954a75ec","updated":"2023-02-22 13:45:42.000000000","message":"Done","commit_id":"eef67f5319da794b7786ed0613cdbc77889551af"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"bfd67aef540c710568c7c3cf67917edf6547017b","unresolved":true,"context_lines":[{"line_number":1544,"context_line":"        \"\"\""},{"line_number":1545,"context_line":"        if (src_shard_range.lower \u003d\u003d ShardRange.MAX or"},{"line_number":1546,"context_line":"                src_shard_range.upper \u003d\u003d ShardRange.MIN):"},{"line_number":1547,"context_line":"            return"},{"line_number":1548,"context_line":""},{"line_number":1549,"context_line":"        batch_size \u003d batch_size or self.cleave_row_batch_size"},{"line_number":1550,"context_line":"        for include_deleted in (False, True):"}],"source_content_type":"text/x-python","patch_set":3,"id":"e95922e1_e5cbf807","line":1547,"updated":"2023-02-21 19:23:11.000000000","message":"This feels a little funny -- it makes sense enough, but it makes me wonder why we\u0027d ever be passing an invented MIN-MIN or MAX-MAX shard range here at all. I thought we only added those for the sake of dealing with gaps/overlaps...","commit_id":"06166ea4dc6e7dd9ce31db17fb8ed6c5ae85e090"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"63698463b38ad2335e92719c6610b4fa4e7828df","unresolved":false,"context_lines":[{"line_number":1544,"context_line":"        \"\"\""},{"line_number":1545,"context_line":"        if (src_shard_range.lower \u003d\u003d ShardRange.MAX or"},{"line_number":1546,"context_line":"                src_shard_range.upper \u003d\u003d ShardRange.MIN):"},{"line_number":1547,"context_line":"            return"},{"line_number":1548,"context_line":""},{"line_number":1549,"context_line":"        batch_size \u003d batch_size or self.cleave_row_batch_size"},{"line_number":1550,"context_line":"        for include_deleted in (False, True):"}],"source_content_type":"text/x-python","patch_set":3,"id":"fedcdf44_206d70c3","line":1547,"in_reply_to":"0da26cdc_448977bb","updated":"2023-02-22 14:44:29.000000000","message":"Done","commit_id":"06166ea4dc6e7dd9ce31db17fb8ed6c5ae85e090"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"67092f34747a88a9613ed804fb6fe1a86ef37eef","unresolved":true,"context_lines":[{"line_number":1544,"context_line":"        \"\"\""},{"line_number":1545,"context_line":"        if (src_shard_range.lower \u003d\u003d ShardRange.MAX or"},{"line_number":1546,"context_line":"                src_shard_range.upper \u003d\u003d ShardRange.MIN):"},{"line_number":1547,"context_line":"            return"},{"line_number":1548,"context_line":""},{"line_number":1549,"context_line":"        batch_size \u003d batch_size or self.cleave_row_batch_size"},{"line_number":1550,"context_line":"        for include_deleted in (False, True):"}],"source_content_type":"text/x-python","patch_set":3,"id":"0da26cdc_448977bb","line":1547,"in_reply_to":"e95922e1_e5cbf807","updated":"2023-02-22 13:45:42.000000000","message":"I don\u0027t think we do currently pass in null-namespaces, but it\u0027s a generic enough method to be defensive in case it is ever called with a null-namespace, and without this guard it would do the wrong thing and yield objects.\n\nI\u0027ll add a comment.","commit_id":"06166ea4dc6e7dd9ce31db17fb8ed6c5ae85e090"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"bfd67aef540c710568c7c3cf67917edf6547017b","unresolved":true,"context_lines":[{"line_number":1560,"context_line":"                    include_deleted\u003dinclude_deleted,"},{"line_number":1561,"context_line":"                    since_row\u003dsince_row)"},{"line_number":1562,"context_line":"                self.logger.debug("},{"line_number":1563,"context_line":"                    \u0027got %s %s objects from %s in %ss\u0027,"},{"line_number":1564,"context_line":"                    len(objects),"},{"line_number":1565,"context_line":"                    \u0027deleted\u0027 if include_deleted else \u0027undeleted\u0027,"},{"line_number":1566,"context_line":"                    broker.db_file,"}],"source_content_type":"text/x-python","patch_set":3,"id":"0b4a177f_cd650c0d","line":1563,"range":{"start_line":1563,"start_character":31,"end_line":1563,"end_character":38},"updated":"2023-02-21 19:23:11.000000000","message":"nit: Maybe better as \"rows\"?","commit_id":"06166ea4dc6e7dd9ce31db17fb8ed6c5ae85e090"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"67092f34747a88a9613ed804fb6fe1a86ef37eef","unresolved":false,"context_lines":[{"line_number":1560,"context_line":"                    include_deleted\u003dinclude_deleted,"},{"line_number":1561,"context_line":"                    since_row\u003dsince_row)"},{"line_number":1562,"context_line":"                self.logger.debug("},{"line_number":1563,"context_line":"                    \u0027got %s %s objects from %s in %ss\u0027,"},{"line_number":1564,"context_line":"                    len(objects),"},{"line_number":1565,"context_line":"                    \u0027deleted\u0027 if include_deleted else \u0027undeleted\u0027,"},{"line_number":1566,"context_line":"                    broker.db_file,"}],"source_content_type":"text/x-python","patch_set":3,"id":"5af31dda_6905fd2c","line":1563,"range":{"start_line":1563,"start_character":31,"end_line":1563,"end_character":38},"in_reply_to":"0b4a177f_cd650c0d","updated":"2023-02-22 13:45:42.000000000","message":"Done","commit_id":"06166ea4dc6e7dd9ce31db17fb8ed6c5ae85e090"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"bfd67aef540c710568c7c3cf67917edf6547017b","unresolved":true,"context_lines":[{"line_number":1562,"context_line":"                self.logger.debug("},{"line_number":1563,"context_line":"                    \u0027got %s %s objects from %s in %ss\u0027,"},{"line_number":1564,"context_line":"                    len(objects),"},{"line_number":1565,"context_line":"                    \u0027deleted\u0027 if include_deleted else \u0027undeleted\u0027,"},{"line_number":1566,"context_line":"                    broker.db_file,"},{"line_number":1567,"context_line":"                    time.time() - start)"},{"line_number":1568,"context_line":"                if objects:"}],"source_content_type":"text/x-python","patch_set":3,"id":"b41f943b_94041fa7","line":1565,"range":{"start_line":1565,"start_character":55,"end_line":1565,"end_character":64},"updated":"2023-02-21 19:23:11.000000000","message":"I\u0027m not sure I\u0027m sold on \"undeleted\" for the nomenclature -- makes it sound (to me) like it used to be deleted, but now is not. \"Live\" or maybe \"active\" suits me better.","commit_id":"06166ea4dc6e7dd9ce31db17fb8ed6c5ae85e090"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"67092f34747a88a9613ed804fb6fe1a86ef37eef","unresolved":false,"context_lines":[{"line_number":1562,"context_line":"                self.logger.debug("},{"line_number":1563,"context_line":"                    \u0027got %s %s objects from %s in %ss\u0027,"},{"line_number":1564,"context_line":"                    len(objects),"},{"line_number":1565,"context_line":"                    \u0027deleted\u0027 if include_deleted else \u0027undeleted\u0027,"},{"line_number":1566,"context_line":"                    broker.db_file,"},{"line_number":1567,"context_line":"                    time.time() - start)"},{"line_number":1568,"context_line":"                if objects:"}],"source_content_type":"text/x-python","patch_set":3,"id":"07d70db5_e34c3f5a","line":1565,"range":{"start_line":1565,"start_character":55,"end_line":1565,"end_character":64},"in_reply_to":"b41f943b_94041fa7","updated":"2023-02-22 13:45:42.000000000","message":"Done. I don\u0027t really want to introduce new vernacular for objects that have not been deleted, so I\u0027ll just log \"deleted\u003dFalse|True\".","commit_id":"06166ea4dc6e7dd9ce31db17fb8ed6c5ae85e090"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"bfd67aef540c710568c7c3cf67917edf6547017b","unresolved":true,"context_lines":[{"line_number":1601,"context_line":"        dest_shard_range_iter \u003d iter(dest_shard_ranges())"},{"line_number":1602,"context_line":"        dest_shard_range \u003d None"},{"line_number":1603,"context_line":"        src_shard_range_marker \u003d src_shard_range.lower"},{"line_number":1604,"context_line":"        for dest_shard_range in dest_shard_range_iter:"},{"line_number":1605,"context_line":"            if dest_shard_range.upper \u003c\u003d src_shard_range.lower:"},{"line_number":1606,"context_line":"                continue"},{"line_number":1607,"context_line":""}],"source_content_type":"text/x-python","patch_set":3,"id":"1977a85f_bb06eb77","line":1604,"updated":"2023-02-21 19:23:11.000000000","message":"So this is the key change from pre- https://review.opendev.org/c/openstack/swift/+/841612/ -- we only walk dest_shard_range_iter *once*.","commit_id":"06166ea4dc6e7dd9ce31db17fb8ed6c5ae85e090"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"67092f34747a88a9613ed804fb6fe1a86ef37eef","unresolved":false,"context_lines":[{"line_number":1601,"context_line":"        dest_shard_range_iter \u003d iter(dest_shard_ranges())"},{"line_number":1602,"context_line":"        dest_shard_range \u003d None"},{"line_number":1603,"context_line":"        src_shard_range_marker \u003d src_shard_range.lower"},{"line_number":1604,"context_line":"        for dest_shard_range in dest_shard_range_iter:"},{"line_number":1605,"context_line":"            if dest_shard_range.upper \u003c\u003d src_shard_range.lower:"},{"line_number":1606,"context_line":"                continue"},{"line_number":1607,"context_line":""}],"source_content_type":"text/x-python","patch_set":3,"id":"5ed2f1d4_15687404","line":1604,"in_reply_to":"1977a85f_bb06eb77","updated":"2023-02-22 13:45:42.000000000","message":"we only ever iterated dest_shard_ranges once (previously, the iter is only constructed once in the for loop)...which was part of the previous bug - as soon as undeleted row names passed the current dest shard range upper, we moved on to the next dest shard range, ignoring any deleted rows.","commit_id":"06166ea4dc6e7dd9ce31db17fb8ed6c5ae85e090"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"bfd67aef540c710568c7c3cf67917edf6547017b","unresolved":true,"context_lines":[{"line_number":1611,"context_line":"                    lower\u003dsrc_shard_range_marker, upper\u003ddest_shard_range.lower)"},{"line_number":1612,"context_line":"                for objs, info in self.yield_objects(broker, sub_src_range):"},{"line_number":1613,"context_line":"                    yield objs, None, info"},{"line_number":1614,"context_line":"            src_shard_range_marker \u003d dest_shard_range.upper"},{"line_number":1615,"context_line":""},{"line_number":1616,"context_line":"            sub_src_range \u003d src_shard_range.copy("},{"line_number":1617,"context_line":"                lower\u003dmax(dest_shard_range.lower, src_shard_range.lower),"}],"source_content_type":"text/x-python","patch_set":3,"id":"04ba5b67_312d13e2","line":1614,"updated":"2023-02-21 19:23:11.000000000","message":"nit: I don\u0027t think it impacts anything, but I think it\u0027d make it easier to reason about when stepping through code if this was done *after* yielding out objects for dest_shard_range.","commit_id":"06166ea4dc6e7dd9ce31db17fb8ed6c5ae85e090"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"67092f34747a88a9613ed804fb6fe1a86ef37eef","unresolved":false,"context_lines":[{"line_number":1611,"context_line":"                    lower\u003dsrc_shard_range_marker, upper\u003ddest_shard_range.lower)"},{"line_number":1612,"context_line":"                for objs, info in self.yield_objects(broker, sub_src_range):"},{"line_number":1613,"context_line":"                    yield objs, None, info"},{"line_number":1614,"context_line":"            src_shard_range_marker \u003d dest_shard_range.upper"},{"line_number":1615,"context_line":""},{"line_number":1616,"context_line":"            sub_src_range \u003d src_shard_range.copy("},{"line_number":1617,"context_line":"                lower\u003dmax(dest_shard_range.lower, src_shard_range.lower),"}],"source_content_type":"text/x-python","patch_set":3,"id":"01c45657_ceb03f96","line":1614,"in_reply_to":"04ba5b67_312d13e2","updated":"2023-02-22 13:45:42.000000000","message":"Done","commit_id":"06166ea4dc6e7dd9ce31db17fb8ed6c5ae85e090"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"bfd67aef540c710568c7c3cf67917edf6547017b","unresolved":true,"context_lines":[{"line_number":1625,"context_line":"        else:"},{"line_number":1626,"context_line":"            # dest_shard_ranges_iter was exhausted before reaching the end of"},{"line_number":1627,"context_line":"            # the source namespace"},{"line_number":1628,"context_line":"            if dest_shard_range:"},{"line_number":1629,"context_line":"                sub_src_range \u003d src_shard_range.copy("},{"line_number":1630,"context_line":"                    lower\u003dmax(dest_shard_range.upper, src_shard_range.lower))"},{"line_number":1631,"context_line":"                for objs, info in self.yield_objects(broker, sub_src_range):"}],"source_content_type":"text/x-python","patch_set":3,"id":"4325af2f_68f3a902","line":1628,"updated":"2023-02-21 19:23:11.000000000","message":"No else? We know from the pre-flight that there\u0027s at least one misplaced object; why isn\u0027t it getting yielded as unplaced? I\u0027m guessing we have this `if` for the sake of the\n\n max(dest_shard_range.upper, src_shard_range.lower)\n\nbelow, but couldn\u0027t we just use `src_shard_range_marker`?","commit_id":"06166ea4dc6e7dd9ce31db17fb8ed6c5ae85e090"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"67092f34747a88a9613ed804fb6fe1a86ef37eef","unresolved":false,"context_lines":[{"line_number":1625,"context_line":"        else:"},{"line_number":1626,"context_line":"            # dest_shard_ranges_iter was exhausted before reaching the end of"},{"line_number":1627,"context_line":"            # the source namespace"},{"line_number":1628,"context_line":"            if dest_shard_range:"},{"line_number":1629,"context_line":"                sub_src_range \u003d src_shard_range.copy("},{"line_number":1630,"context_line":"                    lower\u003dmax(dest_shard_range.upper, src_shard_range.lower))"},{"line_number":1631,"context_line":"                for objs, info in self.yield_objects(broker, sub_src_range):"}],"source_content_type":"text/x-python","patch_set":3,"id":"9d65835e_dfe332c0","line":1628,"in_reply_to":"4325af2f_68f3a902","updated":"2023-02-22 13:45:42.000000000","message":"good point\n\nI was covering the case where there were dest ranges that did not cover the entire source range, but not allowing for no dest ranges.","commit_id":"06166ea4dc6e7dd9ce31db17fb8ed6c5ae85e090"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"c4330f9b3c46b8b086cd01a23ba81b6d0ada25fe","unresolved":true,"context_lines":[{"line_number":1636,"context_line":"            # dest_shard_ranges_iter was exhausted before reaching the end of"},{"line_number":1637,"context_line":"            # the source namespace"},{"line_number":1638,"context_line":"            sub_src_range \u003d src_shard_range.copy("},{"line_number":1639,"context_line":"                lower\u003dmax(src_shard_range_marker, src_shard_range.lower))"},{"line_number":1640,"context_line":"            for objs, info in self.yield_objects(broker, sub_src_range):"},{"line_number":1641,"context_line":"                yield objs, None, info"},{"line_number":1642,"context_line":""}],"source_content_type":"text/x-python","patch_set":5,"id":"e793e354_878fa4fd","line":1639,"updated":"2023-02-23 16:57:51.000000000","message":"nit: Can just say `src_shard_range_marker` -- we already know it\u0027s `\u003e\u003d src_shard_range.lower` because of L1615-1616","commit_id":"7fb195123d2d94bb3dc838ed4ebc744882178c6d"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"1a790baec6cfea430ee675b38df806f975210d58","unresolved":true,"context_lines":[{"line_number":1636,"context_line":"            # dest_shard_ranges_iter was exhausted before reaching the end of"},{"line_number":1637,"context_line":"            # the source namespace"},{"line_number":1638,"context_line":"            sub_src_range \u003d src_shard_range.copy("},{"line_number":1639,"context_line":"                lower\u003dmax(src_shard_range_marker, src_shard_range.lower))"},{"line_number":1640,"context_line":"            for objs, info in self.yield_objects(broker, sub_src_range):"},{"line_number":1641,"context_line":"                yield objs, None, info"},{"line_number":1642,"context_line":""}],"source_content_type":"text/x-python","patch_set":5,"id":"cf52583f_2ab3400e","line":1639,"in_reply_to":"e793e354_878fa4fd","updated":"2023-02-23 18:34:16.000000000","message":"Done thanks","commit_id":"7fb195123d2d94bb3dc838ed4ebc744882178c6d"}]}