)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"02c556ac35785b268b8222a9358dabfff5ce1690","unresolved":true,"context_lines":[{"line_number":4,"context_line":"Commit:     Matthew Oliver \u003cmatt@oliver.net.au\u003e"},{"line_number":5,"context_line":"CommitDate: 2022-11-30 11:31:09 +1100"},{"line_number":6,"context_line":""},{"line_number":7,"context_line":"WIP sharder: update own_sr stats explicitly"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"Previously the own shard range stats were updated as a side effect of"},{"line_number":10,"context_line":"the shard range being fetched from the conrainer DB, but they were not"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":6,"id":"9f874a20_cf265ca6","line":7,"range":{"start_line":7,"start_character":0,"end_line":7,"end_character":3},"updated":"2022-11-30 23:36:05.000000000","message":"This seems to be shaping up -- when do we want to remove the WIP? Or rather, what\u0027s still out there that makes us want to keep it?","commit_id":"1ee33cf5047e3b36d4d93d598ca4d7f0210e7cfe"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"2f24919102650af093ae39885abc5ffa773977ca","unresolved":false,"context_lines":[{"line_number":4,"context_line":"Commit:     Matthew Oliver \u003cmatt@oliver.net.au\u003e"},{"line_number":5,"context_line":"CommitDate: 2022-11-30 11:31:09 +1100"},{"line_number":6,"context_line":""},{"line_number":7,"context_line":"WIP sharder: update own_sr stats explicitly"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"Previously the own shard range stats were updated as a side effect of"},{"line_number":10,"context_line":"the shard range being fetched from the conrainer DB, but they were not"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":6,"id":"efd01260_a0b2a23d","line":7,"range":{"start_line":7,"start_character":0,"end_line":7,"end_character":3},"in_reply_to":"9f874a20_cf265ca6","updated":"2022-12-01 05:38:32.000000000","message":"Oh, boo! How\u0027d I manage to review the *previous* patchset?","commit_id":"1ee33cf5047e3b36d4d93d598ca4d7f0210e7cfe"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"02c556ac35785b268b8222a9358dabfff5ce1690","unresolved":true,"context_lines":[{"line_number":7,"context_line":"WIP sharder: update own_sr stats explicitly"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"Previously the own shard range stats were updated as a side effect of"},{"line_number":10,"context_line":"the shard range being fetched from the conrainer DB, but they were not"},{"line_number":11,"context_line":"persisted. This leads to potential confusion because the own shard"},{"line_number":12,"context_line":"range object is not necessarily representative of what is in the"},{"line_number":13,"context_line":"DB. Also, We don\u0027t need the own_shard_range stats to be updated every"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":6,"id":"202cbf4e_af08c5d0","line":10,"range":{"start_line":10,"start_character":39,"end_line":10,"end_character":48},"updated":"2022-11-30 23:36:05.000000000","message":"nit: container","commit_id":"1ee33cf5047e3b36d4d93d598ca4d7f0210e7cfe"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"2f24919102650af093ae39885abc5ffa773977ca","unresolved":false,"context_lines":[{"line_number":7,"context_line":"WIP sharder: update own_sr stats explicitly"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"Previously the own shard range stats were updated as a side effect of"},{"line_number":10,"context_line":"the shard range being fetched from the conrainer DB, but they were not"},{"line_number":11,"context_line":"persisted. This leads to potential confusion because the own shard"},{"line_number":12,"context_line":"range object is not necessarily representative of what is in the"},{"line_number":13,"context_line":"DB. Also, We don\u0027t need the own_shard_range stats to be updated every"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":6,"id":"c977c362_2c8b534f","line":10,"range":{"start_line":10,"start_character":39,"end_line":10,"end_character":48},"in_reply_to":"202cbf4e_af08c5d0","updated":"2022-12-01 05:38:32.000000000","message":"Done","commit_id":"1ee33cf5047e3b36d4d93d598ca4d7f0210e7cfe"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"02c556ac35785b268b8222a9358dabfff5ce1690","unresolved":true,"context_lines":[{"line_number":9,"context_line":"Previously the own shard range stats were updated as a side effect of"},{"line_number":10,"context_line":"the shard range being fetched from the conrainer DB, but they were not"},{"line_number":11,"context_line":"persisted. This leads to potential confusion because the own shard"},{"line_number":12,"context_line":"range object is not necessarily representative of what is in the"},{"line_number":13,"context_line":"DB. Also, We don\u0027t need the own_shard_range stats to be updated every"},{"line_number":14,"context_line":"time it is fetched from the container DB - only on some uses. This"},{"line_number":15,"context_line":"patch therefore makes the stats updating happen as an explicit"},{"line_number":16,"context_line":"separate operation, when needed. This is also more consistent with how"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":6,"id":"d1e1a069_affce29b","line":13,"range":{"start_line":12,"start_character":50,"end_line":13,"end_character":2},"updated":"2022-11-30 23:36:05.000000000","message":"Well... *which part* of the DB? It seems perfectly representative of *something*, just not necessarily the thing ops are expecting ;-)","commit_id":"1ee33cf5047e3b36d4d93d598ca4d7f0210e7cfe"}],"/PATCHSET_LEVEL":[{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"6e277f82d8699519d22d3e6e8e60bd7b56e2d76a","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":1,"id":"018a575c_5a4ba443","updated":"2022-08-07 23:29:58.000000000","message":"I like it. it makes sense in my opinion to pull out the stats updates from get_own_shard_range, make it less complecated and doing too much and make stats updates more explicit.\n\nI\u0027ll tidy this up a little, but mostly so I can play with rebasing the \"no stats updates before CLEAVED\" patch on this, as that\u0027ll look much cleaner.","commit_id":"139edb1d134fb82e533eadb45a43df3d01bb3b7b"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"de35b62ef8e3ed9b6a744b18043a8697600e8752","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":1,"id":"93c39d18_9fc81358","updated":"2022-08-05 17:46:45.000000000","message":"WIP","commit_id":"139edb1d134fb82e533eadb45a43df3d01bb3b7b"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"2905585e2621bedb401c8479f1128d180e847492","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":1,"id":"2b2c7412_10f25009","in_reply_to":"93c39d18_9fc81358","updated":"2022-08-07 23:31:04.000000000","message":"lol, wrong -1 to get it into WIP, it should be a workflow -1","commit_id":"139edb1d134fb82e533eadb45a43df3d01bb3b7b"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"aca48d97529d6ada7a52bf8b1bfee4c231cc27a8","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":2,"id":"1338d135_e436c54b","updated":"2022-08-15 17:39:54.000000000","message":"https://paste.opendev.org/show/betNMNahIpcqHlBCOMaB/ seems to leave tests happy OMM, but I\u0027ve still got questions about what to do in the sharder.","commit_id":"fe39596194433ce983d19224a03ff2b8773ad1c9"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"2b2b2dbb799bf716bc1976c20eee7f1c757285ec","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":3,"id":"d58beba8_0f571405","updated":"2022-11-28 06:10:03.000000000","message":"just running the tests locally, otherwise, new patchset incoming.","commit_id":"29e9dace0cd075e72c166474704046dbbcdf1f09"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"02c556ac35785b268b8222a9358dabfff5ce1690","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":7,"id":"0d916cd7_3c540071","updated":"2022-11-30 23:36:05.000000000","message":"I\u0027ve got this feeling like we ought to update swift/cli/manage_shard_ranges.py too, probably down in _enable_sharding().","commit_id":"adfbee107c0fee591799b703de2e9e3abef45277"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"b17289a8611be46662c0a89e83aeb7d09232a491","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":8,"id":"601239bd_44f099fe","updated":"2022-12-08 18:03:17.000000000","message":"I was still a little worried about the defaulted shard range, but Al talked me down. :-)","commit_id":"001d931e6a77e7ee9167562a51f8c87ed76e559a"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"06e069e03028c6b92ce38fc6a7730562e4b88613","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":8,"id":"d8a8016d_ccc2fc9b","updated":"2022-12-06 03:34:27.000000000","message":"Yup, nice splitting the updating of stats out of get_own_shard_range. The function should just give us what is currently serialized in the DB.\n\nMoving it out so it is also used properly in s-m-s-r, also good.","commit_id":"001d931e6a77e7ee9167562a51f8c87ed76e559a"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"50faef0ee1d8cb7e90ea3a1706ba15d95ae88f0c","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":8,"id":"90143da8_57f85cb5","updated":"2022-12-09 17:33:47.000000000","message":"recheck\n\nLooks like grenade should be fixed now (other projects have seen passing jobs, anyway).","commit_id":"001d931e6a77e7ee9167562a51f8c87ed76e559a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"cdc812d8c0b0ab950d996bade5041ca605f45bf7","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":8,"id":"741e06c9_dfd21542","updated":"2022-12-09 08:44:04.000000000","message":"recheck\n\ntempest failure seems unrelated","commit_id":"001d931e6a77e7ee9167562a51f8c87ed76e559a"}],"swift/cli/manage_shard_ranges.py":[{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"cdc812d8c0b0ab950d996bade5041ca605f45bf7","unresolved":true,"context_lines":[{"line_number":380,"context_line":"    print(\u0027db_state \u003d %s\u0027 % db_state)"},{"line_number":381,"context_line":"    info \u003d broker.get_info()"},{"line_number":382,"context_line":"    print(\u0027object_count \u003d %d\u0027 % info[\u0027object_count\u0027])"},{"line_number":383,"context_line":"    print(\u0027bytes_used \u003d %d\u0027 % info[\u0027bytes_used\u0027])"},{"line_number":384,"context_line":"    if db_state \u003d\u003d \u0027sharding\u0027:"},{"line_number":385,"context_line":"        print(\u0027Retiring db id: %s\u0027 % broker.get_brokers()[0].get_info()[\u0027id\u0027])"},{"line_number":386,"context_line":"        print(\u0027Cleaving context: %s\u0027 %"}],"source_content_type":"text/x-python","patch_set":8,"id":"8d372606_797b186f","line":383,"updated":"2022-12-09 08:44:04.000000000","message":"I preferred the dump of own_sr to be true to what is in the DB and separately report the up to date stats","commit_id":"001d931e6a77e7ee9167562a51f8c87ed76e559a"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"b17289a8611be46662c0a89e83aeb7d09232a491","unresolved":true,"context_lines":[{"line_number":509,"context_line":"        own_shard_range.epoch \u003d Timestamp.now()"},{"line_number":510,"context_line":"        own_shard_range.state_timestamp \u003d own_shard_range.epoch"},{"line_number":511,"context_line":"    # initialise own_shard_range with current broker object stats..."},{"line_number":512,"context_line":"    update_own_shard_range_stats(broker, own_shard_range)"},{"line_number":513,"context_line":""},{"line_number":514,"context_line":"    with broker.updated_timeout(args.enable_timeout):"},{"line_number":515,"context_line":"        broker.merge_shard_ranges([own_shard_range])"}],"source_content_type":"text/x-python","patch_set":8,"id":"4cbbe513_4cf325fe","line":512,"updated":"2022-12-08 18:03:17.000000000","message":"I suppose, as long as we\u0027re updating it when we\u0027re manually sharding...","commit_id":"001d931e6a77e7ee9167562a51f8c87ed76e559a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"cdc812d8c0b0ab950d996bade5041ca605f45bf7","unresolved":true,"context_lines":[{"line_number":509,"context_line":"        own_shard_range.epoch \u003d Timestamp.now()"},{"line_number":510,"context_line":"        own_shard_range.state_timestamp \u003d own_shard_range.epoch"},{"line_number":511,"context_line":"    # initialise own_shard_range with current broker object stats..."},{"line_number":512,"context_line":"    update_own_shard_range_stats(broker, own_shard_range)"},{"line_number":513,"context_line":""},{"line_number":514,"context_line":"    with broker.updated_timeout(args.enable_timeout):"},{"line_number":515,"context_line":"        broker.merge_shard_ranges([own_shard_range])"}],"source_content_type":"text/x-python","patch_set":8,"id":"c07a8a6f_476b7d66","line":512,"updated":"2022-12-09 08:44:04.000000000","message":"previously this would have been a side-effect of get_own_shard_range so this is preserving same behavior","commit_id":"001d931e6a77e7ee9167562a51f8c87ed76e559a"}],"swift/container/backend.py":[{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"6e277f82d8699519d22d3e6e8e60bd7b56e2d76a","unresolved":true,"context_lines":[{"line_number":1907,"context_line":"        if own_shard_range:"},{"line_number":1908,"context_line":"            info \u003d self.get_info()"},{"line_number":1909,"context_line":"            own_shard_range.update_meta("},{"line_number":1910,"context_line":"                info[\u0027object_count\u0027], info[\u0027bytes_used\u0027])"},{"line_number":1911,"context_line":"        return own_shard_range"},{"line_number":1912,"context_line":""},{"line_number":1913,"context_line":"    def is_own_shard_range(self, shard_range):"}],"source_content_type":"text/x-python","patch_set":1,"id":"47d77be2_d94fb2ce","side":"PARENT","line":1910,"updated":"2022-08-07 23:29:58.000000000","message":"Yeah, ok. I do find it a little crazy that we use this function to get_own_shard_range, but it doesn\u0027t reflact how it\u0027s been serialised at that point in time. And I don\u0027t think anyone really expects to pull the most updated stats out when I\u0027m just asking for my own_shard_range (or at least it isn\u0027t expected).\n\nSo pulling this out explicitly I can really get behind! Thanks. This patch needs a bit of a clean up, but yeah we can put the \u0027not update stats before CLEAVED\u0027 patch behind this and get a cleaner output and less complecating this function. I like it!","commit_id":"3a7b89506d4b1d5ec28c28c4edf0cfeaccf19dc6"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"2b2b2dbb799bf716bc1976c20eee7f1c757285ec","unresolved":false,"context_lines":[{"line_number":1907,"context_line":"        if own_shard_range:"},{"line_number":1908,"context_line":"            info \u003d self.get_info()"},{"line_number":1909,"context_line":"            own_shard_range.update_meta("},{"line_number":1910,"context_line":"                info[\u0027object_count\u0027], info[\u0027bytes_used\u0027])"},{"line_number":1911,"context_line":"        return own_shard_range"},{"line_number":1912,"context_line":""},{"line_number":1913,"context_line":"    def is_own_shard_range(self, shard_range):"}],"source_content_type":"text/x-python","patch_set":1,"id":"2d4f6528_f98053d3","side":"PARENT","line":1910,"in_reply_to":"47d77be2_d94fb2ce","updated":"2022-11-28 06:10:03.000000000","message":"Ack","commit_id":"3a7b89506d4b1d5ec28c28c4edf0cfeaccf19dc6"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"8eca018af7120f45b37ef18d39ef1473be44059c","unresolved":true,"context_lines":[{"line_number":421,"context_line":"        Returns True if a broker has shard range state that would be necessary"},{"line_number":422,"context_line":"        for sharding to have been initiated, False otherwise."},{"line_number":423,"context_line":"        \"\"\""},{"line_number":424,"context_line":"        own_shard_range \u003d self.get_own_shard_range()"},{"line_number":425,"context_line":"        if own_shard_range.state in (ShardRange.SHARDING,"},{"line_number":426,"context_line":"                                     ShardRange.SHRINKING,"},{"line_number":427,"context_line":"                                     ShardRange.SHARDED,"}],"source_content_type":"text/x-python","patch_set":2,"id":"2ef0afd8_fbad3f36","line":424,"updated":"2022-08-15 04:36:13.000000000","message":"Check -- stats don\u0027t need to be up-to-date here...","commit_id":"fe39596194433ce983d19224a03ff2b8773ad1c9"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a2adcdc6fe6d29a51adf7e8cd5851b567b5b2b0f","unresolved":false,"context_lines":[{"line_number":421,"context_line":"        Returns True if a broker has shard range state that would be necessary"},{"line_number":422,"context_line":"        for sharding to have been initiated, False otherwise."},{"line_number":423,"context_line":"        \"\"\""},{"line_number":424,"context_line":"        own_shard_range \u003d self.get_own_shard_range()"},{"line_number":425,"context_line":"        if own_shard_range.state in (ShardRange.SHARDING,"},{"line_number":426,"context_line":"                                     ShardRange.SHRINKING,"},{"line_number":427,"context_line":"                                     ShardRange.SHARDED,"}],"source_content_type":"text/x-python","patch_set":2,"id":"85ce64be_dabcac90","line":424,"in_reply_to":"2ef0afd8_fbad3f36","updated":"2022-12-01 14:24:15.000000000","message":"Ack","commit_id":"fe39596194433ce983d19224a03ff2b8773ad1c9"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"8eca018af7120f45b37ef18d39ef1473be44059c","unresolved":true,"context_lines":[{"line_number":1864,"context_line":"            required_upper \u003d min(end_marker or own_shard_range.upper,"},{"line_number":1865,"context_line":"                                 own_shard_range.upper)"},{"line_number":1866,"context_line":"            if required_upper \u003e last_upper:"},{"line_number":1867,"context_line":"                filler_sr \u003d self.get_own_shard_range()"},{"line_number":1868,"context_line":"                filler_sr.lower \u003d last_upper"},{"line_number":1869,"context_line":"                filler_sr.upper \u003d required_upper"},{"line_number":1870,"context_line":"                shard_ranges.append(filler_sr)"}],"source_content_type":"text/x-python","patch_set":2,"id":"ce5c81ea_024f2344","line":1867,"updated":"2022-08-15 04:36:13.000000000","message":"...nor here...","commit_id":"fe39596194433ce983d19224a03ff2b8773ad1c9"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a2adcdc6fe6d29a51adf7e8cd5851b567b5b2b0f","unresolved":false,"context_lines":[{"line_number":1864,"context_line":"            required_upper \u003d min(end_marker or own_shard_range.upper,"},{"line_number":1865,"context_line":"                                 own_shard_range.upper)"},{"line_number":1866,"context_line":"            if required_upper \u003e last_upper:"},{"line_number":1867,"context_line":"                filler_sr \u003d self.get_own_shard_range()"},{"line_number":1868,"context_line":"                filler_sr.lower \u003d last_upper"},{"line_number":1869,"context_line":"                filler_sr.upper \u003d required_upper"},{"line_number":1870,"context_line":"                shard_ranges.append(filler_sr)"}],"source_content_type":"text/x-python","patch_set":2,"id":"a30fa149_fa01ef27","line":1867,"in_reply_to":"ce5c81ea_024f2344","updated":"2022-12-01 14:24:15.000000000","message":"Ack","commit_id":"fe39596194433ce983d19224a03ff2b8773ad1c9"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"8eca018af7120f45b37ef18d39ef1473be44059c","unresolved":true,"context_lines":[{"line_number":1903,"context_line":"            default shard range is returned."},{"line_number":1904,"context_line":"        :return: an instance of :class:`~swift.common.utils.ShardRange`"},{"line_number":1905,"context_line":"        \"\"\""},{"line_number":1906,"context_line":"        own_shard_range \u003d self._own_shard_range(no_default\u003dno_default)"},{"line_number":1907,"context_line":"        return own_shard_range"},{"line_number":1908,"context_line":""},{"line_number":1909,"context_line":"    def is_own_shard_range(self, shard_range):"}],"source_content_type":"text/x-python","patch_set":2,"id":"b8dcc813_307821f8","line":1906,"range":{"start_line":1906,"start_character":31,"end_line":1906,"end_character":47},"updated":"2022-08-15 04:36:13.000000000","message":"Should we get rid of the helper and migrate existing callers to use get_won_shard_range()? Would also mean we could clean up some comments, like down in is_root_container():\n\n\u003e Use internal method so we don\u0027t try to update stats.","commit_id":"fe39596194433ce983d19224a03ff2b8773ad1c9"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"2b2b2dbb799bf716bc1976c20eee7f1c757285ec","unresolved":false,"context_lines":[{"line_number":1903,"context_line":"            default shard range is returned."},{"line_number":1904,"context_line":"        :return: an instance of :class:`~swift.common.utils.ShardRange`"},{"line_number":1905,"context_line":"        \"\"\""},{"line_number":1906,"context_line":"        own_shard_range \u003d self._own_shard_range(no_default\u003dno_default)"},{"line_number":1907,"context_line":"        return own_shard_range"},{"line_number":1908,"context_line":""},{"line_number":1909,"context_line":"    def is_own_shard_range(self, shard_range):"}],"source_content_type":"text/x-python","patch_set":2,"id":"e961e19b_d694dce8","line":1906,"range":{"start_line":1906,"start_character":31,"end_line":1906,"end_character":47},"in_reply_to":"b8dcc813_307821f8","updated":"2022-11-28 06:10:03.000000000","message":"Done","commit_id":"fe39596194433ce983d19224a03ff2b8773ad1c9"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"8eca018af7120f45b37ef18d39ef1473be44059c","unresolved":true,"context_lines":[{"line_number":1954,"context_line":"        :return: True if the fresh DB was successfully created, False"},{"line_number":1955,"context_line":"            otherwise."},{"line_number":1956,"context_line":"        \"\"\""},{"line_number":1957,"context_line":"        epoch \u003d self.get_own_shard_range().epoch"},{"line_number":1958,"context_line":"        if not epoch:"},{"line_number":1959,"context_line":"            self.logger.warning(\"Container \u0027%s\u0027 cannot be set to sharding \""},{"line_number":1960,"context_line":"                                \"state: missing epoch\", self.path)"}],"source_content_type":"text/x-python","patch_set":2,"id":"da4792be_c0d1fbef","line":1957,"updated":"2022-08-15 04:36:13.000000000","message":"...nor here...","commit_id":"fe39596194433ce983d19224a03ff2b8773ad1c9"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a2adcdc6fe6d29a51adf7e8cd5851b567b5b2b0f","unresolved":false,"context_lines":[{"line_number":1954,"context_line":"        :return: True if the fresh DB was successfully created, False"},{"line_number":1955,"context_line":"            otherwise."},{"line_number":1956,"context_line":"        \"\"\""},{"line_number":1957,"context_line":"        epoch \u003d self.get_own_shard_range().epoch"},{"line_number":1958,"context_line":"        if not epoch:"},{"line_number":1959,"context_line":"            self.logger.warning(\"Container \u0027%s\u0027 cannot be set to sharding \""},{"line_number":1960,"context_line":"                                \"state: missing epoch\", self.path)"}],"source_content_type":"text/x-python","patch_set":2,"id":"1d251419_46beee93","line":1957,"in_reply_to":"da4792be_c0d1fbef","updated":"2022-12-01 14:24:15.000000000","message":"Ack","commit_id":"fe39596194433ce983d19224a03ff2b8773ad1c9"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"8eca018af7120f45b37ef18d39ef1473be44059c","unresolved":true,"context_lines":[{"line_number":2287,"context_line":"            # container not big enough to shard"},{"line_number":2288,"context_line":"            return [], False"},{"line_number":2289,"context_line":""},{"line_number":2290,"context_line":"        own_shard_range \u003d self.get_own_shard_range()"},{"line_number":2291,"context_line":"        progress \u003d 0"},{"line_number":2292,"context_line":"        progress_reliable \u003d True"},{"line_number":2293,"context_line":"        # update initial state to account for any existing shard ranges"}],"source_content_type":"text/x-python","patch_set":2,"id":"6f5020bb_bb4563f8","line":2290,"updated":"2022-08-15 04:36:13.000000000","message":"...and not here -- so if we ever need up-to-date stats, it\u0027ll be in the sharder.","commit_id":"fe39596194433ce983d19224a03ff2b8773ad1c9"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a2adcdc6fe6d29a51adf7e8cd5851b567b5b2b0f","unresolved":false,"context_lines":[{"line_number":2287,"context_line":"            # container not big enough to shard"},{"line_number":2288,"context_line":"            return [], False"},{"line_number":2289,"context_line":""},{"line_number":2290,"context_line":"        own_shard_range \u003d self.get_own_shard_range()"},{"line_number":2291,"context_line":"        progress \u003d 0"},{"line_number":2292,"context_line":"        progress_reliable \u003d True"},{"line_number":2293,"context_line":"        # update initial state to account for any existing shard ranges"}],"source_content_type":"text/x-python","patch_set":2,"id":"1b711582_9fa037a4","line":2290,"in_reply_to":"6f5020bb_bb4563f8","updated":"2022-12-01 14:24:15.000000000","message":"Ack","commit_id":"fe39596194433ce983d19224a03ff2b8773ad1c9"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"02c556ac35785b268b8222a9358dabfff5ce1690","unresolved":true,"context_lines":[{"line_number":1889,"context_line":""},{"line_number":1890,"context_line":"        return shard_ranges"},{"line_number":1891,"context_line":""},{"line_number":1892,"context_line":"    def _own_shard_range(self, no_default\u003dFalse):"},{"line_number":1893,"context_line":"        shard_ranges \u003d self.get_shard_ranges(include_own\u003dTrue,"},{"line_number":1894,"context_line":"                                             include_deleted\u003dTrue,"},{"line_number":1895,"context_line":"                                             exclude_others\u003dTrue)"}],"source_content_type":"text/x-python","patch_set":6,"id":"a2538958_be4b3251","side":"PARENT","line":1892,"updated":"2022-11-30 23:36:05.000000000","message":"I love that we\u0027re pruning this!","commit_id":"a1939cba037926400cd777059ee511481916cb08"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a2adcdc6fe6d29a51adf7e8cd5851b567b5b2b0f","unresolved":false,"context_lines":[{"line_number":1889,"context_line":""},{"line_number":1890,"context_line":"        return shard_ranges"},{"line_number":1891,"context_line":""},{"line_number":1892,"context_line":"    def _own_shard_range(self, no_default\u003dFalse):"},{"line_number":1893,"context_line":"        shard_ranges \u003d self.get_shard_ranges(include_own\u003dTrue,"},{"line_number":1894,"context_line":"                                             include_deleted\u003dTrue,"},{"line_number":1895,"context_line":"                                             exclude_others\u003dTrue)"}],"source_content_type":"text/x-python","patch_set":6,"id":"bba58b08_5cfe771f","side":"PARENT","line":1892,"in_reply_to":"a2538958_be4b3251","updated":"2022-12-01 14:24:15.000000000","message":"Ack","commit_id":"a1939cba037926400cd777059ee511481916cb08"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"02c556ac35785b268b8222a9358dabfff5ce1690","unresolved":true,"context_lines":[{"line_number":1895,"context_line":"        such range has been persisted in the broker\u0027s shard ranges table then a"},{"line_number":1896,"context_line":"        default shard range representing the entire namespace will be returned."},{"line_number":1897,"context_line":""},{"line_number":1898,"context_line":"        The returned shard range will be updated with the current object stats"},{"line_number":1899,"context_line":"        for this broker and a meta timestamp set to the current time. For these"},{"line_number":1900,"context_line":"        values to be persisted the caller must merge the shard range."},{"line_number":1901,"context_line":""}],"source_content_type":"text/x-python","patch_set":6,"id":"62bb59b9_6fce4484","line":1898,"range":{"start_line":1898,"start_character":33,"end_line":1898,"end_character":78},"updated":"2022-11-30 23:36:05.000000000","message":"Is this still true? Pretty sure the *opposite* is true now...","commit_id":"1ee33cf5047e3b36d4d93d598ca4d7f0210e7cfe"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a2adcdc6fe6d29a51adf7e8cd5851b567b5b2b0f","unresolved":false,"context_lines":[{"line_number":1895,"context_line":"        such range has been persisted in the broker\u0027s shard ranges table then a"},{"line_number":1896,"context_line":"        default shard range representing the entire namespace will be returned."},{"line_number":1897,"context_line":""},{"line_number":1898,"context_line":"        The returned shard range will be updated with the current object stats"},{"line_number":1899,"context_line":"        for this broker and a meta timestamp set to the current time. For these"},{"line_number":1900,"context_line":"        values to be persisted the caller must merge the shard range."},{"line_number":1901,"context_line":""}],"source_content_type":"text/x-python","patch_set":6,"id":"63220364_3e1aea83","line":1898,"range":{"start_line":1898,"start_character":33,"end_line":1898,"end_character":78},"in_reply_to":"62bb59b9_6fce4484","updated":"2022-12-01 14:24:15.000000000","message":"Done","commit_id":"1ee33cf5047e3b36d4d93d598ca4d7f0210e7cfe"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"02c556ac35785b268b8222a9358dabfff5ce1690","unresolved":true,"context_lines":[{"line_number":1914,"context_line":"        else:"},{"line_number":1915,"context_line":"            own_shard_range \u003d ShardRange("},{"line_number":1916,"context_line":"                self.path, Timestamp.now(), ShardRange.MIN, ShardRange.MAX,"},{"line_number":1917,"context_line":"                state\u003dShardRange.ACTIVE)"},{"line_number":1918,"context_line":"        return own_shard_range"},{"line_number":1919,"context_line":""},{"line_number":1920,"context_line":"    def is_own_shard_range(self, shard_range):"}],"source_content_type":"text/x-python","patch_set":6,"id":"d0ab575c_b2b5a168","line":1917,"updated":"2022-11-30 23:36:05.000000000","message":"I wonder if it\u0027d be worth us getting current stats here -- I mean, 0 almost certainly isn\u0027t going to be right...","commit_id":"1ee33cf5047e3b36d4d93d598ca4d7f0210e7cfe"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a2adcdc6fe6d29a51adf7e8cd5851b567b5b2b0f","unresolved":true,"context_lines":[{"line_number":1914,"context_line":"        else:"},{"line_number":1915,"context_line":"            own_shard_range \u003d ShardRange("},{"line_number":1916,"context_line":"                self.path, Timestamp.now(), ShardRange.MIN, ShardRange.MAX,"},{"line_number":1917,"context_line":"                state\u003dShardRange.ACTIVE)"},{"line_number":1918,"context_line":"        return own_shard_range"},{"line_number":1919,"context_line":""},{"line_number":1920,"context_line":"    def is_own_shard_range(self, shard_range):"}],"source_content_type":"text/x-python","patch_set":6,"id":"c7db830b_5fcb59d5","line":1917,"in_reply_to":"d0ab575c_b2b5a168","updated":"2022-12-01 14:24:15.000000000","message":"I can\u0027t find an case where we care about the stats of a default ShardRange, and it feels odd that we\u0027d make the stats of a default accurate when the bounds and state aren\u0027t. I think I\u0027d prefer the docstring to make it clear that the stats of the returned shard range may be out of date.","commit_id":"1ee33cf5047e3b36d4d93d598ca4d7f0210e7cfe"}],"swift/container/sharder.py":[{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"6e277f82d8699519d22d3e6e8e60bd7b56e2d76a","unresolved":true,"context_lines":[{"line_number":2059,"context_line":"                          quote(broker.path), tombstones)"},{"line_number":2060,"context_line":"        own_shard_range.update_tombstones(tombstones)"},{"line_number":2061,"context_line":"        self._update_own_shard_range_stats(broker, own_shard_range)"},{"line_number":2062,"context_line":"        print (Timestamp.now().internal)"},{"line_number":2063,"context_line":"        if own_shard_range.reported:"},{"line_number":2064,"context_line":"            return"},{"line_number":2065,"context_line":""}],"source_content_type":"text/x-python","patch_set":1,"id":"06ff553c_16ae1a57","line":2062,"updated":"2022-08-07 23:29:58.000000000","message":"I assume this is some debugging that slipped in :)","commit_id":"139edb1d134fb82e533eadb45a43df3d01bb3b7b"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"02c556ac35785b268b8222a9358dabfff5ce1690","unresolved":false,"context_lines":[{"line_number":2059,"context_line":"                          quote(broker.path), tombstones)"},{"line_number":2060,"context_line":"        own_shard_range.update_tombstones(tombstones)"},{"line_number":2061,"context_line":"        self._update_own_shard_range_stats(broker, own_shard_range)"},{"line_number":2062,"context_line":"        print (Timestamp.now().internal)"},{"line_number":2063,"context_line":"        if own_shard_range.reported:"},{"line_number":2064,"context_line":"            return"},{"line_number":2065,"context_line":""}],"source_content_type":"text/x-python","patch_set":1,"id":"526f2965_d311646a","line":2062,"in_reply_to":"06ff553c_16ae1a57","updated":"2022-11-30 23:36:05.000000000","message":"Done","commit_id":"139edb1d134fb82e533eadb45a43df3d01bb3b7b"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"6e277f82d8699519d22d3e6e8e60bd7b56e2d76a","unresolved":false,"context_lines":[{"line_number":2101,"context_line":"            if is_leader and broker.is_root_container():"},{"line_number":2102,"context_line":"                # bootstrap sharding of root container"},{"line_number":2103,"context_line":"                own_shard_range \u003d broker.get_own_shard_range()"},{"line_number":2104,"context_line":"                self._update_own_shard_range_stats(broker, own_shard_range)"},{"line_number":2105,"context_line":"                self._find_and_enable_sharding_candidates("},{"line_number":2106,"context_line":"                    broker, shard_ranges\u003d[own_shard_range])"},{"line_number":2107,"context_line":""}],"source_content_type":"text/x-python","patch_set":1,"id":"6ebc21e7_af920453","line":2104,"updated":"2022-08-07 23:29:58.000000000","message":"Yeah we definitely want the most updated stats here. Nice.","commit_id":"139edb1d134fb82e533eadb45a43df3d01bb3b7b"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"8eca018af7120f45b37ef18d39ef1473be44059c","unresolved":true,"context_lines":[{"line_number":901,"context_line":""},{"line_number":902,"context_line":"        if compactible_ranges:"},{"line_number":903,"context_line":"            own_shard_range \u003d broker.get_own_shard_range()"},{"line_number":904,"context_line":"            shrink_candidate \u003d self._make_stats_info("},{"line_number":905,"context_line":"                broker, node, own_shard_range)"},{"line_number":906,"context_line":"            # The number of ranges/donors that can be shrunk if the"},{"line_number":907,"context_line":"            # tool is used with the current max_shrinking, max_expanding"}],"source_content_type":"text/x-python","patch_set":2,"id":"fc860868_9242f78b","line":904,"updated":"2022-08-15 04:36:13.000000000","message":"Should we call _update_own_shard_range_stats() before this?","commit_id":"fe39596194433ce983d19224a03ff2b8773ad1c9"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"02c556ac35785b268b8222a9358dabfff5ce1690","unresolved":false,"context_lines":[{"line_number":901,"context_line":""},{"line_number":902,"context_line":"        if compactible_ranges:"},{"line_number":903,"context_line":"            own_shard_range \u003d broker.get_own_shard_range()"},{"line_number":904,"context_line":"            shrink_candidate \u003d self._make_stats_info("},{"line_number":905,"context_line":"                broker, node, own_shard_range)"},{"line_number":906,"context_line":"            # The number of ranges/donors that can be shrunk if the"},{"line_number":907,"context_line":"            # tool is used with the current max_shrinking, max_expanding"}],"source_content_type":"text/x-python","patch_set":2,"id":"776d3342_b5a69cd2","line":904,"in_reply_to":"5371af0a_f46a996c","updated":"2022-11-30 23:36:05.000000000","message":"I think I must\u0027ve just missed the call in _make_stats_info() before.","commit_id":"fe39596194433ce983d19224a03ff2b8773ad1c9"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"59659e1694ca76d5e730101de425b55f33eb6d1e","unresolved":true,"context_lines":[{"line_number":901,"context_line":""},{"line_number":902,"context_line":"        if compactible_ranges:"},{"line_number":903,"context_line":"            own_shard_range \u003d broker.get_own_shard_range()"},{"line_number":904,"context_line":"            shrink_candidate \u003d self._make_stats_info("},{"line_number":905,"context_line":"                broker, node, own_shard_range)"},{"line_number":906,"context_line":"            # The number of ranges/donors that can be shrunk if the"},{"line_number":907,"context_line":"            # tool is used with the current max_shrinking, max_expanding"}],"source_content_type":"text/x-python","patch_set":2,"id":"5371af0a_f46a996c","line":904,"in_reply_to":"a853cfa2_13f0e009","updated":"2022-11-30 17:06:12.000000000","message":"_update_own_shard_range_stats() was being called from inside _make_stats_info but I prefer it being explicitly called here, which ends up being a \u0027Yes\u0027 to Tim\u0027s comment","commit_id":"fe39596194433ce983d19224a03ff2b8773ad1c9"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"2b2b2dbb799bf716bc1976c20eee7f1c757285ec","unresolved":true,"context_lines":[{"line_number":901,"context_line":""},{"line_number":902,"context_line":"        if compactible_ranges:"},{"line_number":903,"context_line":"            own_shard_range \u003d broker.get_own_shard_range()"},{"line_number":904,"context_line":"            shrink_candidate \u003d self._make_stats_info("},{"line_number":905,"context_line":"                broker, node, own_shard_range)"},{"line_number":906,"context_line":"            # The number of ranges/donors that can be shrunk if the"},{"line_number":907,"context_line":"            # tool is used with the current max_shrinking, max_expanding"}],"source_content_type":"text/x-python","patch_set":2,"id":"a853cfa2_13f0e009","line":904,"in_reply_to":"fc860868_9242f78b","updated":"2022-11-28 06:10:03.000000000","message":"It is a function to update stats, so I think it makes sense to be here. Unfortually in other places we call this method (like in _identify_sharding_candidate below) after we check to see if it\u0027s a candidate (which needs updated stats). So in that case it\u0027s being called twice.. but in the other cases, in `_record_sharding_progress` and `_identify_shrinking_candidate` it makes sense to be done in here.","commit_id":"fe39596194433ce983d19224a03ff2b8773ad1c9"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"8eca018af7120f45b37ef18d39ef1473be44059c","unresolved":true,"context_lines":[{"line_number":934,"context_line":"                    # broker to be recorded"},{"line_number":935,"context_line":"                    return"},{"line_number":936,"context_line":""},{"line_number":937,"context_line":"            info \u003d self._make_stats_info(broker, node, own_shard_range)"},{"line_number":938,"context_line":"            info[\u0027state\u0027] \u003d own_shard_range.state_text"},{"line_number":939,"context_line":"            info[\u0027db_state\u0027] \u003d broker.get_db_state()"},{"line_number":940,"context_line":"            states \u003d [ShardRange.FOUND, ShardRange.CREATED,"}],"source_content_type":"text/x-python","patch_set":2,"id":"c055c3b5_b2cc26be","line":937,"updated":"2022-08-15 04:36:13.000000000","message":"Again, I feel like we might want up-to-date stats...","commit_id":"fe39596194433ce983d19224a03ff2b8773ad1c9"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"02c556ac35785b268b8222a9358dabfff5ce1690","unresolved":false,"context_lines":[{"line_number":934,"context_line":"                    # broker to be recorded"},{"line_number":935,"context_line":"                    return"},{"line_number":936,"context_line":""},{"line_number":937,"context_line":"            info \u003d self._make_stats_info(broker, node, own_shard_range)"},{"line_number":938,"context_line":"            info[\u0027state\u0027] \u003d own_shard_range.state_text"},{"line_number":939,"context_line":"            info[\u0027db_state\u0027] \u003d broker.get_db_state()"},{"line_number":940,"context_line":"            states \u003d [ShardRange.FOUND, ShardRange.CREATED,"}],"source_content_type":"text/x-python","patch_set":2,"id":"791d4431_9ba5b870","line":937,"in_reply_to":"15a34bd4_01178e00","updated":"2022-11-30 23:36:05.000000000","message":"Ack","commit_id":"fe39596194433ce983d19224a03ff2b8773ad1c9"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"2b2b2dbb799bf716bc1976c20eee7f1c757285ec","unresolved":true,"context_lines":[{"line_number":934,"context_line":"                    # broker to be recorded"},{"line_number":935,"context_line":"                    return"},{"line_number":936,"context_line":""},{"line_number":937,"context_line":"            info \u003d self._make_stats_info(broker, node, own_shard_range)"},{"line_number":938,"context_line":"            info[\u0027state\u0027] \u003d own_shard_range.state_text"},{"line_number":939,"context_line":"            info[\u0027db_state\u0027] \u003d broker.get_db_state()"},{"line_number":940,"context_line":"            states \u003d [ShardRange.FOUND, ShardRange.CREATED,"}],"source_content_type":"text/x-python","patch_set":2,"id":"15a34bd4_01178e00","line":937,"in_reply_to":"c055c3b5_b2cc26be","updated":"2022-11-28 06:10:03.000000000","message":"We do, up in `_make_stats_info on line 932. So OSR by this point is updated.\n\nBut if what you mean, should we be updating it before we find compactible ranges. Well I thought that too, until it isn\u0027t taking the OSR into account, because it assumes the root, so is a special case to push back into it and needed the object count for the OSR isn\u0027t required for that.","commit_id":"fe39596194433ce983d19224a03ff2b8773ad1c9"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"2b2b2dbb799bf716bc1976c20eee7f1c757285ec","unresolved":true,"context_lines":[{"line_number":914,"context_line":""},{"line_number":915,"context_line":"    def _identify_sharding_candidate(self, broker, node):"},{"line_number":916,"context_line":"        own_shard_range \u003d broker.get_own_shard_range()"},{"line_number":917,"context_line":"        self._update_own_shard_range_stats(broker, own_shard_range)"},{"line_number":918,"context_line":"        if is_sharding_candidate("},{"line_number":919,"context_line":"                own_shard_range, self.shard_container_threshold):"},{"line_number":920,"context_line":"            self.sharding_candidates.append("}],"source_content_type":"text/x-python","patch_set":3,"id":"dabb5b6b_dcb17da5","line":917,"updated":"2022-11-28 06:10:03.000000000","message":"So this one has to be run so the is_sharding_candidate below is using the latest stats.\n\nEven though _update_own_shard_range_stats is called in the _make_stats_info as well.\n\nBut _make_stats_info is called elsewhere, and I think it does make sense that _make_stats_info does update the OSR stats.. seems like the right place to do it.\n\nMaybe this needs to be reworked so it only needs to be called once? Maybe generate the sharding candidate stats dict (and update OSR) before we check is sharding candidate? Then append it, if it is? Or is it better to just update the stats twice to make it clearer?\n\nOn second thought, the _make_stats_info calls stat on the file too.. so extra IO so maybe it isn\u0027t worth it. Will leave it as it is for now.","commit_id":"29e9dace0cd075e72c166474704046dbbcdf1f09"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"2f24919102650af093ae39885abc5ffa773977ca","unresolved":false,"context_lines":[{"line_number":914,"context_line":""},{"line_number":915,"context_line":"    def _identify_sharding_candidate(self, broker, node):"},{"line_number":916,"context_line":"        own_shard_range \u003d broker.get_own_shard_range()"},{"line_number":917,"context_line":"        self._update_own_shard_range_stats(broker, own_shard_range)"},{"line_number":918,"context_line":"        if is_sharding_candidate("},{"line_number":919,"context_line":"                own_shard_range, self.shard_container_threshold):"},{"line_number":920,"context_line":"            self.sharding_candidates.append("}],"source_content_type":"text/x-python","patch_set":3,"id":"23783deb_806ff822","line":917,"in_reply_to":"b10ab371_56c6146f","updated":"2022-12-01 05:38:32.000000000","message":"Done","commit_id":"29e9dace0cd075e72c166474704046dbbcdf1f09"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"59659e1694ca76d5e730101de425b55f33eb6d1e","unresolved":true,"context_lines":[{"line_number":914,"context_line":""},{"line_number":915,"context_line":"    def _identify_sharding_candidate(self, broker, node):"},{"line_number":916,"context_line":"        own_shard_range \u003d broker.get_own_shard_range()"},{"line_number":917,"context_line":"        self._update_own_shard_range_stats(broker, own_shard_range)"},{"line_number":918,"context_line":"        if is_sharding_candidate("},{"line_number":919,"context_line":"                own_shard_range, self.shard_container_threshold):"},{"line_number":920,"context_line":"            self.sharding_candidates.append("}],"source_content_type":"text/x-python","patch_set":3,"id":"b10ab371_56c6146f","line":917,"in_reply_to":"dabb5b6b_dcb17da5","updated":"2022-11-30 17:06:12.000000000","message":"I prefer avoiding the side effect of \u0027make info\u0027 - I\u0027ll try pulling out the update_own_shard_range_stats call, which will also avoid the repeated call here.","commit_id":"29e9dace0cd075e72c166474704046dbbcdf1f09"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"02c556ac35785b268b8222a9358dabfff5ce1690","unresolved":true,"context_lines":[{"line_number":929,"context_line":"    def _update_own_shard_range_stats(self, broker, own_shard_range):"},{"line_number":930,"context_line":"        info \u003d broker.get_info()"},{"line_number":931,"context_line":"        own_shard_range.update_meta("},{"line_number":932,"context_line":"            info[\u0027object_count\u0027], info[\u0027bytes_used\u0027])"},{"line_number":933,"context_line":"        return own_shard_range"},{"line_number":934,"context_line":""},{"line_number":935,"context_line":"    def _make_stats_info(self, broker, node, own_shard_range):"}],"source_content_type":"text/x-python","patch_set":6,"id":"ed8d1602_da4f0044","line":932,"updated":"2022-11-30 23:36:05.000000000","message":"🤔 When do we write back the updated stats? Is it just during _update_root_container() and the initial bootstrapping in _process_broker()? So then... once it goes to sharding, those stats are stuck in time... yeah? Could be useful, having that a little more explicitly.","commit_id":"1ee33cf5047e3b36d4d93d598ca4d7f0210e7cfe"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a2adcdc6fe6d29a51adf7e8cd5851b567b5b2b0f","unresolved":false,"context_lines":[{"line_number":929,"context_line":"    def _update_own_shard_range_stats(self, broker, own_shard_range):"},{"line_number":930,"context_line":"        info \u003d broker.get_info()"},{"line_number":931,"context_line":"        own_shard_range.update_meta("},{"line_number":932,"context_line":"            info[\u0027object_count\u0027], info[\u0027bytes_used\u0027])"},{"line_number":933,"context_line":"        return own_shard_range"},{"line_number":934,"context_line":""},{"line_number":935,"context_line":"    def _make_stats_info(self, broker, node, own_shard_range):"}],"source_content_type":"text/x-python","patch_set":6,"id":"0af5f15e_308a4a96","line":932,"in_reply_to":"ed8d1602_da4f0044","updated":"2022-12-01 14:24:15.000000000","message":"correct. But we\u0027re not changing when the stats were persisted, we\u0027re just stopping the return from get_own_shard_range having stats that may not equal the persisted stats.\n\nI\u0027ll add a comment though","commit_id":"1ee33cf5047e3b36d4d93d598ca4d7f0210e7cfe"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"b17289a8611be46662c0a89e83aeb7d09232a491","unresolved":true,"context_lines":[{"line_number":2310,"context_line":"            if is_leader and broker.is_root_container():"},{"line_number":2311,"context_line":"                # bootstrap sharding of root container"},{"line_number":2312,"context_line":"                own_shard_range \u003d broker.get_own_shard_range()"},{"line_number":2313,"context_line":"                update_own_shard_range_stats(broker, own_shard_range)"},{"line_number":2314,"context_line":"                self._find_and_enable_sharding_candidates("},{"line_number":2315,"context_line":"                    broker, shard_ranges\u003d[own_shard_range])"},{"line_number":2316,"context_line":""}],"source_content_type":"text/x-python","patch_set":8,"id":"f9ad7182_9481c847","line":2313,"updated":"2022-12-08 18:03:17.000000000","message":"... as well as when we\u0027re autosharding... it\u0027s probably fine.","commit_id":"001d931e6a77e7ee9167562a51f8c87ed76e559a"}],"test/unit/container/test_backend.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"8eca018af7120f45b37ef18d39ef1473be44059c","unresolved":true,"context_lines":[{"line_number":4117,"context_line":"        # fill gaps"},{"line_number":4118,"context_line":"        filler \u003d own_shard_range.copy()"},{"line_number":4119,"context_line":"        filler.lower \u003d \u0027h\u0027"},{"line_number":4120,"context_line":"        with mock_timestamp_now():"},{"line_number":4121,"context_line":"            actual \u003d broker.get_shard_ranges(fill_gaps\u003dTrue)"},{"line_number":4122,"context_line":"        self.assertEqual([dict(sr) for sr in undeleted + [filler]],"},{"line_number":4123,"context_line":"                         [dict(sr) for sr in actual])"}],"source_content_type":"text/x-python","patch_set":2,"id":"a9e3b37b_1b634e59","line":4120,"updated":"2022-08-15 04:36:13.000000000","message":"Do we still need all these mock_timestamp_now() contexts?","commit_id":"fe39596194433ce983d19224a03ff2b8773ad1c9"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"2b2b2dbb799bf716bc1976c20eee7f1c757285ec","unresolved":false,"context_lines":[{"line_number":4117,"context_line":"        # fill gaps"},{"line_number":4118,"context_line":"        filler \u003d own_shard_range.copy()"},{"line_number":4119,"context_line":"        filler.lower \u003d \u0027h\u0027"},{"line_number":4120,"context_line":"        with mock_timestamp_now():"},{"line_number":4121,"context_line":"            actual \u003d broker.get_shard_ranges(fill_gaps\u003dTrue)"},{"line_number":4122,"context_line":"        self.assertEqual([dict(sr) for sr in undeleted + [filler]],"},{"line_number":4123,"context_line":"                         [dict(sr) for sr in actual])"}],"source_content_type":"text/x-python","patch_set":2,"id":"9e61b31e_ecc0ed78","line":4120,"in_reply_to":"a9e3b37b_1b634e59","updated":"2022-11-28 06:10:03.000000000","message":"nope, doesn\u0027t look like it!","commit_id":"fe39596194433ce983d19224a03ff2b8773ad1c9"}]}