)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"d9ccef0a44e8dd065c20991b77348db915ee993e","unresolved":true,"context_lines":[{"line_number":4,"context_line":"Commit:     Christian Ohanaja \u003ccohanaja@nvidia.com\u003e"},{"line_number":5,"context_line":"CommitDate: 2025-10-08 15:38:04 -0700"},{"line_number":6,"context_line":""},{"line_number":7,"context_line":"Adds processing time metric to sharder recon logs"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"This adds some new time.time() logs around _process_broker that allow us"},{"line_number":10,"context_line":"to see the length of time processing a db took. The information gets"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":16,"id":"9afd6473_97364d4f","line":7,"updated":"2025-10-10 03:28:56.000000000","message":"s/\"processing time metric\"/\"new sharding_in_progress metrics\"","commit_id":"a67a64d0de25e6c42ae7d957da2db2f66996b4e5"},{"author":{"_account_id":38368,"name":"Christian Ohanaja","display_name":"Christian Ohanaja","email":"cohanaja@nvidia.com","username":"cohanaja"},"change_message_id":"9e429f4bf023d48b922e9de20ebcbb4576a0046d","unresolved":false,"context_lines":[{"line_number":4,"context_line":"Commit:     Christian Ohanaja \u003ccohanaja@nvidia.com\u003e"},{"line_number":5,"context_line":"CommitDate: 2025-10-08 15:38:04 -0700"},{"line_number":6,"context_line":""},{"line_number":7,"context_line":"Adds processing time metric to sharder recon logs"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"This adds some new time.time() logs around _process_broker that allow us"},{"line_number":10,"context_line":"to see the length of time processing a db took. The information gets"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":16,"id":"4e5d9a8f_128d2f68","line":7,"in_reply_to":"9afd6473_97364d4f","updated":"2025-10-10 17:18:14.000000000","message":"Done","commit_id":"a67a64d0de25e6c42ae7d957da2db2f66996b4e5"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"d9ccef0a44e8dd065c20991b77348db915ee993e","unresolved":true,"context_lines":[{"line_number":9,"context_line":"This adds some new time.time() logs around _process_broker that allow us"},{"line_number":10,"context_line":"to see the length of time processing a db took. The information gets"},{"line_number":11,"context_line":"added to the in_progress_shard range for the corresponding db."},{"line_number":12,"context_line":""},{"line_number":13,"context_line":"Change-Id: I4ad1e22fd011652d80269a76353e5fc14db249f5"},{"line_number":14,"context_line":"Signed-off-by: Christian Ohanaja \u003ccohanaja@nvidia.com\u003e"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":16,"id":"2e283e01_c943e76b","line":12,"updated":"2025-10-10 03:28:56.000000000","message":"please add descriptions for each new recon metric, like\n```\nUpgradeImpact: below new recon metrics have been added into ``sharding_in_progress`` section of sharder recon.\ntotal_replicate_calls: Number of replication operations during cleaving\ntotal_replicate_time: Cumulative time spent on replication\nprocessing_time: Time to process a single broker\ndb_sharding_total_elapsed: Total time since sharding started\ntombstones: number of tombstones in total.\n```","commit_id":"a67a64d0de25e6c42ae7d957da2db2f66996b4e5"},{"author":{"_account_id":38368,"name":"Christian Ohanaja","display_name":"Christian Ohanaja","email":"cohanaja@nvidia.com","username":"cohanaja"},"change_message_id":"9e429f4bf023d48b922e9de20ebcbb4576a0046d","unresolved":false,"context_lines":[{"line_number":9,"context_line":"This adds some new time.time() logs around _process_broker that allow us"},{"line_number":10,"context_line":"to see the length of time processing a db took. The information gets"},{"line_number":11,"context_line":"added to the in_progress_shard range for the corresponding db."},{"line_number":12,"context_line":""},{"line_number":13,"context_line":"Change-Id: I4ad1e22fd011652d80269a76353e5fc14db249f5"},{"line_number":14,"context_line":"Signed-off-by: Christian Ohanaja \u003ccohanaja@nvidia.com\u003e"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":16,"id":"55c11f21_03500745","line":12,"in_reply_to":"2e283e01_c943e76b","updated":"2025-10-10 17:18:14.000000000","message":"Done","commit_id":"a67a64d0de25e6c42ae7d957da2db2f66996b4e5"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"254c9071998c517e7b9d57dfc82daec13cfbeb79","unresolved":true,"context_lines":[{"line_number":16,"context_line":""},{"line_number":17,"context_line":"UpgradeImpact: below new recon metrics have been added into"},{"line_number":18,"context_line":"``sharding_in_progress`` section of sharder recon."},{"line_number":19,"context_line":"total_replicate_calls: Number of replication operations during cleaving"},{"line_number":20,"context_line":"total_replicate_time: Cumulative time spent on replication"},{"line_number":21,"context_line":"processing_time: Time to process a single broker"},{"line_number":22,"context_line":"db_sharding_total_elapsed: Total time since sharding started tombstones:"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":17,"id":"971a1623_2c7af252","line":19,"updated":"2025-10-11 03:24:35.000000000","message":"please add some spaces before each name of recon stat:\n```\n  total_replicate_calls: Number of replication operations during cleaving\n  total_replicate_time: Cumulative time spent on replication\n  processing_time: Time to process a single broker\n  db_sharding_total_elapsed: Total time since sharding started\n  tombstones: number of tombstones in total.\n```","commit_id":"6e6e5f81bb0e215234ffeaf94a84ea831ccfd643"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"e3191f8888ad8f7e9c6f733d0baec105faa0dde1","unresolved":false,"context_lines":[{"line_number":16,"context_line":""},{"line_number":17,"context_line":"UpgradeImpact: below new recon metrics have been added into"},{"line_number":18,"context_line":"``sharding_in_progress`` section of sharder recon."},{"line_number":19,"context_line":"total_replicate_calls: Number of replication operations during cleaving"},{"line_number":20,"context_line":"total_replicate_time: Cumulative time spent on replication"},{"line_number":21,"context_line":"processing_time: Time to process a single broker"},{"line_number":22,"context_line":"db_sharding_total_elapsed: Total time since sharding started tombstones:"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":17,"id":"89983716_2d27c65b","line":19,"in_reply_to":"971a1623_2c7af252","updated":"2025-10-11 03:27:47.000000000","message":"Done","commit_id":"6e6e5f81bb0e215234ffeaf94a84ea831ccfd643"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"9bc59668d3dd7c2623fe9cad579a085fd5d2457e","unresolved":true,"context_lines":[{"line_number":14,"context_line":"the number of replicate calls + time spent running those replicate"},{"line_number":15,"context_line":"calls."},{"line_number":16,"context_line":""},{"line_number":17,"context_line":"UpgradeImpact: below new recon metrics have been added into"},{"line_number":18,"context_line":"``sharding_in_progress`` section of sharder recon."},{"line_number":19,"context_line":"total_replicate_calls: Number of replication operations during cleaving"},{"line_number":20,"context_line":"total_replicate_time: Cumulative time spent on replication"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":20,"id":"65615cbb_79ff0663","line":17,"updated":"2025-10-13 23:32:46.000000000","message":"my previous updates into this section was already gone, I think @cohanaja@nvidia.com forgot to rebase and overwrote my previous change at: https://review.opendev.org/c/openstack/swift/+/962305/17..18//COMMIT_MSG","commit_id":"e80a21041291b999d7cb3b8fccfc4ead4acda82f"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"762b3f755e6260d83bf135aae7e4e6fb76a62c02","unresolved":false,"context_lines":[{"line_number":14,"context_line":"the number of replicate calls + time spent running those replicate"},{"line_number":15,"context_line":"calls."},{"line_number":16,"context_line":""},{"line_number":17,"context_line":"UpgradeImpact: below new recon metrics have been added into"},{"line_number":18,"context_line":"``sharding_in_progress`` section of sharder recon."},{"line_number":19,"context_line":"total_replicate_calls: Number of replication operations during cleaving"},{"line_number":20,"context_line":"total_replicate_time: Cumulative time spent on replication"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":20,"id":"9a359d7d_a1071b35","line":17,"in_reply_to":"65615cbb_79ff0663","updated":"2025-10-13 23:38:56.000000000","message":"Done","commit_id":"e80a21041291b999d7cb3b8fccfc4ead4acda82f"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"032f681be93b8ec03a408edf8bd6378ff7196da8","unresolved":true,"context_lines":[{"line_number":19,"context_line":"    total_replicate_time: Cumulative time spent on replication during"},{"line_number":20,"context_line":"        cleaving after it completes."},{"line_number":21,"context_line":"    processing_time: Time spent to process a single sharding broker."},{"line_number":22,"context_line":"    total_sharding_time: Total time since sharding started, this"},{"line_number":23,"context_line":"        timestamp in recon file  will keep increasing even after"},{"line_number":24,"context_line":"        sharding completes."},{"line_number":25,"context_line":"    tombstones: number of tombstones container has in total."}],"source_content_type":"text/x-gerrit-commit-message","patch_set":43,"id":"689f1a35_e0a2e0c0","line":22,"updated":"2025-11-27 00:14:56.000000000","message":"one last comment before the merge, this should be something like\n```\ntotal_sharding_time: Total time sharder spent on sharding from start to finish.\n```","commit_id":"6ff6944debc2d5ba8aba57ec484617655a5a897c"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"6199050bbcde57fac27c7a805a8b8e9fde649904","unresolved":false,"context_lines":[{"line_number":19,"context_line":"    total_replicate_time: Cumulative time spent on replication during"},{"line_number":20,"context_line":"        cleaving after it completes."},{"line_number":21,"context_line":"    processing_time: Time spent to process a single sharding broker."},{"line_number":22,"context_line":"    total_sharding_time: Total time since sharding started, this"},{"line_number":23,"context_line":"        timestamp in recon file  will keep increasing even after"},{"line_number":24,"context_line":"        sharding completes."},{"line_number":25,"context_line":"    tombstones: number of tombstones container has in total."}],"source_content_type":"text/x-gerrit-commit-message","patch_set":43,"id":"223db811_cf77437a","line":22,"in_reply_to":"689f1a35_e0a2e0c0","updated":"2025-12-04 19:57:41.000000000","message":"okay, this is only a change on commit message.","commit_id":"6ff6944debc2d5ba8aba57ec484617655a5a897c"}],"/PATCHSET_LEVEL":[{"author":{"_account_id":38368,"name":"Christian Ohanaja","display_name":"Christian Ohanaja","email":"cohanaja@nvidia.com","username":"cohanaja"},"change_message_id":"751959f941f7bddcc9211702d015d84edcdb3afb","unresolved":true,"context_lines":[],"source_content_type":"","patch_set":3,"id":"f7fc8abc_e368dd64","updated":"2025-09-29 19:08:49.000000000","message":"Haven\u0027t written tests for this yet, just trying to get a solid handle on what we track/when","commit_id":"e157d9c3933f0d5d857cf69a088923684e7230b7"},{"author":{"_account_id":38368,"name":"Christian Ohanaja","display_name":"Christian Ohanaja","email":"cohanaja@nvidia.com","username":"cohanaja"},"change_message_id":"bd6f3a4bf88150b6d97a086c725c611040fee708","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":3,"id":"9be7c7b8_595c4a44","in_reply_to":"f7fc8abc_e368dd64","updated":"2025-10-02 23:33:55.000000000","message":"Done","commit_id":"e157d9c3933f0d5d857cf69a088923684e7230b7"},{"author":{"_account_id":38368,"name":"Christian Ohanaja","display_name":"Christian Ohanaja","email":"cohanaja@nvidia.com","username":"cohanaja"},"change_message_id":"e6bc066a1541c9d711991bdaa533bf4e02beb4ec","unresolved":true,"context_lines":[],"source_content_type":"","patch_set":5,"id":"988243f6_05a0de82","updated":"2025-10-02 18:39:10.000000000","message":"I like what the PR does, things are working well. Struggling with adjusting fake_periods in test_run_forever to not complain about the new time.time() bits we\u0027re adding to one_shard_cycle though. WIP nearly done.","commit_id":"5a9ef6d033045aadd2f04610c43525ba678c4b67"},{"author":{"_account_id":38368,"name":"Christian Ohanaja","display_name":"Christian Ohanaja","email":"cohanaja@nvidia.com","username":"cohanaja"},"change_message_id":"bd6f3a4bf88150b6d97a086c725c611040fee708","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":5,"id":"ec7ae6f0_e999782b","in_reply_to":"988243f6_05a0de82","updated":"2025-10-02 23:33:55.000000000","message":"Done","commit_id":"5a9ef6d033045aadd2f04610c43525ba678c4b67"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"82340b6a65c90275148bbde56dee125dec8dcd38","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":10,"id":"d42a6dad_ecd8c9d9","updated":"2025-10-07 16:46:02.000000000","message":"This looks much better, good job! you are almost there!","commit_id":"02a51f22c7343c9832dd264b7cb2ab6316794824"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"c1b640669f509b3711af83945c1c4f557bfa09ed","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":16,"id":"f986544c_3dff5be8","updated":"2025-10-10 15:57:14.000000000","message":"The production code looks good to me, need more work on test cases though","commit_id":"a67a64d0de25e6c42ae7d957da2db2f66996b4e5"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"762b3f755e6260d83bf135aae7e4e6fb76a62c02","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":21,"id":"1dc16832_aff93398","updated":"2025-10-13 23:38:56.000000000","message":"recheck","commit_id":"5ed86abe4c157f9146d342cea19ebcb4f9b74c8c"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"d4598392dbe0c57a5c6acf8cfe71d9904555a110","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":27,"id":"46e0fad5_a1f345f6","updated":"2025-10-30 04:40:28.000000000","message":"IMHO, the new stats metrics ``time_to_first_cleave`` and ``time_to_last_cleave`` are not correct, can we roll back those changes and re-implement them correctly in a new patch? the new patch will be rebased on top of this existing patch.\n\nAlso, I am not sure those db_logging changes should be in this patch either.","commit_id":"021d3673378b6a1beb5c0e3e0f0415fe42280ef9"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"241bdcf6c5a939dc425b336b477f0a6bbe7c0c80","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":35,"id":"6b6def5a_4a5dc9de","updated":"2025-11-03 15:55:54.000000000","message":"I found CleavingContext.load(broker) - but in my testing that stops giving you the \"right\" context once the database has shifted from the original database to the sharded _\u003cepoch\u003e.db (which causes a re-id?) ... I\u0027m not sure when that happens exactly.  It might be worth it to plumb the context we\u0027re using to stuff the replicate count/timing INTO all the way through update_recon_stats to avoid having to look it up again...","commit_id":"565917f6126148c43814f4b5bd7a9615d20a14d3"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"c0d9e8dbf61056d8fc9057c2cf2fd4c0928dcde8","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":37,"id":"d86bdba7_1d8d25b0","updated":"2025-11-21 16:55:44.000000000","message":"it\u0027s a new approach to calculate the total sharding time, but I don\u0027t think we need to track the in-progress timestamps before sharding completes: ``sharding_total_elapsed \u003d time.time() - float(own_shard_range.epoch)``","commit_id":"886ac32c0e21a226ac6e0b7757540fadc2659a90"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"032f681be93b8ec03a408edf8bd6378ff7196da8","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":43,"id":"dd69b081_05abc5bf","updated":"2025-11-27 00:14:56.000000000","message":"The CleavingContext improvement patch will be a standalone follow-on patch, and this one is almost ready to land!","commit_id":"6ff6944debc2d5ba8aba57ec484617655a5a897c"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"6199050bbcde57fac27c7a805a8b8e9fde649904","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":44,"id":"1f146890_4371a445","updated":"2025-12-04 19:57:41.000000000","message":"Nice work, these stats are very helpful to look into how well sharders do internally!","commit_id":"9babfefade23df1c9c3d562dcea8e21372516bb0"}],"doc/source/overview_container_sharding.rst":[{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"6c94eb218818a6b6f535283dcc3602a03f29e746","unresolved":true,"context_lines":[{"line_number":169,"context_line":"                \"cleaved\": 2,"},{"line_number":170,"context_line":"                \"container\": \"c1\","},{"line_number":171,"context_line":"                \"created\": 5,"},{"line_number":172,"context_line":"                \"db_sharding_total_elapsed\": 210.0911569595337,"},{"line_number":173,"context_line":"                \"db_state\": \"sharding\","},{"line_number":174,"context_line":"                \"error\": null,"},{"line_number":175,"context_line":"                \"file_size\": 26624,"}],"source_content_type":"text/x-rst","patch_set":22,"id":"1f410614_6bbc8352","line":172,"updated":"2025-10-13 23:55:40.000000000","message":"we probably only need three decimal places，but let\u0027s not change it after release is built","commit_id":"2aa162f9d07d9ff72322b889aa9ca2a69e7e80aa"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"c7a100f680bc4c01b93d73bf4b64867077e9dfd5","unresolved":true,"context_lines":[{"line_number":169,"context_line":"                \"cleaved\": 2,"},{"line_number":170,"context_line":"                \"container\": \"c1\","},{"line_number":171,"context_line":"                \"created\": 5,"},{"line_number":172,"context_line":"                \"db_sharding_total_elapsed\": 210.0911569595337,"},{"line_number":173,"context_line":"                \"db_state\": \"sharding\","},{"line_number":174,"context_line":"                \"error\": null,"},{"line_number":175,"context_line":"                \"file_size\": 26624,"}],"source_content_type":"text/x-rst","patch_set":22,"id":"f1af814c_20ea5f1c","line":172,"in_reply_to":"1f410614_6bbc8352","updated":"2025-10-28 00:31:34.000000000","message":"probably we should save the timestamp of completion into CleavingContext as well, then we can replace ``db_sharding_total_elapsed`` with the the actual ``db_sharding_completion_time``","commit_id":"2aa162f9d07d9ff72322b889aa9ca2a69e7e80aa"},{"author":{"_account_id":38368,"name":"Christian Ohanaja","display_name":"Christian Ohanaja","email":"cohanaja@nvidia.com","username":"cohanaja"},"change_message_id":"87200bbfb95cd6e67a62331146bc22303d3e83e1","unresolved":false,"context_lines":[{"line_number":169,"context_line":"                \"cleaved\": 2,"},{"line_number":170,"context_line":"                \"container\": \"c1\","},{"line_number":171,"context_line":"                \"created\": 5,"},{"line_number":172,"context_line":"                \"db_sharding_total_elapsed\": 210.0911569595337,"},{"line_number":173,"context_line":"                \"db_state\": \"sharding\","},{"line_number":174,"context_line":"                \"error\": null,"},{"line_number":175,"context_line":"                \"file_size\": 26624,"}],"source_content_type":"text/x-rst","patch_set":22,"id":"735566f2_014eec32","line":172,"in_reply_to":"f1af814c_20ea5f1c","updated":"2025-11-22 04:41:23.000000000","message":"Done","commit_id":"2aa162f9d07d9ff72322b889aa9ca2a69e7e80aa"}],"swift/container/sharder.py":[{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"c99c6b30e0a256565228ff0ee2f14697d2d55a46","unresolved":true,"context_lines":[{"line_number":1059,"context_line":"            \u0027max_cycle_time\u0027: longest_time,"},{"line_number":1060,"context_line":"            \u0027mean_time\u0027: mean_time,"},{"line_number":1061,"context_line":"            \u0027median_time\u0027: median_time,"},{"line_number":1062,"context_line":"            \u0027p90_time\u0027: get_percentile_timing(0.9),"},{"line_number":1063,"context_line":"            \u0027p95_time\u0027: get_percentile_timing(0.95),"},{"line_number":1064,"context_line":"            \u0027p99_time\u0027: get_percentile_timing(0.99),"},{"line_number":1065,"context_line":"        }"}],"source_content_type":"text/x-python","patch_set":1,"id":"90fc3754_8f936f63","line":1062,"updated":"2025-09-26 05:26:32.000000000","message":"p90/p95/p99 of cycle times are calculated after each sharder run, I wonder how many entries will be in ``self.cycle_timings`` after each run. E.g. if there are only 10 containers with sharding enabled, do we want to calculate p90/p95/p99 or just list those 10 container cycle times? I am thinking, eventually we\u0027d like to calculate p90/p95/p99 of cycle times on the cluster level, so what information stored here will be important.","commit_id":"61dc95ae948f2aedcdded7c14fbf26f372aa2992"},{"author":{"_account_id":38368,"name":"Christian Ohanaja","display_name":"Christian Ohanaja","email":"cohanaja@nvidia.com","username":"cohanaja"},"change_message_id":"87200bbfb95cd6e67a62331146bc22303d3e83e1","unresolved":false,"context_lines":[{"line_number":1059,"context_line":"            \u0027max_cycle_time\u0027: longest_time,"},{"line_number":1060,"context_line":"            \u0027mean_time\u0027: mean_time,"},{"line_number":1061,"context_line":"            \u0027median_time\u0027: median_time,"},{"line_number":1062,"context_line":"            \u0027p90_time\u0027: get_percentile_timing(0.9),"},{"line_number":1063,"context_line":"            \u0027p95_time\u0027: get_percentile_timing(0.95),"},{"line_number":1064,"context_line":"            \u0027p99_time\u0027: get_percentile_timing(0.99),"},{"line_number":1065,"context_line":"        }"}],"source_content_type":"text/x-python","patch_set":1,"id":"825b0ec9_067f3848","line":1062,"in_reply_to":"190abd2f_dfc75232","updated":"2025-11-22 04:41:23.000000000","message":"Done","commit_id":"61dc95ae948f2aedcdded7c14fbf26f372aa2992"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"629c0db60ee7d42615f78c5661f0e06ad9b70541","unresolved":true,"context_lines":[{"line_number":1059,"context_line":"            \u0027max_cycle_time\u0027: longest_time,"},{"line_number":1060,"context_line":"            \u0027mean_time\u0027: mean_time,"},{"line_number":1061,"context_line":"            \u0027median_time\u0027: median_time,"},{"line_number":1062,"context_line":"            \u0027p90_time\u0027: get_percentile_timing(0.9),"},{"line_number":1063,"context_line":"            \u0027p95_time\u0027: get_percentile_timing(0.95),"},{"line_number":1064,"context_line":"            \u0027p99_time\u0027: get_percentile_timing(0.99),"},{"line_number":1065,"context_line":"        }"}],"source_content_type":"text/x-python","patch_set":1,"id":"7257ca89_b36d4286","line":1062,"in_reply_to":"2c8a2f77_654395ee","updated":"2025-09-29 16:38:05.000000000","message":"since those active sharding is not often on eahc node, probably we can just output the raw individual timing of each cycle for those active container sharding on each node, and collect all those timings on the cluster level and calculate the p90/P95 and etc.","commit_id":"61dc95ae948f2aedcdded7c14fbf26f372aa2992"},{"author":{"_account_id":38368,"name":"Christian Ohanaja","display_name":"Christian Ohanaja","email":"cohanaja@nvidia.com","username":"cohanaja"},"change_message_id":"751959f941f7bddcc9211702d015d84edcdb3afb","unresolved":true,"context_lines":[{"line_number":1059,"context_line":"            \u0027max_cycle_time\u0027: longest_time,"},{"line_number":1060,"context_line":"            \u0027mean_time\u0027: mean_time,"},{"line_number":1061,"context_line":"            \u0027median_time\u0027: median_time,"},{"line_number":1062,"context_line":"            \u0027p90_time\u0027: get_percentile_timing(0.9),"},{"line_number":1063,"context_line":"            \u0027p95_time\u0027: get_percentile_timing(0.95),"},{"line_number":1064,"context_line":"            \u0027p99_time\u0027: get_percentile_timing(0.99),"},{"line_number":1065,"context_line":"        }"}],"source_content_type":"text/x-python","patch_set":1,"id":"7fdf7b59_8d93ee1f","line":1062,"in_reply_to":"7257ca89_b36d4286","updated":"2025-09-29 19:08:49.000000000","message":"Gotcha, I updated it so we track cycle times specifically during the if db_state \u003d\u003d SHARDING part of process_broker, which also gives us the benefit of cutting out the auditing step + cycles where we just visit a db","commit_id":"61dc95ae948f2aedcdded7c14fbf26f372aa2992"},{"author":{"_account_id":38368,"name":"Christian Ohanaja","display_name":"Christian Ohanaja","email":"cohanaja@nvidia.com","username":"cohanaja"},"change_message_id":"6126599ba95e9102e41bbc9eab504ce3825ed338","unresolved":true,"context_lines":[{"line_number":1059,"context_line":"            \u0027max_cycle_time\u0027: longest_time,"},{"line_number":1060,"context_line":"            \u0027mean_time\u0027: mean_time,"},{"line_number":1061,"context_line":"            \u0027median_time\u0027: median_time,"},{"line_number":1062,"context_line":"            \u0027p90_time\u0027: get_percentile_timing(0.9),"},{"line_number":1063,"context_line":"            \u0027p95_time\u0027: get_percentile_timing(0.95),"},{"line_number":1064,"context_line":"            \u0027p99_time\u0027: get_percentile_timing(0.99),"},{"line_number":1065,"context_line":"        }"}],"source_content_type":"text/x-python","patch_set":1,"id":"190abd2f_dfc75232","line":1062,"in_reply_to":"7fdf7b59_8d93ee1f","updated":"2025-10-02 21:57:08.000000000","message":"Acknowledged, this now tracks raw cycle times for actively sharding containers","commit_id":"61dc95ae948f2aedcdded7c14fbf26f372aa2992"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"f492767b4473e1ceab0aad47ae6e92458e4e2c5a","unresolved":true,"context_lines":[{"line_number":1059,"context_line":"            \u0027max_cycle_time\u0027: longest_time,"},{"line_number":1060,"context_line":"            \u0027mean_time\u0027: mean_time,"},{"line_number":1061,"context_line":"            \u0027median_time\u0027: median_time,"},{"line_number":1062,"context_line":"            \u0027p90_time\u0027: get_percentile_timing(0.9),"},{"line_number":1063,"context_line":"            \u0027p95_time\u0027: get_percentile_timing(0.95),"},{"line_number":1064,"context_line":"            \u0027p99_time\u0027: get_percentile_timing(0.99),"},{"line_number":1065,"context_line":"        }"}],"source_content_type":"text/x-python","patch_set":1,"id":"2c8a2f77_654395ee","line":1062,"in_reply_to":"90fc3754_8f936f63","updated":"2025-09-26 16:47:11.000000000","message":"okay, I looked into one container server, there were lots of databases (more than tens of thousands) scanned in ``visited``, so that means there will be enough entries in ``self.cycle_timings``. however I only saw a few of them were in ``sharding_in_progress`` and I assume they would consume most of time during each sharder\u0027s run while other cycles were just skipping idle databases and super quick, and current calculations will be skewed because of this unbalanced distribution.","commit_id":"61dc95ae948f2aedcdded7c14fbf26f372aa2992"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"c99c6b30e0a256565228ff0ee2f14697d2d55a46","unresolved":true,"context_lines":[{"line_number":2552,"context_line":"                    self._process_broker(broker, node, part)"},{"line_number":2553,"context_line":"                    processing_end \u003d time.time()"},{"line_number":2554,"context_line":"                    elapsed \u003d processing_end - processing_start"},{"line_number":2555,"context_line":"                    container_hash, *_ \u003d parse_db_filename(broker.db_file)"},{"line_number":2556,"context_line":"                    self.cycle_timings.append({"},{"line_number":2557,"context_line":"                        \u0027container_hash\u0027: container_hash,"},{"line_number":2558,"context_line":"                        \u0027elapsed\u0027: elapsed,"}],"source_content_type":"text/x-python","patch_set":1,"id":"7cc81e21_1d6c4255","line":2555,"updated":"2025-09-26 05:26:32.000000000","message":"need to catch possible exception raised inside ``parse_db_filename``","commit_id":"61dc95ae948f2aedcdded7c14fbf26f372aa2992"},{"author":{"_account_id":38368,"name":"Christian Ohanaja","display_name":"Christian Ohanaja","email":"cohanaja@nvidia.com","username":"cohanaja"},"change_message_id":"751959f941f7bddcc9211702d015d84edcdb3afb","unresolved":false,"context_lines":[{"line_number":2552,"context_line":"                    self._process_broker(broker, node, part)"},{"line_number":2553,"context_line":"                    processing_end \u003d time.time()"},{"line_number":2554,"context_line":"                    elapsed \u003d processing_end - processing_start"},{"line_number":2555,"context_line":"                    container_hash, *_ \u003d parse_db_filename(broker.db_file)"},{"line_number":2556,"context_line":"                    self.cycle_timings.append({"},{"line_number":2557,"context_line":"                        \u0027container_hash\u0027: container_hash,"},{"line_number":2558,"context_line":"                        \u0027elapsed\u0027: elapsed,"}],"source_content_type":"text/x-python","patch_set":1,"id":"6c9cf54f_010bf1f4","line":2555,"in_reply_to":"7cc81e21_1d6c4255","updated":"2025-09-29 19:08:49.000000000","message":"Done","commit_id":"61dc95ae948f2aedcdded7c14fbf26f372aa2992"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"e913d4a7e7ef563899df4bc0a5e96201eb5ad7c7","unresolved":true,"context_lines":[{"line_number":1009,"context_line":"                \u0027meta_timestamp\u0027: own_shard_range.meta_timestamp.internal,"},{"line_number":1010,"context_line":"                \u0027file_size\u0027: file_size}"},{"line_number":1011,"context_line":""},{"line_number":1012,"context_line":"        if processing_time is not None and own_shard_range.epoch is not None:"},{"line_number":1013,"context_line":"            info[\u0027processing_time\u0027] \u003d processing_time"},{"line_number":1014,"context_line":"            # something in test_sharder is mocking time.time in a way"},{"line_number":1015,"context_line":"            # that causes this to fail, looking into it."}],"source_content_type":"text/x-python","patch_set":5,"id":"d4832add_26dc4be8","line":1012,"updated":"2025-10-02 22:01:12.000000000","message":"is the condition ``own_shard_range.epoch`` necessary?","commit_id":"5a9ef6d033045aadd2f04610c43525ba678c4b67"},{"author":{"_account_id":38368,"name":"Christian Ohanaja","display_name":"Christian Ohanaja","email":"cohanaja@nvidia.com","username":"cohanaja"},"change_message_id":"bd6f3a4bf88150b6d97a086c725c611040fee708","unresolved":false,"context_lines":[{"line_number":1009,"context_line":"                \u0027meta_timestamp\u0027: own_shard_range.meta_timestamp.internal,"},{"line_number":1010,"context_line":"                \u0027file_size\u0027: file_size}"},{"line_number":1011,"context_line":""},{"line_number":1012,"context_line":"        if processing_time is not None and own_shard_range.epoch is not None:"},{"line_number":1013,"context_line":"            info[\u0027processing_time\u0027] \u003d processing_time"},{"line_number":1014,"context_line":"            # something in test_sharder is mocking time.time in a way"},{"line_number":1015,"context_line":"            # that causes this to fail, looking into it."}],"source_content_type":"text/x-python","patch_set":5,"id":"f7303580_54e092f7","line":1012,"in_reply_to":"d4832add_26dc4be8","updated":"2025-10-02 23:33:55.000000000","message":"Good eye, this was here because for one of the tests (one_shard_cycle IIRC) there wasn\u0027t an epoch assigned to the initial shard ranges which I changed a bit ago","commit_id":"5a9ef6d033045aadd2f04610c43525ba678c4b67"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"e913d4a7e7ef563899df4bc0a5e96201eb5ad7c7","unresolved":true,"context_lines":[{"line_number":1073,"context_line":"                return"},{"line_number":1074,"context_line":""},{"line_number":1075,"context_line":"        update_own_shard_range_stats(broker, own_shard_range)"},{"line_number":1076,"context_line":"        info \u003d self._make_stats_info(broker, node, own_shard_range,"},{"line_number":1077,"context_line":"                                     processing_time)"},{"line_number":1078,"context_line":"        info[\u0027state\u0027] \u003d own_shard_range.state_text"},{"line_number":1079,"context_line":"        info[\u0027db_state\u0027] \u003d broker.get_db_state()"}],"source_content_type":"text/x-python","patch_set":5,"id":"2f639e27_17e6d5eb","line":1076,"updated":"2025-10-02 22:01:12.000000000","message":"I think we can just add ``processing_time`` at here\n\n```\n        info \u003d self._make_stats_info(broker, node, own_shard_range,\n                                     processing_time)\n        if processing_time is not None:\n            info[\u0027processing_time\u0027] \u003d processing_time\n        info[\u0027state\u0027] \u003d own_shard_range.state_text\n```","commit_id":"5a9ef6d033045aadd2f04610c43525ba678c4b67"},{"author":{"_account_id":38368,"name":"Christian Ohanaja","display_name":"Christian Ohanaja","email":"cohanaja@nvidia.com","username":"cohanaja"},"change_message_id":"bd6f3a4bf88150b6d97a086c725c611040fee708","unresolved":false,"context_lines":[{"line_number":1073,"context_line":"                return"},{"line_number":1074,"context_line":""},{"line_number":1075,"context_line":"        update_own_shard_range_stats(broker, own_shard_range)"},{"line_number":1076,"context_line":"        info \u003d self._make_stats_info(broker, node, own_shard_range,"},{"line_number":1077,"context_line":"                                     processing_time)"},{"line_number":1078,"context_line":"        info[\u0027state\u0027] \u003d own_shard_range.state_text"},{"line_number":1079,"context_line":"        info[\u0027db_state\u0027] \u003d broker.get_db_state()"}],"source_content_type":"text/x-python","patch_set":5,"id":"45f211a3_98ac7889","line":1076,"in_reply_to":"2f639e27_17e6d5eb","updated":"2025-10-02 23:33:55.000000000","message":"Acknowledged","commit_id":"5a9ef6d033045aadd2f04610c43525ba678c4b67"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"e913d4a7e7ef563899df4bc0a5e96201eb5ad7c7","unresolved":true,"context_lines":[{"line_number":2512,"context_line":"            broker \u003d ContainerBroker(path, logger\u003dself.logger,"},{"line_number":2513,"context_line":"                                     timeout\u003dself.broker_timeout)"},{"line_number":2514,"context_line":"            error \u003d None"},{"line_number":2515,"context_line":"            processing_time \u003d None"},{"line_number":2516,"context_line":"            try:"},{"line_number":2517,"context_line":"                self._identify_sharding_candidate(broker, node)"},{"line_number":2518,"context_line":"                if sharding_enabled(broker):"}],"source_content_type":"text/x-python","patch_set":5,"id":"f7165c92_a3a303cc","line":2515,"updated":"2025-10-02 22:01:12.000000000","message":"these two lines can be combined together as\n```\nerror \u003d processing_time \u003d None\n```","commit_id":"5a9ef6d033045aadd2f04610c43525ba678c4b67"},{"author":{"_account_id":38368,"name":"Christian Ohanaja","display_name":"Christian Ohanaja","email":"cohanaja@nvidia.com","username":"cohanaja"},"change_message_id":"bd6f3a4bf88150b6d97a086c725c611040fee708","unresolved":false,"context_lines":[{"line_number":2512,"context_line":"            broker \u003d ContainerBroker(path, logger\u003dself.logger,"},{"line_number":2513,"context_line":"                                     timeout\u003dself.broker_timeout)"},{"line_number":2514,"context_line":"            error \u003d None"},{"line_number":2515,"context_line":"            processing_time \u003d None"},{"line_number":2516,"context_line":"            try:"},{"line_number":2517,"context_line":"                self._identify_sharding_candidate(broker, node)"},{"line_number":2518,"context_line":"                if sharding_enabled(broker):"}],"source_content_type":"text/x-python","patch_set":5,"id":"89b81a18_8dc5542f","line":2515,"in_reply_to":"f7165c92_a3a303cc","updated":"2025-10-02 23:33:55.000000000","message":"Done","commit_id":"5a9ef6d033045aadd2f04610c43525ba678c4b67"},{"author":{"_account_id":38368,"name":"Christian Ohanaja","display_name":"Christian Ohanaja","email":"cohanaja@nvidia.com","username":"cohanaja"},"change_message_id":"6126599ba95e9102e41bbc9eab504ce3825ed338","unresolved":false,"context_lines":[{"line_number":1011,"context_line":""},{"line_number":1012,"context_line":"        if processing_time is not None and own_shard_range.epoch is not None:"},{"line_number":1013,"context_line":"            info[\u0027processing_time\u0027] \u003d processing_time"},{"line_number":1014,"context_line":"            sharding_total_elapsed \u003d time.time() - float(own_shard_range.epoch)"},{"line_number":1015,"context_line":"            info[\u0027db_sharding_total_elapsed\u0027] \u003d sharding_total_elapsed"},{"line_number":1016,"context_line":"        return info"},{"line_number":1017,"context_line":""}],"source_content_type":"text/x-python","patch_set":7,"id":"4295cab8_3c6d6768","line":1014,"updated":"2025-10-02 21:57:08.000000000","message":"Since these will only appear on the sharding_in_progress candidates stats, we don\u0027t have to worry about the eternally increasing elapsed cycle time issue from before","commit_id":"f098f0578214156ec13f3157948817c122c69ada"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"572475f46344a80a467cf7e320cc5facbc249dea","unresolved":true,"context_lines":[{"line_number":1011,"context_line":""},{"line_number":1012,"context_line":"        if processing_time is not None and own_shard_range.epoch is not None:"},{"line_number":1013,"context_line":"            info[\u0027processing_time\u0027] \u003d processing_time"},{"line_number":1014,"context_line":"            sharding_total_elapsed \u003d time.time() - float(own_shard_range.epoch)"},{"line_number":1015,"context_line":"            info[\u0027db_sharding_total_elapsed\u0027] \u003d sharding_total_elapsed"},{"line_number":1016,"context_line":"        return info"},{"line_number":1017,"context_line":""}],"source_content_type":"text/x-python","patch_set":7,"id":"57ba26bd_3b0151ef","line":1014,"in_reply_to":"4295cab8_3c6d6768","updated":"2025-10-02 22:45:14.000000000","message":"if we are going to evaluate the performance gain of rsync over usync, I think what we need here is the ``process_broker`` call time which was implemented in previous iteration: https://review.opendev.org/c/openstack/swift/+/962305/5/swift/container/sharder.py","commit_id":"f098f0578214156ec13f3157948817c122c69ada"},{"author":{"_account_id":38368,"name":"Christian Ohanaja","display_name":"Christian Ohanaja","email":"cohanaja@nvidia.com","username":"cohanaja"},"change_message_id":"bd6f3a4bf88150b6d97a086c725c611040fee708","unresolved":true,"context_lines":[{"line_number":1011,"context_line":""},{"line_number":1012,"context_line":"        if processing_time is not None and own_shard_range.epoch is not None:"},{"line_number":1013,"context_line":"            info[\u0027processing_time\u0027] \u003d processing_time"},{"line_number":1014,"context_line":"            sharding_total_elapsed \u003d time.time() - float(own_shard_range.epoch)"},{"line_number":1015,"context_line":"            info[\u0027db_sharding_total_elapsed\u0027] \u003d sharding_total_elapsed"},{"line_number":1016,"context_line":"        return info"},{"line_number":1017,"context_line":""}],"source_content_type":"text/x-python","patch_set":7,"id":"8445857e_922ffa3d","line":1014,"in_reply_to":"57ba26bd_3b0151ef","updated":"2025-10-02 23:33:55.000000000","message":"Oh the process_broker call time is what the elapsed time is. If you mean the time in process_broker specifically around `if db_state \u003d\u003d SHARDING`, IIRC in our last meeting we talked about how we might need the time around everything that happens in _process_broker_ since sometimes the issue with slow sharding could be the auditing step","commit_id":"f098f0578214156ec13f3157948817c122c69ada"},{"author":{"_account_id":38368,"name":"Christian Ohanaja","display_name":"Christian Ohanaja","email":"cohanaja@nvidia.com","username":"cohanaja"},"change_message_id":"87200bbfb95cd6e67a62331146bc22303d3e83e1","unresolved":false,"context_lines":[{"line_number":1011,"context_line":""},{"line_number":1012,"context_line":"        if processing_time is not None and own_shard_range.epoch is not None:"},{"line_number":1013,"context_line":"            info[\u0027processing_time\u0027] \u003d processing_time"},{"line_number":1014,"context_line":"            sharding_total_elapsed \u003d time.time() - float(own_shard_range.epoch)"},{"line_number":1015,"context_line":"            info[\u0027db_sharding_total_elapsed\u0027] \u003d sharding_total_elapsed"},{"line_number":1016,"context_line":"        return info"},{"line_number":1017,"context_line":""}],"source_content_type":"text/x-python","patch_set":7,"id":"3ff92ac4_af0e151b","line":1014,"in_reply_to":"8445857e_922ffa3d","updated":"2025-11-22 04:41:23.000000000","message":"Done","commit_id":"f098f0578214156ec13f3157948817c122c69ada"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"faec4db9858ba2a296970d34663f36622f8b5067","unresolved":true,"context_lines":[{"line_number":1067,"context_line":"        if processing_time is not None:"},{"line_number":1068,"context_line":"            info[\u0027processing_time\u0027] \u003d processing_time"},{"line_number":1069,"context_line":"            sharding_total_elapsed \u003d time.time() - float(own_shard_range.epoch)"},{"line_number":1070,"context_line":"            info[\u0027db_sharding_total_elapsed\u0027] \u003d sharding_total_elapsed"},{"line_number":1071,"context_line":""},{"line_number":1072,"context_line":"        info[\u0027state\u0027] \u003d own_shard_range.state_text"},{"line_number":1073,"context_line":"        info[\u0027db_state\u0027] \u003d broker.get_db_state()"}],"source_content_type":"text/x-python","patch_set":9,"id":"34c50648_b98da072","line":1070,"updated":"2025-10-03 20:36:54.000000000","message":"I am not sure if ``db_sharding_total_elapsed`` will be accurate, since ``_record_sharding_progress`` still will get called within some time (``recon_sharded_timeout``) after container sharding is done, so we will probably will often see the last ``db_sharding_total_elapsed`` shown in the recon file is larger than the actual time this container took to finish sharding.\n\ncan we test this patch in vasio to see if this is the case?","commit_id":"4c3cb2b1939911027549419fd09cd4276728051b"},{"author":{"_account_id":38368,"name":"Christian Ohanaja","display_name":"Christian Ohanaja","email":"cohanaja@nvidia.com","username":"cohanaja"},"change_message_id":"f1d752c553710c4ef936bf0aca63e0e0efc2d018","unresolved":true,"context_lines":[{"line_number":1067,"context_line":"        if processing_time is not None:"},{"line_number":1068,"context_line":"            info[\u0027processing_time\u0027] \u003d processing_time"},{"line_number":1069,"context_line":"            sharding_total_elapsed \u003d time.time() - float(own_shard_range.epoch)"},{"line_number":1070,"context_line":"            info[\u0027db_sharding_total_elapsed\u0027] \u003d sharding_total_elapsed"},{"line_number":1071,"context_line":""},{"line_number":1072,"context_line":"        info[\u0027state\u0027] \u003d own_shard_range.state_text"},{"line_number":1073,"context_line":"        info[\u0027db_state\u0027] \u003d broker.get_db_state()"}],"source_content_type":"text/x-python","patch_set":9,"id":"b1a8f9f5_1ebb7482","line":1070,"in_reply_to":"34c50648_b98da072","updated":"2025-10-07 18:17:13.000000000","message":"It does continually increase even after sharding completes, yeah","commit_id":"4c3cb2b1939911027549419fd09cd4276728051b"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"762b3f755e6260d83bf135aae7e4e6fb76a62c02","unresolved":false,"context_lines":[{"line_number":1067,"context_line":"        if processing_time is not None:"},{"line_number":1068,"context_line":"            info[\u0027processing_time\u0027] \u003d processing_time"},{"line_number":1069,"context_line":"            sharding_total_elapsed \u003d time.time() - float(own_shard_range.epoch)"},{"line_number":1070,"context_line":"            info[\u0027db_sharding_total_elapsed\u0027] \u003d sharding_total_elapsed"},{"line_number":1071,"context_line":""},{"line_number":1072,"context_line":"        info[\u0027state\u0027] \u003d own_shard_range.state_text"},{"line_number":1073,"context_line":"        info[\u0027db_state\u0027] \u003d broker.get_db_state()"}],"source_content_type":"text/x-python","patch_set":9,"id":"83a9f81c_776d579d","line":1070,"in_reply_to":"a9bf8a70_6083eafd","updated":"2025-10-13 23:38:56.000000000","message":"I added a description of this ``issue`` in the commit message.","commit_id":"4c3cb2b1939911027549419fd09cd4276728051b"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"9bc59668d3dd7c2623fe9cad579a085fd5d2457e","unresolved":true,"context_lines":[{"line_number":1067,"context_line":"        if processing_time is not None:"},{"line_number":1068,"context_line":"            info[\u0027processing_time\u0027] \u003d processing_time"},{"line_number":1069,"context_line":"            sharding_total_elapsed \u003d time.time() - float(own_shard_range.epoch)"},{"line_number":1070,"context_line":"            info[\u0027db_sharding_total_elapsed\u0027] \u003d sharding_total_elapsed"},{"line_number":1071,"context_line":""},{"line_number":1072,"context_line":"        info[\u0027state\u0027] \u003d own_shard_range.state_text"},{"line_number":1073,"context_line":"        info[\u0027db_state\u0027] \u003d broker.get_db_state()"}],"source_content_type":"text/x-python","patch_set":9,"id":"a9bf8a70_6083eafd","line":1070,"in_reply_to":"b1a8f9f5_1ebb7482","updated":"2025-10-13 23:32:46.000000000","message":"sharding is done:\n```\n    \"sharding_in_progress\": {\n      \"all\": [\n        {\n          \"account\": \"AUTH_test\",\n          \"active\": 4,\n          \"cleaved\": 0,\n          \"container\": \"test2-001\",\n          \"created\": 0,\n          \"db_sharding_total_elapsed\": 150.17631578445435,\n          \"db_state\": \"sharded\",\n          \"error\": null,\n          \"file_size\": 53248,\n          \"found\": 0,\n          \"meta_timestamp\": \"1760397574.57078\",\n          \"node_index\": 1,\n          \"object_count\": 100,\n          \"path\": \"/srv/node4/sdb4/containers/349/d68/57466e72fb44896b95bc5c83171fcd68/57466e72fb44896b95bc5c83171fcd68_1760397424.39450.db\",\n          \"processing_time\": 0.004283428192138672,\n          \"root\": \"AUTH_test/test2-001\",\n          \"state\": \"sharded\",\n          \"tombstones\": -1,\n          \"total_replicate_calls\": 4,\n          \"total_replicate_time\": 0.0754995346069336\n        }\n      ]\n    },\n```\n\nbut the ``db_sharding_total_elapsed`` timestamp kept increasing.\n```\n    \"sharding_in_progress\": {\n      \"all\": [\n        {\n          \"account\": \"AUTH_test\",\n          \"active\": 4,\n          \"cleaved\": 0,\n          \"container\": \"test2-001\",\n          \"created\": 0,\n          \"db_sharding_total_elapsed\": 210.0911569595337,\n          \"db_state\": \"sharded\",\n          \"error\": null,\n          \"file_size\": 53248,\n          \"found\": 0,\n          \"meta_timestamp\": \"1760397634.48558\",\n          \"node_index\": 1,\n          \"object_count\": 100,\n          \"path\": \"/srv/node4/sdb4/containers/349/d68/57466e72fb44896b95bc5c83171fcd68/57466e72fb44896b95bc5c83171fcd68_1760397424.39450.db\",\n          \"processing_time\": 0.003812551498413086,\n          \"root\": \"AUTH_test/test2-001\",\n          \"state\": \"sharded\",\n          \"tombstones\": -1,\n          \"total_replicate_calls\": 4,\n          \"total_replicate_time\": 0.0754995346069336\n        }\n      ]\n    },\n```","commit_id":"4c3cb2b1939911027549419fd09cd4276728051b"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"82340b6a65c90275148bbde56dee125dec8dcd38","unresolved":true,"context_lines":[{"line_number":2077,"context_line":"                self._increment_stat(\u0027cleaved\u0027, \u0027failure\u0027, statsd\u003dTrue)"},{"line_number":2078,"context_line":"                result \u003d CLEAVE_FAILED"},{"line_number":2079,"context_line":"            else:"},{"line_number":2080,"context_line":"                elapsed \u003d round(time.time() - start, 3)"},{"line_number":2081,"context_line":"                self._min_stat(\u0027cleaved\u0027, \u0027min_time\u0027, elapsed)"},{"line_number":2082,"context_line":"                self._max_stat(\u0027cleaved\u0027, \u0027max_time\u0027, elapsed)"},{"line_number":2083,"context_line":"                self.info(broker, \u0027Cleaved %s in %gs\u0027, shard_range,"}],"source_content_type":"text/x-python","patch_set":10,"id":"54ba7f62_22780c16","side":"PARENT","line":2080,"updated":"2025-10-07 16:46:02.000000000","message":"the existing ``elapsed`` measure the total time spent in cleaving a single shard range, this includes: \n1. Merging objects from source broker to shard broker\n2. Replicating the shard container to other nodes.\n\nand the new proposed ``cleaving_context.replication_time`` only measures the accumulated time spent on the second steps. let\u0027s keep the original ``elapsed``.","commit_id":"b035ed1385ee729411adbd459bb03033e90bb13a"},{"author":{"_account_id":38368,"name":"Christian Ohanaja","display_name":"Christian Ohanaja","email":"cohanaja@nvidia.com","username":"cohanaja"},"change_message_id":"9c1801f2e73260a4e8c27bbda3261c4c78b89646","unresolved":false,"context_lines":[{"line_number":2077,"context_line":"                self._increment_stat(\u0027cleaved\u0027, \u0027failure\u0027, statsd\u003dTrue)"},{"line_number":2078,"context_line":"                result \u003d CLEAVE_FAILED"},{"line_number":2079,"context_line":"            else:"},{"line_number":2080,"context_line":"                elapsed \u003d round(time.time() - start, 3)"},{"line_number":2081,"context_line":"                self._min_stat(\u0027cleaved\u0027, \u0027min_time\u0027, elapsed)"},{"line_number":2082,"context_line":"                self._max_stat(\u0027cleaved\u0027, \u0027max_time\u0027, elapsed)"},{"line_number":2083,"context_line":"                self.info(broker, \u0027Cleaved %s in %gs\u0027, shard_range,"}],"source_content_type":"text/x-python","patch_set":10,"id":"35684b02_38972df6","side":"PARENT","line":2080,"in_reply_to":"54ba7f62_22780c16","updated":"2025-10-07 18:16:00.000000000","message":"Acknowledged","commit_id":"b035ed1385ee729411adbd459bb03033e90bb13a"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"82340b6a65c90275148bbde56dee125dec8dcd38","unresolved":true,"context_lines":[{"line_number":654,"context_line":"        yield \u0027misplaced_done\u0027, self.misplaced_done"},{"line_number":655,"context_line":"        yield \u0027ranges_done\u0027, self.ranges_done"},{"line_number":656,"context_line":"        yield \u0027ranges_todo\u0027, self.ranges_todo"},{"line_number":657,"context_line":"        yield \u0027replication_count\u0027, self.replication_count"},{"line_number":658,"context_line":"        yield \u0027replication_time\u0027, self.replication_time"},{"line_number":659,"context_line":""},{"line_number":660,"context_line":"    def __repr__(self):"}],"source_content_type":"text/x-python","patch_set":10,"id":"c6936033_1d0865a3","line":657,"updated":"2025-10-07 16:46:02.000000000","message":"okay, new stats ``replication_count`` and ``replication_time`` will be stored persistently into database file on disk.\n\nI think this is necessary, the disk I/O overhead of storing a few extra fields in sysmeta is negligible compared to the benefit of having complete performance history.","commit_id":"02a51f22c7343c9832dd264b7cb2ab6316794824"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"73b9c57f48b9f23cefcdcc299a84e284cd6cb43b","unresolved":false,"context_lines":[{"line_number":654,"context_line":"        yield \u0027misplaced_done\u0027, self.misplaced_done"},{"line_number":655,"context_line":"        yield \u0027ranges_done\u0027, self.ranges_done"},{"line_number":656,"context_line":"        yield \u0027ranges_todo\u0027, self.ranges_todo"},{"line_number":657,"context_line":"        yield \u0027replication_count\u0027, self.replication_count"},{"line_number":658,"context_line":"        yield \u0027replication_time\u0027, self.replication_time"},{"line_number":659,"context_line":""},{"line_number":660,"context_line":"    def __repr__(self):"}],"source_content_type":"text/x-python","patch_set":10,"id":"76da06c7_7a0cc333","line":657,"in_reply_to":"c6936033_1d0865a3","updated":"2025-10-08 16:36:00.000000000","message":"Acknowledged","commit_id":"02a51f22c7343c9832dd264b7cb2ab6316794824"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"82340b6a65c90275148bbde56dee125dec8dcd38","unresolved":true,"context_lines":[{"line_number":1077,"context_line":"        recent_context \u003d contexts[-1][0]"},{"line_number":1078,"context_line":"        if (recent_context.replication_count"},{"line_number":1079,"context_line":"                and recent_context.replication_time):"},{"line_number":1080,"context_line":"            info[\"total_replicate_calls\"] \u003d recent_context.replication_count"},{"line_number":1081,"context_line":"            info[\"total_replicate_time\"] \u003d recent_context.replication_time"},{"line_number":1082,"context_line":""},{"line_number":1083,"context_line":"        info[\u0027processing_time\u0027] \u003d processing_time"}],"source_content_type":"text/x-python","patch_set":10,"id":"68ec3147_ac162654","line":1080,"updated":"2025-10-07 16:46:02.000000000","message":"when you add new metrics here, they will be included in the recon for every ``process_broker`` is called on a sharding_in_progress container.\n\nHowever, since we are only interested in the final ``info[\"total_replicate_time\"]`` and ``info[\"total_replicate_calls\"]`` after the container sharding is done, if they will only be included in the recon stats after sharding is done, I think it be easier for the recon parser to aggregate those data points later?","commit_id":"02a51f22c7343c9832dd264b7cb2ab6316794824"},{"author":{"_account_id":38368,"name":"Christian Ohanaja","display_name":"Christian Ohanaja","email":"cohanaja@nvidia.com","username":"cohanaja"},"change_message_id":"9c1801f2e73260a4e8c27bbda3261c4c78b89646","unresolved":true,"context_lines":[{"line_number":1077,"context_line":"        recent_context \u003d contexts[-1][0]"},{"line_number":1078,"context_line":"        if (recent_context.replication_count"},{"line_number":1079,"context_line":"                and recent_context.replication_time):"},{"line_number":1080,"context_line":"            info[\"total_replicate_calls\"] \u003d recent_context.replication_count"},{"line_number":1081,"context_line":"            info[\"total_replicate_time\"] \u003d recent_context.replication_time"},{"line_number":1082,"context_line":""},{"line_number":1083,"context_line":"        info[\u0027processing_time\u0027] \u003d processing_time"}],"source_content_type":"text/x-python","patch_set":10,"id":"ec48e3eb_652252e7","line":1080,"in_reply_to":"68ec3147_ac162654","updated":"2025-10-07 18:16:00.000000000","message":"Good point, modified it so we only log the last context\u0027s rep time + call count in recon","commit_id":"02a51f22c7343c9832dd264b7cb2ab6316794824"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"9bc59668d3dd7c2623fe9cad579a085fd5d2457e","unresolved":false,"context_lines":[{"line_number":1077,"context_line":"        recent_context \u003d contexts[-1][0]"},{"line_number":1078,"context_line":"        if (recent_context.replication_count"},{"line_number":1079,"context_line":"                and recent_context.replication_time):"},{"line_number":1080,"context_line":"            info[\"total_replicate_calls\"] \u003d recent_context.replication_count"},{"line_number":1081,"context_line":"            info[\"total_replicate_time\"] \u003d recent_context.replication_time"},{"line_number":1082,"context_line":""},{"line_number":1083,"context_line":"        info[\u0027processing_time\u0027] \u003d processing_time"}],"source_content_type":"text/x-python","patch_set":10,"id":"a50f53d1_60c2f96d","line":1080,"in_reply_to":"ec48e3eb_652252e7","updated":"2025-10-13 23:32:46.000000000","message":"Done","commit_id":"02a51f22c7343c9832dd264b7cb2ab6316794824"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"82340b6a65c90275148bbde56dee125dec8dcd38","unresolved":true,"context_lines":[{"line_number":2078,"context_line":"        else:  # result \u003d\u003d CLEAVE_SUCCESS:"},{"line_number":2079,"context_line":"            self.info(broker, \u0027Replicating new shard container %s for %s\u0027,"},{"line_number":2080,"context_line":"                      quote(shard_broker.path), own_shard_range)"},{"line_number":2081,"context_line":""},{"line_number":2082,"context_line":"            success, responses \u003d self._replicate_object("},{"line_number":2083,"context_line":"                shard_part, shard_broker.db_file, node_id)"},{"line_number":2084,"context_line":"            elapsed \u003d round(time.time() - start, 3)"}],"source_content_type":"text/x-python","patch_set":10,"id":"685c770e_30e6404d","line":2081,"updated":"2025-10-07 16:46:02.000000000","message":"so this should be something like this:\n```\n            replicate_start \u003d time.time()\n            success, responses \u003d self._replicate_object(\n                shard_part, shard_broker.db_file, node_id)\n            cleaving_context.replication_time +\u003d time.time() - replicate_start\n            cleaving_context.replication_count +\u003d 1\n```","commit_id":"02a51f22c7343c9832dd264b7cb2ab6316794824"},{"author":{"_account_id":38368,"name":"Christian Ohanaja","display_name":"Christian Ohanaja","email":"cohanaja@nvidia.com","username":"cohanaja"},"change_message_id":"9c1801f2e73260a4e8c27bbda3261c4c78b89646","unresolved":false,"context_lines":[{"line_number":2078,"context_line":"        else:  # result \u003d\u003d CLEAVE_SUCCESS:"},{"line_number":2079,"context_line":"            self.info(broker, \u0027Replicating new shard container %s for %s\u0027,"},{"line_number":2080,"context_line":"                      quote(shard_broker.path), own_shard_range)"},{"line_number":2081,"context_line":""},{"line_number":2082,"context_line":"            success, responses \u003d self._replicate_object("},{"line_number":2083,"context_line":"                shard_part, shard_broker.db_file, node_id)"},{"line_number":2084,"context_line":"            elapsed \u003d round(time.time() - start, 3)"}],"source_content_type":"text/x-python","patch_set":10,"id":"f1d3312e_de22032f","line":2081,"in_reply_to":"685c770e_30e6404d","updated":"2025-10-07 18:16:00.000000000","message":"Thanks!","commit_id":"02a51f22c7343c9832dd264b7cb2ab6316794824"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"82340b6a65c90275148bbde56dee125dec8dcd38","unresolved":true,"context_lines":[{"line_number":2531,"context_line":""},{"line_number":2532,"context_line":"                    process_start \u003d time.time()"},{"line_number":2533,"context_line":"                    self._process_broker(broker, node, part)"},{"line_number":2534,"context_line":"                    processing_time \u003d time.time() - float(process_start)"},{"line_number":2535,"context_line":""},{"line_number":2536,"context_line":"                    self._increment_stat(\u0027visited\u0027, \u0027success\u0027, statsd\u003dTrue)"},{"line_number":2537,"context_line":"                else:"}],"source_content_type":"text/x-python","patch_set":10,"id":"b2a29fcb_ec958512","line":2534,"updated":"2025-10-07 16:46:02.000000000","message":"just use this, since time.time() already returns a float number.\n```\nprocessing_time \u003d time.time() - process_start\n```","commit_id":"02a51f22c7343c9832dd264b7cb2ab6316794824"},{"author":{"_account_id":38368,"name":"Christian Ohanaja","display_name":"Christian Ohanaja","email":"cohanaja@nvidia.com","username":"cohanaja"},"change_message_id":"9c1801f2e73260a4e8c27bbda3261c4c78b89646","unresolved":false,"context_lines":[{"line_number":2531,"context_line":""},{"line_number":2532,"context_line":"                    process_start \u003d time.time()"},{"line_number":2533,"context_line":"                    self._process_broker(broker, node, part)"},{"line_number":2534,"context_line":"                    processing_time \u003d time.time() - float(process_start)"},{"line_number":2535,"context_line":""},{"line_number":2536,"context_line":"                    self._increment_stat(\u0027visited\u0027, \u0027success\u0027, statsd\u003dTrue)"},{"line_number":2537,"context_line":"                else:"}],"source_content_type":"text/x-python","patch_set":10,"id":"b82d6aca_d45203e8","line":2534,"in_reply_to":"b2a29fcb_ec958512","updated":"2025-10-07 18:16:00.000000000","message":"Acknowledged","commit_id":"02a51f22c7343c9832dd264b7cb2ab6316794824"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"8a55562800846082ee59c728342b61de05f2a682","unresolved":true,"context_lines":[{"line_number":1013,"context_line":"                \u0027account\u0027: broker.account,"},{"line_number":1014,"context_line":"                \u0027container\u0027: broker.container,"},{"line_number":1015,"context_line":"                \u0027root\u0027: broker.root_path,"},{"line_number":1016,"context_line":"                \u0027object_count\u0027: own_shard_range.object_count,"},{"line_number":1017,"context_line":"                \u0027meta_timestamp\u0027: own_shard_range.meta_timestamp.internal,"},{"line_number":1018,"context_line":"                \u0027file_size\u0027: file_size}"},{"line_number":1019,"context_line":""}],"source_content_type":"text/x-python","patch_set":11,"id":"f2e1721a_8f6904bb","line":1016,"updated":"2025-10-07 20:36:43.000000000","message":"\u0027tombstones\u0027 does get replicated within ``_replicate_objects()``, let\u0027s include it as well:\n```\n\u0027tombstones\u0027: own_shard_range.tombstones\n```","commit_id":"5965077cebe46a8d178708544ae10e1ab20dc4e3"},{"author":{"_account_id":38368,"name":"Christian Ohanaja","display_name":"Christian Ohanaja","email":"cohanaja@nvidia.com","username":"cohanaja"},"change_message_id":"0e65b21e6c708a08cdbeccceb5cc2645849019e4","unresolved":false,"context_lines":[{"line_number":1013,"context_line":"                \u0027account\u0027: broker.account,"},{"line_number":1014,"context_line":"                \u0027container\u0027: broker.container,"},{"line_number":1015,"context_line":"                \u0027root\u0027: broker.root_path,"},{"line_number":1016,"context_line":"                \u0027object_count\u0027: own_shard_range.object_count,"},{"line_number":1017,"context_line":"                \u0027meta_timestamp\u0027: own_shard_range.meta_timestamp.internal,"},{"line_number":1018,"context_line":"                \u0027file_size\u0027: file_size}"},{"line_number":1019,"context_line":""}],"source_content_type":"text/x-python","patch_set":11,"id":"9f407635_c25f2f7b","line":1016,"in_reply_to":"f2e1721a_8f6904bb","updated":"2025-10-08 18:21:01.000000000","message":"Done","commit_id":"5965077cebe46a8d178708544ae10e1ab20dc4e3"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"8a55562800846082ee59c728342b61de05f2a682","unresolved":true,"context_lines":[{"line_number":1077,"context_line":"        update_own_shard_range_stats(broker, own_shard_range)"},{"line_number":1078,"context_line":"        info \u003d self._make_stats_info(broker, node, own_shard_range)"},{"line_number":1079,"context_line":""},{"line_number":1080,"context_line":"        if last_context and last_context.cleaving_done:"},{"line_number":1081,"context_line":"            info[\"total_replicate_calls\"] \u003d last_context.replication_count"},{"line_number":1082,"context_line":"            info[\"total_replicate_time\"] \u003d last_context.replication_time"},{"line_number":1083,"context_line":""}],"source_content_type":"text/x-python","patch_set":11,"id":"0e1553f9_25240855","line":1080,"updated":"2025-10-07 20:36:43.000000000","message":"I think the condition of \"sharding completed\" at here is ``db_state \u003d\u003d SHARDED``","commit_id":"5965077cebe46a8d178708544ae10e1ab20dc4e3"},{"author":{"_account_id":38368,"name":"Christian Ohanaja","display_name":"Christian Ohanaja","email":"cohanaja@nvidia.com","username":"cohanaja"},"change_message_id":"0e65b21e6c708a08cdbeccceb5cc2645849019e4","unresolved":false,"context_lines":[{"line_number":1077,"context_line":"        update_own_shard_range_stats(broker, own_shard_range)"},{"line_number":1078,"context_line":"        info \u003d self._make_stats_info(broker, node, own_shard_range)"},{"line_number":1079,"context_line":""},{"line_number":1080,"context_line":"        if last_context and last_context.cleaving_done:"},{"line_number":1081,"context_line":"            info[\"total_replicate_calls\"] \u003d last_context.replication_count"},{"line_number":1082,"context_line":"            info[\"total_replicate_time\"] \u003d last_context.replication_time"},{"line_number":1083,"context_line":""}],"source_content_type":"text/x-python","patch_set":11,"id":"53fb5606_21b8f334","line":1080,"in_reply_to":"0e1553f9_25240855","updated":"2025-10-08 18:21:01.000000000","message":"Done","commit_id":"5965077cebe46a8d178708544ae10e1ab20dc4e3"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"dcd0a0f54690c81f63b7c5d0b590ee96d7532b94","unresolved":true,"context_lines":[{"line_number":1072,"context_line":"                # broker to be recorded"},{"line_number":1073,"context_line":"                return"},{"line_number":1074,"context_line":""},{"line_number":1075,"context_line":"            last_context \u003d contexts[-1][0]"},{"line_number":1076,"context_line":""},{"line_number":1077,"context_line":"        update_own_shard_range_stats(broker, own_shard_range)"},{"line_number":1078,"context_line":"        info \u003d self._make_stats_info(broker, node, own_shard_range)"}],"source_content_type":"text/x-python","patch_set":12,"id":"8814aa73_cf7fe881","line":1075,"updated":"2025-10-08 20:05:12.000000000","message":"I think it\u0027s possible ``contexts[-1][0]`` might not be the latest one? it would be more robust to be:\n\n```\n    # Sort by timestamp first\n    contexts_sorted \u003d sorted(contexts, key\u003dlambda x: float(x[1]))\n    last_context \u003d contexts[-1][0]\n```","commit_id":"32369bedf2eb93e2fbb57c067d4a4cf06c143207"},{"author":{"_account_id":38368,"name":"Christian Ohanaja","display_name":"Christian Ohanaja","email":"cohanaja@nvidia.com","username":"cohanaja"},"change_message_id":"e06a0733be0b2bfa5165e6a3c378dbcfcee3e88e","unresolved":false,"context_lines":[{"line_number":1072,"context_line":"                # broker to be recorded"},{"line_number":1073,"context_line":"                return"},{"line_number":1074,"context_line":""},{"line_number":1075,"context_line":"            last_context \u003d contexts[-1][0]"},{"line_number":1076,"context_line":""},{"line_number":1077,"context_line":"        update_own_shard_range_stats(broker, own_shard_range)"},{"line_number":1078,"context_line":"        info \u003d self._make_stats_info(broker, node, own_shard_range)"}],"source_content_type":"text/x-python","patch_set":12,"id":"295073c2_e211f3b6","line":1075,"in_reply_to":"8814aa73_cf7fe881","updated":"2025-10-08 21:05:29.000000000","message":"Done","commit_id":"32369bedf2eb93e2fbb57c067d4a4cf06c143207"},{"author":{"_account_id":38368,"name":"Christian Ohanaja","display_name":"Christian Ohanaja","email":"cohanaja@nvidia.com","username":"cohanaja"},"change_message_id":"e06a0733be0b2bfa5165e6a3c378dbcfcee3e88e","unresolved":false,"context_lines":[{"line_number":1072,"context_line":"                # broker to be recorded"},{"line_number":1073,"context_line":"                return"},{"line_number":1074,"context_line":""},{"line_number":1075,"context_line":"            last_context \u003d contexts[-1][0]"},{"line_number":1076,"context_line":""},{"line_number":1077,"context_line":"        update_own_shard_range_stats(broker, own_shard_range)"},{"line_number":1078,"context_line":"        info \u003d self._make_stats_info(broker, node, own_shard_range)"}],"source_content_type":"text/x-python","patch_set":12,"id":"6c32c0fb_37a66cec","line":1075,"in_reply_to":"8814aa73_cf7fe881","updated":"2025-10-08 21:05:29.000000000","message":"Good idea!","commit_id":"32369bedf2eb93e2fbb57c067d4a4cf06c143207"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"dcd0a0f54690c81f63b7c5d0b590ee96d7532b94","unresolved":true,"context_lines":[{"line_number":1053,"context_line":"            category[\u0027top\u0027] \u003d candidates"},{"line_number":1054,"context_line":""},{"line_number":1055,"context_line":"    def _record_sharding_progress(self, broker, node, error,"},{"line_number":1056,"context_line":"                                  processing_time\u003dNone):"},{"line_number":1057,"context_line":"        db_state \u003d broker.get_db_state()"},{"line_number":1058,"context_line":"        if db_state not in (UNSHARDED, SHARDING, SHARDED):"},{"line_number":1059,"context_line":"            return"}],"source_content_type":"text/x-python","patch_set":14,"id":"f825ac43_8a7d1c5d","line":1056,"updated":"2025-10-08 20:05:12.000000000","message":"``processing_time\u003dNone`` is not necessary, since the caller always passes ``processing_time`` into it.","commit_id":"6bce68942cf419a058ad2b2a99fb0faf6616f500"},{"author":{"_account_id":38368,"name":"Christian Ohanaja","display_name":"Christian Ohanaja","email":"cohanaja@nvidia.com","username":"cohanaja"},"change_message_id":"e06a0733be0b2bfa5165e6a3c378dbcfcee3e88e","unresolved":false,"context_lines":[{"line_number":1053,"context_line":"            category[\u0027top\u0027] \u003d candidates"},{"line_number":1054,"context_line":""},{"line_number":1055,"context_line":"    def _record_sharding_progress(self, broker, node, error,"},{"line_number":1056,"context_line":"                                  processing_time\u003dNone):"},{"line_number":1057,"context_line":"        db_state \u003d broker.get_db_state()"},{"line_number":1058,"context_line":"        if db_state not in (UNSHARDED, SHARDING, SHARDED):"},{"line_number":1059,"context_line":"            return"}],"source_content_type":"text/x-python","patch_set":14,"id":"6dd76800_83f88235","line":1056,"in_reply_to":"f825ac43_8a7d1c5d","updated":"2025-10-08 21:05:29.000000000","message":"Acknowledged","commit_id":"6bce68942cf419a058ad2b2a99fb0faf6616f500"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"dcd0a0f54690c81f63b7c5d0b590ee96d7532b94","unresolved":true,"context_lines":[{"line_number":1061,"context_line":"        if own_shard_range.state not in ShardRange.CLEAVING_STATES:"},{"line_number":1062,"context_line":"            return"},{"line_number":1063,"context_line":""},{"line_number":1064,"context_line":"        contexts \u003d CleavingContext.load_all(broker)"},{"line_number":1065,"context_line":"        last_context \u003d None"},{"line_number":1066,"context_line":"        if db_state \u003d\u003d SHARDED:"},{"line_number":1067,"context_line":"            if not contexts:"}],"source_content_type":"text/x-python","patch_set":14,"id":"91eb3d12_45279eea","line":1064,"updated":"2025-10-08 20:05:12.000000000","message":"this line of code should still go to the original code block within ``if db_state \u003d\u003d SHARDED``, since ``last_context`` won\u0027t be a valid value unless ``db_state \u003d\u003d SHARDED`` and we only record  ``replication_count`` and ``replication_time`` after container is sharded.","commit_id":"6bce68942cf419a058ad2b2a99fb0faf6616f500"},{"author":{"_account_id":38368,"name":"Christian Ohanaja","display_name":"Christian Ohanaja","email":"cohanaja@nvidia.com","username":"cohanaja"},"change_message_id":"e06a0733be0b2bfa5165e6a3c378dbcfcee3e88e","unresolved":false,"context_lines":[{"line_number":1061,"context_line":"        if own_shard_range.state not in ShardRange.CLEAVING_STATES:"},{"line_number":1062,"context_line":"            return"},{"line_number":1063,"context_line":""},{"line_number":1064,"context_line":"        contexts \u003d CleavingContext.load_all(broker)"},{"line_number":1065,"context_line":"        last_context \u003d None"},{"line_number":1066,"context_line":"        if db_state \u003d\u003d SHARDED:"},{"line_number":1067,"context_line":"            if not contexts:"}],"source_content_type":"text/x-python","patch_set":14,"id":"eef7c88a_8f2186e2","line":1064,"in_reply_to":"91eb3d12_45279eea","updated":"2025-10-08 21:05:29.000000000","message":"Done","commit_id":"6bce68942cf419a058ad2b2a99fb0faf6616f500"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"dcd0a0f54690c81f63b7c5d0b590ee96d7532b94","unresolved":true,"context_lines":[{"line_number":1062,"context_line":"            return"},{"line_number":1063,"context_line":""},{"line_number":1064,"context_line":"        contexts \u003d CleavingContext.load_all(broker)"},{"line_number":1065,"context_line":"        last_context \u003d None"},{"line_number":1066,"context_line":"        if db_state \u003d\u003d SHARDED:"},{"line_number":1067,"context_line":"            if not contexts:"},{"line_number":1068,"context_line":"                return"}],"source_content_type":"text/x-python","patch_set":14,"id":"e2ba623c_e2f816f2","line":1065,"updated":"2025-10-08 20:05:12.000000000","message":"s/last_context/sharded_context  (name change)","commit_id":"6bce68942cf419a058ad2b2a99fb0faf6616f500"},{"author":{"_account_id":38368,"name":"Christian Ohanaja","display_name":"Christian Ohanaja","email":"cohanaja@nvidia.com","username":"cohanaja"},"change_message_id":"e06a0733be0b2bfa5165e6a3c378dbcfcee3e88e","unresolved":false,"context_lines":[{"line_number":1062,"context_line":"            return"},{"line_number":1063,"context_line":""},{"line_number":1064,"context_line":"        contexts \u003d CleavingContext.load_all(broker)"},{"line_number":1065,"context_line":"        last_context \u003d None"},{"line_number":1066,"context_line":"        if db_state \u003d\u003d SHARDED:"},{"line_number":1067,"context_line":"            if not contexts:"},{"line_number":1068,"context_line":"                return"}],"source_content_type":"text/x-python","patch_set":14,"id":"9186b9fd_2021e71f","line":1065,"in_reply_to":"e2ba623c_e2f816f2","updated":"2025-10-08 21:05:29.000000000","message":"Done","commit_id":"6bce68942cf419a058ad2b2a99fb0faf6616f500"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"dcd0a0f54690c81f63b7c5d0b590ee96d7532b94","unresolved":true,"context_lines":[{"line_number":1078,"context_line":"        update_own_shard_range_stats(broker, own_shard_range)"},{"line_number":1079,"context_line":"        info \u003d self._make_stats_info(broker, node, own_shard_range)"},{"line_number":1080,"context_line":""},{"line_number":1081,"context_line":"        if last_context:"},{"line_number":1082,"context_line":"            info[\"total_replicate_calls\"] \u003d last_context.replication_count"},{"line_number":1083,"context_line":"            info[\"total_replicate_time\"] \u003d last_context.replication_time"},{"line_number":1084,"context_line":""}],"source_content_type":"text/x-python","patch_set":14,"id":"fb485ed2_cb14acd0","line":1081,"updated":"2025-10-08 20:05:12.000000000","message":"I think it\u0027s good to have one more check with last_context.cleaving_done to make sure those two fields have non 0 value. what do you think? @alistairncoles@gmail.com, if a replica receives a fully sharded DB via replication between 3-replicas, it can reach here with ``replication_count\u003d0``.\n\n```\n        if last_context and last_context.cleaving_done:\n            info[\"total_replicate_calls\"] \u003d last_context.replication_count\n            info[\"total_replicate_time\"] \u003d last_context.replication_time\n```","commit_id":"6bce68942cf419a058ad2b2a99fb0faf6616f500"},{"author":{"_account_id":38368,"name":"Christian Ohanaja","display_name":"Christian Ohanaja","email":"cohanaja@nvidia.com","username":"cohanaja"},"change_message_id":"87200bbfb95cd6e67a62331146bc22303d3e83e1","unresolved":false,"context_lines":[{"line_number":1078,"context_line":"        update_own_shard_range_stats(broker, own_shard_range)"},{"line_number":1079,"context_line":"        info \u003d self._make_stats_info(broker, node, own_shard_range)"},{"line_number":1080,"context_line":""},{"line_number":1081,"context_line":"        if last_context:"},{"line_number":1082,"context_line":"            info[\"total_replicate_calls\"] \u003d last_context.replication_count"},{"line_number":1083,"context_line":"            info[\"total_replicate_time\"] \u003d last_context.replication_time"},{"line_number":1084,"context_line":""}],"source_content_type":"text/x-python","patch_set":14,"id":"4dadfe64_12266183","line":1081,"in_reply_to":"fb485ed2_cb14acd0","updated":"2025-11-22 04:41:23.000000000","message":"Done","commit_id":"6bce68942cf419a058ad2b2a99fb0faf6616f500"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"dcd0a0f54690c81f63b7c5d0b590ee96d7532b94","unresolved":true,"context_lines":[{"line_number":1082,"context_line":"            info[\"total_replicate_calls\"] \u003d last_context.replication_count"},{"line_number":1083,"context_line":"            info[\"total_replicate_time\"] \u003d last_context.replication_time"},{"line_number":1084,"context_line":""},{"line_number":1085,"context_line":"        info[\u0027processing_time\u0027] \u003d processing_time"},{"line_number":1086,"context_line":"        sharding_total_elapsed \u003d time.time() - float(own_shard_range.epoch)"},{"line_number":1087,"context_line":"        info[\u0027db_sharding_total_elapsed\u0027] \u003d sharding_total_elapsed"},{"line_number":1088,"context_line":""}],"source_content_type":"text/x-python","patch_set":14,"id":"69a463bb_0df97b29","line":1085,"updated":"2025-10-08 20:05:12.000000000","message":"since ``processing_time`` could be None when caller passes it in (for example, exception raised within _process_broker), we would need\n```\n    if processing_time:\n        info[\u0027processing_time\u0027] \u003d processing_time\n```","commit_id":"6bce68942cf419a058ad2b2a99fb0faf6616f500"},{"author":{"_account_id":38368,"name":"Christian Ohanaja","display_name":"Christian Ohanaja","email":"cohanaja@nvidia.com","username":"cohanaja"},"change_message_id":"e06a0733be0b2bfa5165e6a3c378dbcfcee3e88e","unresolved":false,"context_lines":[{"line_number":1082,"context_line":"            info[\"total_replicate_calls\"] \u003d last_context.replication_count"},{"line_number":1083,"context_line":"            info[\"total_replicate_time\"] \u003d last_context.replication_time"},{"line_number":1084,"context_line":""},{"line_number":1085,"context_line":"        info[\u0027processing_time\u0027] \u003d processing_time"},{"line_number":1086,"context_line":"        sharding_total_elapsed \u003d time.time() - float(own_shard_range.epoch)"},{"line_number":1087,"context_line":"        info[\u0027db_sharding_total_elapsed\u0027] \u003d sharding_total_elapsed"},{"line_number":1088,"context_line":""}],"source_content_type":"text/x-python","patch_set":14,"id":"d8cbf3f9_a7b5af40","line":1085,"in_reply_to":"69a463bb_0df97b29","updated":"2025-10-08 21:05:29.000000000","message":"Done","commit_id":"6bce68942cf419a058ad2b2a99fb0faf6616f500"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"dcd0a0f54690c81f63b7c5d0b590ee96d7532b94","unresolved":true,"context_lines":[{"line_number":2545,"context_line":"                error \u003d err"},{"line_number":2546,"context_line":"            try:"},{"line_number":2547,"context_line":"                self._record_sharding_progress(broker, node, error,"},{"line_number":2548,"context_line":"                                               processing_time)"},{"line_number":2549,"context_line":"            except (Exception, Timeout) as error:"},{"line_number":2550,"context_line":"                self.exception(broker, \u0027Unhandled exception while dumping \u0027"},{"line_number":2551,"context_line":"                                       \u0027progress: %s\u0027, error)"}],"source_content_type":"text/x-python","patch_set":14,"id":"7d280843_843602f2","line":2548,"updated":"2025-10-08 20:05:12.000000000","message":"even if ``_process_broker`` is called, ``_record_sharding_progress`` can be called with ``processing_time\u003dNone`` if an exception occurs during ``_process_broker`` execution, that\u0027ll be great if we can add a test case to cover this","commit_id":"6bce68942cf419a058ad2b2a99fb0faf6616f500"},{"author":{"_account_id":38368,"name":"Christian Ohanaja","display_name":"Christian Ohanaja","email":"cohanaja@nvidia.com","username":"cohanaja"},"change_message_id":"e06a0733be0b2bfa5165e6a3c378dbcfcee3e88e","unresolved":true,"context_lines":[{"line_number":2545,"context_line":"                error \u003d err"},{"line_number":2546,"context_line":"            try:"},{"line_number":2547,"context_line":"                self._record_sharding_progress(broker, node, error,"},{"line_number":2548,"context_line":"                                               processing_time)"},{"line_number":2549,"context_line":"            except (Exception, Timeout) as error:"},{"line_number":2550,"context_line":"                self.exception(broker, \u0027Unhandled exception while dumping \u0027"},{"line_number":2551,"context_line":"                                       \u0027progress: %s\u0027, error)"}],"source_content_type":"text/x-python","patch_set":14,"id":"fe5d751f_f639bacf","line":2548,"in_reply_to":"7d280843_843602f2","updated":"2025-10-08 21:05:29.000000000","message":"`processing_time` isn\u0027t set for the case where when mock_processing throws an error, you can CTRL+F \u0027kapow!\u0027 to see where it is","commit_id":"6bce68942cf419a058ad2b2a99fb0faf6616f500"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"d9ccef0a44e8dd065c20991b77348db915ee993e","unresolved":false,"context_lines":[{"line_number":2545,"context_line":"                error \u003d err"},{"line_number":2546,"context_line":"            try:"},{"line_number":2547,"context_line":"                self._record_sharding_progress(broker, node, error,"},{"line_number":2548,"context_line":"                                               processing_time)"},{"line_number":2549,"context_line":"            except (Exception, Timeout) as error:"},{"line_number":2550,"context_line":"                self.exception(broker, \u0027Unhandled exception while dumping \u0027"},{"line_number":2551,"context_line":"                                       \u0027progress: %s\u0027, error)"}],"source_content_type":"text/x-python","patch_set":14,"id":"11a3f71d_3d523231","line":2548,"in_reply_to":"fe5d751f_f639bacf","updated":"2025-10-10 03:28:56.000000000","message":"Acknowledged","commit_id":"6bce68942cf419a058ad2b2a99fb0faf6616f500"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"d9ccef0a44e8dd065c20991b77348db915ee993e","unresolved":false,"context_lines":[{"line_number":2545,"context_line":"                error \u003d err"},{"line_number":2546,"context_line":"            try:"},{"line_number":2547,"context_line":"                self._record_sharding_progress(broker, node, error,"},{"line_number":2548,"context_line":"                                               processing_time)"},{"line_number":2549,"context_line":"            except (Exception, Timeout) as error:"},{"line_number":2550,"context_line":"                self.exception(broker, \u0027Unhandled exception while dumping \u0027"},{"line_number":2551,"context_line":"                                       \u0027progress: %s\u0027, error)"}],"source_content_type":"text/x-python","patch_set":14,"id":"d31a8d0c_c08981fc","line":2548,"in_reply_to":"fe5d751f_f639bacf","updated":"2025-10-10 03:28:56.000000000","message":"Acknowledged","commit_id":"6bce68942cf419a058ad2b2a99fb0faf6616f500"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"d9ccef0a44e8dd065c20991b77348db915ee993e","unresolved":true,"context_lines":[{"line_number":1079,"context_line":"        info \u003d self._make_stats_info(broker, node, own_shard_range)"},{"line_number":1080,"context_line":""},{"line_number":1081,"context_line":"        if sharded_context:"},{"line_number":1082,"context_line":"            info[\"total_replicate_calls\"] \u003d sharded_context.replication_count"},{"line_number":1083,"context_line":"            info[\"total_replicate_time\"] \u003d sharded_context.replication_time"},{"line_number":1084,"context_line":""},{"line_number":1085,"context_line":"        if processing_time:"}],"source_content_type":"text/x-python","patch_set":16,"id":"a564102e_40db032f","line":1082,"updated":"2025-10-10 03:28:56.000000000","message":"I\u0027d like to see a test case which has valid value of ``total_replicate_calls`` and ``\"total_replicate_time\"``, and ``total_replicate_calls`` will increase after each replication call. something like this one: https://review.opendev.org/c/openstack/swift/+/963647/1/test/unit/container/test_sharder.py#1795","commit_id":"a67a64d0de25e6c42ae7d957da2db2f66996b4e5"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"254c9071998c517e7b9d57dfc82daec13cfbeb79","unresolved":false,"context_lines":[{"line_number":1079,"context_line":"        info \u003d self._make_stats_info(broker, node, own_shard_range)"},{"line_number":1080,"context_line":""},{"line_number":1081,"context_line":"        if sharded_context:"},{"line_number":1082,"context_line":"            info[\"total_replicate_calls\"] \u003d sharded_context.replication_count"},{"line_number":1083,"context_line":"            info[\"total_replicate_time\"] \u003d sharded_context.replication_time"},{"line_number":1084,"context_line":""},{"line_number":1085,"context_line":"        if processing_time:"}],"source_content_type":"text/x-python","patch_set":16,"id":"bcb0a118_92f309e3","line":1082,"in_reply_to":"0b761fd2_a2f840b2","updated":"2025-10-11 03:24:35.000000000","message":"No problem! per swift dev guideline, the patch author usually adds helper\u0027s name as Co-Author in the commit message if their patch got squashed. ;-) I will add that for you.","commit_id":"a67a64d0de25e6c42ae7d957da2db2f66996b4e5"},{"author":{"_account_id":38368,"name":"Christian Ohanaja","display_name":"Christian Ohanaja","email":"cohanaja@nvidia.com","username":"cohanaja"},"change_message_id":"9e429f4bf023d48b922e9de20ebcbb4576a0046d","unresolved":false,"context_lines":[{"line_number":1079,"context_line":"        info \u003d self._make_stats_info(broker, node, own_shard_range)"},{"line_number":1080,"context_line":""},{"line_number":1081,"context_line":"        if sharded_context:"},{"line_number":1082,"context_line":"            info[\"total_replicate_calls\"] \u003d sharded_context.replication_count"},{"line_number":1083,"context_line":"            info[\"total_replicate_time\"] \u003d sharded_context.replication_time"},{"line_number":1084,"context_line":""},{"line_number":1085,"context_line":"        if processing_time:"}],"source_content_type":"text/x-python","patch_set":16,"id":"0b761fd2_a2f840b2","line":1082,"in_reply_to":"a564102e_40db032f","updated":"2025-10-10 17:18:14.000000000","message":"Done, thank you for the suggested patch!!","commit_id":"a67a64d0de25e6c42ae7d957da2db2f66996b4e5"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"f0a0f2983b182ac7992b6175263ccb4c9cf3f235","unresolved":false,"context_lines":[{"line_number":1079,"context_line":"        info \u003d self._make_stats_info(broker, node, own_shard_range)"},{"line_number":1080,"context_line":""},{"line_number":1081,"context_line":"        if sharded_context:"},{"line_number":1082,"context_line":"            info[\"total_replicate_calls\"] \u003d sharded_context.replication_count"},{"line_number":1083,"context_line":"            info[\"total_replicate_time\"] \u003d sharded_context.replication_time"},{"line_number":1084,"context_line":""},{"line_number":1085,"context_line":"        if processing_time:"}],"source_content_type":"text/x-python","patch_set":21,"id":"92d117a9_fe28d47c","line":1082,"updated":"2025-10-13 23:52:41.000000000","message":"oh, we need update this doc as well: https://github.com/NVIDIA/swift/blob/master/doc/source/overview_container_sharding.rst","commit_id":"5ed86abe4c157f9146d342cea19ebcb4f9b74c8c"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"d4598392dbe0c57a5c6acf8cfe71d9904555a110","unresolved":true,"context_lines":[{"line_number":631,"context_line":"                 last_cleave_to_row\u003dNone, cleaving_done\u003dFalse,"},{"line_number":632,"context_line":"                 misplaced_done\u003dFalse, ranges_done\u003d0, ranges_todo\u003d0,"},{"line_number":633,"context_line":"                 replication_count\u003d0, replication_time\u003d0,"},{"line_number":634,"context_line":"                 first_cleave_time\u003dNone, last_cleave_time\u003dNone):"},{"line_number":635,"context_line":"        self.ref \u003d ref"},{"line_number":636,"context_line":"        self._cursor \u003d None"},{"line_number":637,"context_line":"        self.cursor \u003d cursor"}],"source_content_type":"text/x-python","patch_set":27,"id":"1a602575_e0a9cfa8","line":634,"updated":"2025-10-30 04:40:28.000000000","message":"why ``first_cleave_time`` and ``last_cleave_time`` have default value ``None`` instead of ``0``? let\u0027s use the same default value as ``replication_time``","commit_id":"021d3673378b6a1beb5c0e3e0f0415fe42280ef9"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"aac2216623d546babc6fe263ae68746a2724b641","unresolved":false,"context_lines":[{"line_number":631,"context_line":"                 last_cleave_to_row\u003dNone, cleaving_done\u003dFalse,"},{"line_number":632,"context_line":"                 misplaced_done\u003dFalse, ranges_done\u003d0, ranges_todo\u003d0,"},{"line_number":633,"context_line":"                 replication_count\u003d0, replication_time\u003d0,"},{"line_number":634,"context_line":"                 first_cleave_time\u003dNone, last_cleave_time\u003dNone):"},{"line_number":635,"context_line":"        self.ref \u003d ref"},{"line_number":636,"context_line":"        self._cursor \u003d None"},{"line_number":637,"context_line":"        self.cursor \u003d cursor"}],"source_content_type":"text/x-python","patch_set":27,"id":"e11aef37_497e79da","line":634,"in_reply_to":"1a602575_e0a9cfa8","updated":"2025-11-14 22:07:42.000000000","message":"Done","commit_id":"021d3673378b6a1beb5c0e3e0f0415fe42280ef9"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"d4598392dbe0c57a5c6acf8cfe71d9904555a110","unresolved":true,"context_lines":[{"line_number":1050,"context_line":"        if sharded_context:"},{"line_number":1051,"context_line":"            info[\"total_replicate_calls\"] \u003d sharded_context.replication_count"},{"line_number":1052,"context_line":"            info[\"total_replicate_time\"] \u003d sharded_context.replication_time"},{"line_number":1053,"context_line":"            if (sharded_context.last_cleave_time is not None and"},{"line_number":1054,"context_line":"                    sharded_context.first_cleave_time is not None):"},{"line_number":1055,"context_line":"                elapsed \u003d (sharded_context.last_cleave_time -"},{"line_number":1056,"context_line":"                           sharded_context.first_cleave_time)"}],"source_content_type":"text/x-python","patch_set":27,"id":"a694d20f_761776f0","line":1053,"updated":"2025-10-30 04:40:28.000000000","message":"this will just be\n```\n            if (sharded_context.last_cleave_time and\n                    sharded_context.first_cleave_time):\n```","commit_id":"021d3673378b6a1beb5c0e3e0f0415fe42280ef9"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"aac2216623d546babc6fe263ae68746a2724b641","unresolved":false,"context_lines":[{"line_number":1050,"context_line":"        if sharded_context:"},{"line_number":1051,"context_line":"            info[\"total_replicate_calls\"] \u003d sharded_context.replication_count"},{"line_number":1052,"context_line":"            info[\"total_replicate_time\"] \u003d sharded_context.replication_time"},{"line_number":1053,"context_line":"            if (sharded_context.last_cleave_time is not None and"},{"line_number":1054,"context_line":"                    sharded_context.first_cleave_time is not None):"},{"line_number":1055,"context_line":"                elapsed \u003d (sharded_context.last_cleave_time -"},{"line_number":1056,"context_line":"                           sharded_context.first_cleave_time)"}],"source_content_type":"text/x-python","patch_set":27,"id":"9c517db7_084d282c","line":1053,"in_reply_to":"a694d20f_761776f0","updated":"2025-11-14 22:07:42.000000000","message":"Done","commit_id":"021d3673378b6a1beb5c0e3e0f0415fe42280ef9"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"d4598392dbe0c57a5c6acf8cfe71d9904555a110","unresolved":true,"context_lines":[{"line_number":1054,"context_line":"                    sharded_context.first_cleave_time is not None):"},{"line_number":1055,"context_line":"                elapsed \u003d (sharded_context.last_cleave_time -"},{"line_number":1056,"context_line":"                           sharded_context.first_cleave_time)"},{"line_number":1057,"context_line":"                info[\u0027db_sharding_total_elapsed\u0027] \u003d elapsed"},{"line_number":1058,"context_line":""},{"line_number":1059,"context_line":"        if processing_time:"},{"line_number":1060,"context_line":"            info[\u0027processing_time\u0027] \u003d processing_time"}],"source_content_type":"text/x-python","patch_set":27,"id":"66fae0d7_2f58d378","line":1057,"updated":"2025-10-30 04:40:28.000000000","message":"this is not correct, it should be\n```\n  \u0027db_sharding_total_elapsed\u0027 \u003d time.time()when_sharding_completes - float(own_shard_range.epoch)\n```","commit_id":"021d3673378b6a1beb5c0e3e0f0415fe42280ef9"},{"author":{"_account_id":38368,"name":"Christian Ohanaja","display_name":"Christian Ohanaja","email":"cohanaja@nvidia.com","username":"cohanaja"},"change_message_id":"87200bbfb95cd6e67a62331146bc22303d3e83e1","unresolved":false,"context_lines":[{"line_number":1054,"context_line":"                    sharded_context.first_cleave_time is not None):"},{"line_number":1055,"context_line":"                elapsed \u003d (sharded_context.last_cleave_time -"},{"line_number":1056,"context_line":"                           sharded_context.first_cleave_time)"},{"line_number":1057,"context_line":"                info[\u0027db_sharding_total_elapsed\u0027] \u003d elapsed"},{"line_number":1058,"context_line":""},{"line_number":1059,"context_line":"        if processing_time:"},{"line_number":1060,"context_line":"            info[\u0027processing_time\u0027] \u003d processing_time"}],"source_content_type":"text/x-python","patch_set":27,"id":"7c5941fe_8b6799ca","line":1057,"in_reply_to":"66fae0d7_2f58d378","updated":"2025-11-22 04:41:23.000000000","message":"Done","commit_id":"021d3673378b6a1beb5c0e3e0f0415fe42280ef9"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"d4598392dbe0c57a5c6acf8cfe71d9904555a110","unresolved":true,"context_lines":[{"line_number":2246,"context_line":"                if (len(ranges_done) \u003e 0 and"},{"line_number":2247,"context_line":"                        cleaving_context.first_cleave_time is None):"},{"line_number":2248,"context_line":"                    cleaving_context.first_cleave_time \u003d time.time()"},{"line_number":2249,"context_line":"                    self.logger.timing_since("},{"line_number":2250,"context_line":"                        \u0027sharder.sharding.time_to_first_cleave\u0027,"},{"line_number":2251,"context_line":"                        cleaving_context.first_cleave_time)"},{"line_number":2252,"context_line":"            elif cleave_result \u003d\u003d CLEAVE_FAILED:"}],"source_content_type":"text/x-python","patch_set":27,"id":"06891e0a_234e96dc","line":2249,"updated":"2025-10-30 04:40:28.000000000","message":"this is not correct, ``time_to_first_cleave`` is the time spent from the time sharding is enabled to the time right before starting the first cleaving.","commit_id":"021d3673378b6a1beb5c0e3e0f0415fe42280ef9"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"aac2216623d546babc6fe263ae68746a2724b641","unresolved":false,"context_lines":[{"line_number":2246,"context_line":"                if (len(ranges_done) \u003e 0 and"},{"line_number":2247,"context_line":"                        cleaving_context.first_cleave_time is None):"},{"line_number":2248,"context_line":"                    cleaving_context.first_cleave_time \u003d time.time()"},{"line_number":2249,"context_line":"                    self.logger.timing_since("},{"line_number":2250,"context_line":"                        \u0027sharder.sharding.time_to_first_cleave\u0027,"},{"line_number":2251,"context_line":"                        cleaving_context.first_cleave_time)"},{"line_number":2252,"context_line":"            elif cleave_result \u003d\u003d CLEAVE_FAILED:"}],"source_content_type":"text/x-python","patch_set":27,"id":"0b60ad08_92cc4333","line":2249,"in_reply_to":"06891e0a_234e96dc","updated":"2025-11-14 22:07:42.000000000","message":"Done","commit_id":"021d3673378b6a1beb5c0e3e0f0415fe42280ef9"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"d4598392dbe0c57a5c6acf8cfe71d9904555a110","unresolved":true,"context_lines":[{"line_number":2267,"context_line":"        # timing_since(sharder.sharding.time_to_last_cleave, first_cleave_time)"},{"line_number":2268,"context_line":"        # ?"},{"line_number":2269,"context_line":"        if (cleaving_context.cleaving_done and"},{"line_number":2270,"context_line":"                cleaving_context.last_cleave_time is None):"},{"line_number":2271,"context_line":"            cleaving_context.last_cleave_time \u003d time.time()"},{"line_number":2272,"context_line":"            self.logger.timing_since("},{"line_number":2273,"context_line":"                \u0027sharder.sharding.time_to_last_cleave\u0027,"}],"source_content_type":"text/x-python","patch_set":27,"id":"9d670398_46642bbc","line":2270,"updated":"2025-10-30 04:40:28.000000000","message":"``last_cleave_time`` is set already at line L2133, so below code won\u0027t run at all","commit_id":"021d3673378b6a1beb5c0e3e0f0415fe42280ef9"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"aac2216623d546babc6fe263ae68746a2724b641","unresolved":false,"context_lines":[{"line_number":2267,"context_line":"        # timing_since(sharder.sharding.time_to_last_cleave, first_cleave_time)"},{"line_number":2268,"context_line":"        # ?"},{"line_number":2269,"context_line":"        if (cleaving_context.cleaving_done and"},{"line_number":2270,"context_line":"                cleaving_context.last_cleave_time is None):"},{"line_number":2271,"context_line":"            cleaving_context.last_cleave_time \u003d time.time()"},{"line_number":2272,"context_line":"            self.logger.timing_since("},{"line_number":2273,"context_line":"                \u0027sharder.sharding.time_to_last_cleave\u0027,"}],"source_content_type":"text/x-python","patch_set":27,"id":"06561c4d_fd23f5bf","line":2270,"in_reply_to":"9d670398_46642bbc","updated":"2025-11-14 22:07:42.000000000","message":"Done","commit_id":"021d3673378b6a1beb5c0e3e0f0415fe42280ef9"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"d4598392dbe0c57a5c6acf8cfe71d9904555a110","unresolved":true,"context_lines":[{"line_number":2270,"context_line":"                cleaving_context.last_cleave_time is None):"},{"line_number":2271,"context_line":"            cleaving_context.last_cleave_time \u003d time.time()"},{"line_number":2272,"context_line":"            self.logger.timing_since("},{"line_number":2273,"context_line":"                \u0027sharder.sharding.time_to_last_cleave\u0027,"},{"line_number":2274,"context_line":"                cleaving_context.last_cleave_time)"},{"line_number":2275,"context_line":""},{"line_number":2276,"context_line":"        return (cleaving_context.misplaced_done and"}],"source_content_type":"text/x-python","patch_set":27,"id":"01358153_f3dccd97","line":2273,"updated":"2025-10-30 04:40:28.000000000","message":"this calculation is not correct either, ``time_to_last_cleave`` is the time spent from the time sharding is enabled to the time last cleaving finishes.","commit_id":"021d3673378b6a1beb5c0e3e0f0415fe42280ef9"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"aac2216623d546babc6fe263ae68746a2724b641","unresolved":false,"context_lines":[{"line_number":2270,"context_line":"                cleaving_context.last_cleave_time is None):"},{"line_number":2271,"context_line":"            cleaving_context.last_cleave_time \u003d time.time()"},{"line_number":2272,"context_line":"            self.logger.timing_since("},{"line_number":2273,"context_line":"                \u0027sharder.sharding.time_to_last_cleave\u0027,"},{"line_number":2274,"context_line":"                cleaving_context.last_cleave_time)"},{"line_number":2275,"context_line":""},{"line_number":2276,"context_line":"        return (cleaving_context.misplaced_done and"}],"source_content_type":"text/x-python","patch_set":27,"id":"f03a1079_a7251849","line":2273,"in_reply_to":"01358153_f3dccd97","updated":"2025-11-14 22:07:42.000000000","message":"Done","commit_id":"021d3673378b6a1beb5c0e3e0f0415fe42280ef9"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"241bdcf6c5a939dc425b336b477f0a6bbe7c0c80","unresolved":true,"context_lines":[{"line_number":1033,"context_line":"                return"},{"line_number":1034,"context_line":""},{"line_number":1035,"context_line":"            contexts_sorted \u003d sorted(contexts, key\u003dlambda x: float(x[1]))"},{"line_number":1036,"context_line":"            sharded_context \u003d contexts_sorted[-1][0]"},{"line_number":1037,"context_line":""},{"line_number":1038,"context_line":"        update_own_shard_range_stats(broker, own_shard_range)"},{"line_number":1039,"context_line":"        info \u003d self._make_stats_info(broker, node, own_shard_range)"}],"source_content_type":"text/x-python","patch_set":35,"id":"9a56097d_26ad947e","line":1036,"updated":"2025-11-03 15:55:54.000000000","message":"this seems like a weird way to spell \"this databases context\" - I think it\u0027s maybe picking the one that was most recently updated... which \"should\" be the local db context?","commit_id":"565917f6126148c43814f4b5bd7a9615d20a14d3"},{"author":{"_account_id":38368,"name":"Christian Ohanaja","display_name":"Christian Ohanaja","email":"cohanaja@nvidia.com","username":"cohanaja"},"change_message_id":"87200bbfb95cd6e67a62331146bc22303d3e83e1","unresolved":false,"context_lines":[{"line_number":1033,"context_line":"                return"},{"line_number":1034,"context_line":""},{"line_number":1035,"context_line":"            contexts_sorted \u003d sorted(contexts, key\u003dlambda x: float(x[1]))"},{"line_number":1036,"context_line":"            sharded_context \u003d contexts_sorted[-1][0]"},{"line_number":1037,"context_line":""},{"line_number":1038,"context_line":"        update_own_shard_range_stats(broker, own_shard_range)"},{"line_number":1039,"context_line":"        info \u003d self._make_stats_info(broker, node, own_shard_range)"}],"source_content_type":"text/x-python","patch_set":35,"id":"3c1c18ea_c9331512","line":1036,"in_reply_to":"9a56097d_26ad947e","updated":"2025-11-22 04:41:23.000000000","message":"Done","commit_id":"565917f6126148c43814f4b5bd7a9615d20a14d3"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"c0d9e8dbf61056d8fc9057c2cf2fd4c0928dcde8","unresolved":true,"context_lines":[{"line_number":1041,"context_line":"        if sharded_ctx:"},{"line_number":1042,"context_line":"            info[\"total_replicate_calls\"] \u003d sharded_ctx[0].replication_count"},{"line_number":1043,"context_line":"            info[\"total_replicate_time\"] \u003d sharded_ctx[0].replication_time"},{"line_number":1044,"context_line":"            sharding_total_elapsed \u003d (float(sharded_ctx[1])"},{"line_number":1045,"context_line":"                                      - float(own_shard_range.epoch))"},{"line_number":1046,"context_line":"        else:"},{"line_number":1047,"context_line":"            sharding_total_elapsed \u003d time.time() - float(own_shard_range.epoch)"}],"source_content_type":"text/x-python","patch_set":37,"id":"d031e219_64b115f4","line":1044,"updated":"2025-11-21 16:55:44.000000000","message":"The timestamp represents when the last cleaving context was saved, which should be very close to actual sharding completion time, there may be a small time gap between when the last context was stored and when the database state actually transitioned to SHARDED. \n\nWe have an existing approach to calculate this: ``Timestamp_when_sharding_complete - own_shard_range.epoch``, but I think the new approach is a reasonable approximation for tracking the total sharding duration after completion, after we fix the the bug in context loading: https://review.opendev.org/c/openstack/swift/+/966139/11","commit_id":"886ac32c0e21a226ac6e0b7757540fadc2659a90"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"40489ec4ee018f034423987459f7e71b79817269","unresolved":false,"context_lines":[{"line_number":1041,"context_line":"        if sharded_ctx:"},{"line_number":1042,"context_line":"            info[\"total_replicate_calls\"] \u003d sharded_ctx[0].replication_count"},{"line_number":1043,"context_line":"            info[\"total_replicate_time\"] \u003d sharded_ctx[0].replication_time"},{"line_number":1044,"context_line":"            sharding_total_elapsed \u003d (float(sharded_ctx[1])"},{"line_number":1045,"context_line":"                                      - float(own_shard_range.epoch))"},{"line_number":1046,"context_line":"        else:"},{"line_number":1047,"context_line":"            sharding_total_elapsed \u003d time.time() - float(own_shard_range.epoch)"}],"source_content_type":"text/x-python","patch_set":37,"id":"06337cba_9fc3c636","line":1044,"in_reply_to":"65f4577a_fa592ea3","updated":"2025-11-26 03:20:03.000000000","message":"Done","commit_id":"886ac32c0e21a226ac6e0b7757540fadc2659a90"},{"author":{"_account_id":38368,"name":"Christian Ohanaja","display_name":"Christian Ohanaja","email":"cohanaja@nvidia.com","username":"cohanaja"},"change_message_id":"87200bbfb95cd6e67a62331146bc22303d3e83e1","unresolved":true,"context_lines":[{"line_number":1041,"context_line":"        if sharded_ctx:"},{"line_number":1042,"context_line":"            info[\"total_replicate_calls\"] \u003d sharded_ctx[0].replication_count"},{"line_number":1043,"context_line":"            info[\"total_replicate_time\"] \u003d sharded_ctx[0].replication_time"},{"line_number":1044,"context_line":"            sharding_total_elapsed \u003d (float(sharded_ctx[1])"},{"line_number":1045,"context_line":"                                      - float(own_shard_range.epoch))"},{"line_number":1046,"context_line":"        else:"},{"line_number":1047,"context_line":"            sharding_total_elapsed \u003d time.time() - float(own_shard_range.epoch)"}],"source_content_type":"text/x-python","patch_set":37,"id":"65f4577a_fa592ea3","line":1044,"in_reply_to":"d031e219_64b115f4","updated":"2025-11-22 04:41:23.000000000","message":"Acknowledged + the associated context retrieval patch is also updated with the same timing logic","commit_id":"886ac32c0e21a226ac6e0b7757540fadc2659a90"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"c0d9e8dbf61056d8fc9057c2cf2fd4c0928dcde8","unresolved":true,"context_lines":[{"line_number":1044,"context_line":"            sharding_total_elapsed \u003d (float(sharded_ctx[1])"},{"line_number":1045,"context_line":"                                      - float(own_shard_range.epoch))"},{"line_number":1046,"context_line":"        else:"},{"line_number":1047,"context_line":"            sharding_total_elapsed \u003d time.time() - float(own_shard_range.epoch)"},{"line_number":1048,"context_line":""},{"line_number":1049,"context_line":"        if processing_time:"},{"line_number":1050,"context_line":"            info[\u0027processing_time\u0027] \u003d processing_time"}],"source_content_type":"text/x-python","patch_set":37,"id":"a032ab33_6e3004c8","line":1047,"updated":"2025-11-21 16:55:44.000000000","message":"``sharding_total_elapsed`` could mean ``total time spent on sharding so far before completion`` or ``total time spent on sharding from beginning to end``, but I think only the latter is useful, the in progress timestamps could confuse us when aggregating those recon stats across nodes. probably we should come up with a better name for ``sharding_total_elapsed``, like ``total_sharding_time``?","commit_id":"886ac32c0e21a226ac6e0b7757540fadc2659a90"},{"author":{"_account_id":38368,"name":"Christian Ohanaja","display_name":"Christian Ohanaja","email":"cohanaja@nvidia.com","username":"cohanaja"},"change_message_id":"87200bbfb95cd6e67a62331146bc22303d3e83e1","unresolved":false,"context_lines":[{"line_number":1044,"context_line":"            sharding_total_elapsed \u003d (float(sharded_ctx[1])"},{"line_number":1045,"context_line":"                                      - float(own_shard_range.epoch))"},{"line_number":1046,"context_line":"        else:"},{"line_number":1047,"context_line":"            sharding_total_elapsed \u003d time.time() - float(own_shard_range.epoch)"},{"line_number":1048,"context_line":""},{"line_number":1049,"context_line":"        if processing_time:"},{"line_number":1050,"context_line":"            info[\u0027processing_time\u0027] \u003d processing_time"}],"source_content_type":"text/x-python","patch_set":37,"id":"7595df29_6576288b","line":1047,"in_reply_to":"a032ab33_6e3004c8","updated":"2025-11-22 04:41:23.000000000","message":"Good call + I can see the issue when it comes to stats aggregation (currently it\u0027s always increasing so how do you even graph this realistically?) so I\u0027ve taken out the running elapsed time and it\u0027ll just emit for sharded containers similar to rep_time.","commit_id":"886ac32c0e21a226ac6e0b7757540fadc2659a90"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"6b462658f3f90dadb2ce61ad92b7754396d2c3a3","unresolved":true,"context_lines":[{"line_number":2089,"context_line":"            success, responses \u003d self._replicate_object("},{"line_number":2090,"context_line":"                shard_part, shard_broker.db_file, node_id)"},{"line_number":2091,"context_line":"            cleaving_context.replication_time +\u003d time.time() - replicate_start"},{"line_number":2092,"context_line":"            cleaving_context.replication_count +\u003d 1"},{"line_number":2093,"context_line":""},{"line_number":2094,"context_line":"            replication_successes \u003d responses.count(True)"},{"line_number":2095,"context_line":"            if (not success and (not responses or"}],"source_content_type":"text/x-python","patch_set":37,"id":"4293298d_450a266d","line":2092,"updated":"2025-11-21 18:20:20.000000000","message":"the current implementation of ``replication_count`` tracks number of cleaved shards, but its actual definition is ``Number of replication operations during cleaving after it completes. ``, for usync, it should be much higher than number of cleaved shards because of ``per_diff``.","commit_id":"886ac32c0e21a226ac6e0b7757540fadc2659a90"},{"author":{"_account_id":38368,"name":"Christian Ohanaja","display_name":"Christian Ohanaja","email":"cohanaja@nvidia.com","username":"cohanaja"},"change_message_id":"87200bbfb95cd6e67a62331146bc22303d3e83e1","unresolved":true,"context_lines":[{"line_number":2089,"context_line":"            success, responses \u003d self._replicate_object("},{"line_number":2090,"context_line":"                shard_part, shard_broker.db_file, node_id)"},{"line_number":2091,"context_line":"            cleaving_context.replication_time +\u003d time.time() - replicate_start"},{"line_number":2092,"context_line":"            cleaving_context.replication_count +\u003d 1"},{"line_number":2093,"context_line":""},{"line_number":2094,"context_line":"            replication_successes \u003d responses.count(True)"},{"line_number":2095,"context_line":"            if (not success and (not responses or"}],"source_content_type":"text/x-python","patch_set":37,"id":"ce2be628_11361d45","line":2092,"in_reply_to":"4293298d_450a266d","updated":"2025-11-22 04:41:23.000000000","message":"As agreed, removed the rep_count parts due to redundancy","commit_id":"886ac32c0e21a226ac6e0b7757540fadc2659a90"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"40489ec4ee018f034423987459f7e71b79817269","unresolved":false,"context_lines":[{"line_number":2089,"context_line":"            success, responses \u003d self._replicate_object("},{"line_number":2090,"context_line":"                shard_part, shard_broker.db_file, node_id)"},{"line_number":2091,"context_line":"            cleaving_context.replication_time +\u003d time.time() - replicate_start"},{"line_number":2092,"context_line":"            cleaving_context.replication_count +\u003d 1"},{"line_number":2093,"context_line":""},{"line_number":2094,"context_line":"            replication_successes \u003d responses.count(True)"},{"line_number":2095,"context_line":"            if (not success and (not responses or"}],"source_content_type":"text/x-python","patch_set":37,"id":"f7655373_736ba592","line":2092,"in_reply_to":"ce2be628_11361d45","updated":"2025-11-26 03:20:03.000000000","message":"Done","commit_id":"886ac32c0e21a226ac6e0b7757540fadc2659a90"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"40489ec4ee018f034423987459f7e71b79817269","unresolved":true,"context_lines":[{"line_number":1038,"context_line":"            info[\"total_replicate_time\"] \u003d sharded_ctx[0].replication_time"},{"line_number":1039,"context_line":"            sharding_total_elapsed \u003d (float(sharded_ctx[1])"},{"line_number":1040,"context_line":"                                      - float(own_shard_range.epoch))"},{"line_number":1041,"context_line":"            info[\u0027total_sharding_time\u0027] \u003d sharding_total_elapsed"},{"line_number":1042,"context_line":""},{"line_number":1043,"context_line":"        if processing_time:"},{"line_number":1044,"context_line":"            info[\u0027processing_time\u0027] \u003d processing_time"}],"source_content_type":"text/x-python","patch_set":41,"id":"e9f5b06a_0f02f2f5","line":1041,"updated":"2025-11-26 03:20:03.000000000","message":"for both ``total_replicate_time`` and ``total_sharding_time``, sharder will keep emitting them for each cycle after sharding completes and before ``recon_sharded_timeout``.","commit_id":"a596fb4ba103b93992553b5d8805773af6c6a3c6"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"032f681be93b8ec03a408edf8bd6378ff7196da8","unresolved":false,"context_lines":[{"line_number":1038,"context_line":"            info[\"total_replicate_time\"] \u003d sharded_ctx[0].replication_time"},{"line_number":1039,"context_line":"            sharding_total_elapsed \u003d (float(sharded_ctx[1])"},{"line_number":1040,"context_line":"                                      - float(own_shard_range.epoch))"},{"line_number":1041,"context_line":"            info[\u0027total_sharding_time\u0027] \u003d sharding_total_elapsed"},{"line_number":1042,"context_line":""},{"line_number":1043,"context_line":"        if processing_time:"},{"line_number":1044,"context_line":"            info[\u0027processing_time\u0027] \u003d processing_time"}],"source_content_type":"text/x-python","patch_set":41,"id":"7111f931_908d8f57","line":1041,"in_reply_to":"e9f5b06a_0f02f2f5","updated":"2025-11-27 00:14:56.000000000","message":"Acknowledged","commit_id":"a596fb4ba103b93992553b5d8805773af6c6a3c6"}],"test/probe/test_sharder.py":[{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"d9ccef0a44e8dd065c20991b77348db915ee993e","unresolved":true,"context_lines":[{"line_number":3471,"context_line":"                self.assertEqual(shard_ranges_1, shard_ranges)"},{"line_number":3472,"context_line":"                shard_ranges \u003d broker.get_shard_ranges(include_deleted\u003dTrue)"},{"line_number":3473,"context_line":"                self.assertLengthEqual(shard_ranges, len(exp_shard_ranges))"},{"line_number":3474,"context_line":"                self.maxDiff \u003d None"},{"line_number":3475,"context_line":"                self.assertEqual(exp_shard_ranges, shard_ranges)"},{"line_number":3476,"context_line":"                self.assertEqual(ShardRange.SHARDED,"},{"line_number":3477,"context_line":"                                 broker.get_own_shard_range().state)"}],"source_content_type":"text/x-python","patch_set":16,"id":"addba469_ef55a21a","side":"PARENT","line":3474,"updated":"2025-10-10 03:28:56.000000000","message":"was this an accidental change?","commit_id":"b035ed1385ee729411adbd459bb03033e90bb13a"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"254c9071998c517e7b9d57dfc82daec13cfbeb79","unresolved":true,"context_lines":[{"line_number":3471,"context_line":"                self.assertEqual(shard_ranges_1, shard_ranges)"},{"line_number":3472,"context_line":"                shard_ranges \u003d broker.get_shard_ranges(include_deleted\u003dTrue)"},{"line_number":3473,"context_line":"                self.assertLengthEqual(shard_ranges, len(exp_shard_ranges))"},{"line_number":3474,"context_line":"                self.maxDiff \u003d None"},{"line_number":3475,"context_line":"                self.assertEqual(exp_shard_ranges, shard_ranges)"},{"line_number":3476,"context_line":"                self.assertEqual(ShardRange.SHARDED,"},{"line_number":3477,"context_line":"                                 broker.get_own_shard_range().state)"}],"source_content_type":"text/x-python","patch_set":16,"id":"69d09ec4_36d78291","side":"PARENT","line":3474,"in_reply_to":"3b4152a0_3ad2aecb","updated":"2025-10-11 03:24:35.000000000","message":"no, this line was there before your changes.","commit_id":"b035ed1385ee729411adbd459bb03033e90bb13a"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"e3191f8888ad8f7e9c6f733d0baec105faa0dde1","unresolved":false,"context_lines":[{"line_number":3471,"context_line":"                self.assertEqual(shard_ranges_1, shard_ranges)"},{"line_number":3472,"context_line":"                shard_ranges \u003d broker.get_shard_ranges(include_deleted\u003dTrue)"},{"line_number":3473,"context_line":"                self.assertLengthEqual(shard_ranges, len(exp_shard_ranges))"},{"line_number":3474,"context_line":"                self.maxDiff \u003d None"},{"line_number":3475,"context_line":"                self.assertEqual(exp_shard_ranges, shard_ranges)"},{"line_number":3476,"context_line":"                self.assertEqual(ShardRange.SHARDED,"},{"line_number":3477,"context_line":"                                 broker.get_own_shard_range().state)"}],"source_content_type":"text/x-python","patch_set":16,"id":"f3cece81_6ddf1716","side":"PARENT","line":3474,"in_reply_to":"69d09ec4_36d78291","updated":"2025-10-11 03:27:47.000000000","message":"Done","commit_id":"b035ed1385ee729411adbd459bb03033e90bb13a"},{"author":{"_account_id":38368,"name":"Christian Ohanaja","display_name":"Christian Ohanaja","email":"cohanaja@nvidia.com","username":"cohanaja"},"change_message_id":"9e429f4bf023d48b922e9de20ebcbb4576a0046d","unresolved":true,"context_lines":[{"line_number":3471,"context_line":"                self.assertEqual(shard_ranges_1, shard_ranges)"},{"line_number":3472,"context_line":"                shard_ranges \u003d broker.get_shard_ranges(include_deleted\u003dTrue)"},{"line_number":3473,"context_line":"                self.assertLengthEqual(shard_ranges, len(exp_shard_ranges))"},{"line_number":3474,"context_line":"                self.maxDiff \u003d None"},{"line_number":3475,"context_line":"                self.assertEqual(exp_shard_ranges, shard_ranges)"},{"line_number":3476,"context_line":"                self.assertEqual(ShardRange.SHARDED,"},{"line_number":3477,"context_line":"                                 broker.get_own_shard_range().state)"}],"source_content_type":"text/x-python","patch_set":16,"id":"3b4152a0_3ad2aecb","side":"PARENT","line":3474,"in_reply_to":"addba469_ef55a21a","updated":"2025-10-10 17:18:14.000000000","message":"Oh no that was on purpose, I was removing some self.maxDiffs placed by me and felt like that shouldn\u0027tve been there","commit_id":"b035ed1385ee729411adbd459bb03033e90bb13a"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"d9ccef0a44e8dd065c20991b77348db915ee993e","unresolved":true,"context_lines":[{"line_number":2288,"context_line":"                                         \u0027meta_timestamp\u0027: mock.ANY,"},{"line_number":2289,"context_line":"                                         \u0027node_index\u0027: 0,"},{"line_number":2290,"context_line":"                                         \u0027object_count\u0027: len(obj_names),"},{"line_number":2291,"context_line":"                                         \u0027tombstones\u0027: -1,"},{"line_number":2292,"context_line":"                                         \u0027container\u0027: mock.ANY,"},{"line_number":2293,"context_line":"                                         \u0027path\u0027: mock.ANY,"},{"line_number":2294,"context_line":"                                         \u0027root\u0027: mock.ANY,"}],"source_content_type":"text/x-python","patch_set":16,"id":"420531a5_54048689","line":2291,"updated":"2025-10-10 03:28:56.000000000","message":"okay, ``-1`` is the default value which means unknown","commit_id":"a67a64d0de25e6c42ae7d957da2db2f66996b4e5"},{"author":{"_account_id":38368,"name":"Christian Ohanaja","display_name":"Christian Ohanaja","email":"cohanaja@nvidia.com","username":"cohanaja"},"change_message_id":"9e429f4bf023d48b922e9de20ebcbb4576a0046d","unresolved":false,"context_lines":[{"line_number":2288,"context_line":"                                         \u0027meta_timestamp\u0027: mock.ANY,"},{"line_number":2289,"context_line":"                                         \u0027node_index\u0027: 0,"},{"line_number":2290,"context_line":"                                         \u0027object_count\u0027: len(obj_names),"},{"line_number":2291,"context_line":"                                         \u0027tombstones\u0027: -1,"},{"line_number":2292,"context_line":"                                         \u0027container\u0027: mock.ANY,"},{"line_number":2293,"context_line":"                                         \u0027path\u0027: mock.ANY,"},{"line_number":2294,"context_line":"                                         \u0027root\u0027: mock.ANY,"}],"source_content_type":"text/x-python","patch_set":16,"id":"0b6682e1_e72ad620","line":2291,"in_reply_to":"420531a5_54048689","updated":"2025-10-10 17:18:14.000000000","message":"Acknowledged","commit_id":"a67a64d0de25e6c42ae7d957da2db2f66996b4e5"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"c99e25185d780b7ebef8a12d01ddc20c68dbdadd","unresolved":true,"context_lines":[{"line_number":3471,"context_line":"                self.assertEqual(shard_ranges_1, shard_ranges)"},{"line_number":3472,"context_line":"                shard_ranges \u003d broker.get_shard_ranges(include_deleted\u003dTrue)"},{"line_number":3473,"context_line":"                self.assertLengthEqual(shard_ranges, len(exp_shard_ranges))"},{"line_number":3474,"context_line":"                self.maxDiff \u003d None"},{"line_number":3475,"context_line":"                self.assertEqual(exp_shard_ranges, shard_ranges)"},{"line_number":3476,"context_line":"                self.assertEqual(ShardRange.SHARDED,"},{"line_number":3477,"context_line":"                                 broker.get_own_shard_range().state)"}],"source_content_type":"text/x-python","patch_set":19,"id":"d66945ca_745ebf3b","side":"PARENT","line":3474,"updated":"2025-10-13 18:28:16.000000000","message":"I think this change was probably made by your editor? let\u0027s restore it back.","commit_id":"b035ed1385ee729411adbd459bb03033e90bb13a"},{"author":{"_account_id":38368,"name":"Christian Ohanaja","display_name":"Christian Ohanaja","email":"cohanaja@nvidia.com","username":"cohanaja"},"change_message_id":"8c9f2fa628f4c2aee58e7747da986b1221f27165","unresolved":false,"context_lines":[{"line_number":3471,"context_line":"                self.assertEqual(shard_ranges_1, shard_ranges)"},{"line_number":3472,"context_line":"                shard_ranges \u003d broker.get_shard_ranges(include_deleted\u003dTrue)"},{"line_number":3473,"context_line":"                self.assertLengthEqual(shard_ranges, len(exp_shard_ranges))"},{"line_number":3474,"context_line":"                self.maxDiff \u003d None"},{"line_number":3475,"context_line":"                self.assertEqual(exp_shard_ranges, shard_ranges)"},{"line_number":3476,"context_line":"                self.assertEqual(ShardRange.SHARDED,"},{"line_number":3477,"context_line":"                                 broker.get_own_shard_range().state)"}],"source_content_type":"text/x-python","patch_set":19,"id":"4baa9252_650d6e14","side":"PARENT","line":3474,"in_reply_to":"d66945ca_745ebf3b","updated":"2025-10-13 20:16:20.000000000","message":"Done","commit_id":"b035ed1385ee729411adbd459bb03033e90bb13a"}],"test/unit/cli/test_manage_shard_ranges.py":[{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"73b9c57f48b9f23cefcdcc299a84e284cd6cb43b","unresolved":true,"context_lines":[{"line_number":160,"context_line":"                             func\u003dmock.ANY,"},{"line_number":161,"context_line":"                             rows_per_shard\u003d500000,"},{"line_number":162,"context_line":"                             subcommand\u003d\u0027find\u0027,"},{"line_number":163,"context_line":"                             skip_commits\u003dFalse,"},{"line_number":164,"context_line":"                             verbose\u003d0,"},{"line_number":165,"context_line":"                             minimum_shard_size\u003d100000)"},{"line_number":166,"context_line":"        mocked.assert_called_once_with(mock.ANY, expected)"}],"source_content_type":"text/x-python","patch_set":12,"id":"8d65c7c1_7ade28cc","line":163,"updated":"2025-10-08 16:36:00.000000000","message":"this change is on master already (all same changes below), I think there is a rebase issue on your local branch.","commit_id":"32369bedf2eb93e2fbb57c067d4a4cf06c143207"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"dcd0a0f54690c81f63b7c5d0b590ee96d7532b94","unresolved":false,"context_lines":[{"line_number":160,"context_line":"                             func\u003dmock.ANY,"},{"line_number":161,"context_line":"                             rows_per_shard\u003d500000,"},{"line_number":162,"context_line":"                             subcommand\u003d\u0027find\u0027,"},{"line_number":163,"context_line":"                             skip_commits\u003dFalse,"},{"line_number":164,"context_line":"                             verbose\u003d0,"},{"line_number":165,"context_line":"                             minimum_shard_size\u003d100000)"},{"line_number":166,"context_line":"        mocked.assert_called_once_with(mock.ANY, expected)"}],"source_content_type":"text/x-python","patch_set":12,"id":"d3cd9633_3ea3b663","line":163,"in_reply_to":"257cbd97_10b6581d","updated":"2025-10-08 20:05:12.000000000","message":"Done","commit_id":"32369bedf2eb93e2fbb57c067d4a4cf06c143207"},{"author":{"_account_id":38368,"name":"Christian Ohanaja","display_name":"Christian Ohanaja","email":"cohanaja@nvidia.com","username":"cohanaja"},"change_message_id":"0e65b21e6c708a08cdbeccceb5cc2645849019e4","unresolved":true,"context_lines":[{"line_number":160,"context_line":"                             func\u003dmock.ANY,"},{"line_number":161,"context_line":"                             rows_per_shard\u003d500000,"},{"line_number":162,"context_line":"                             subcommand\u003d\u0027find\u0027,"},{"line_number":163,"context_line":"                             skip_commits\u003dFalse,"},{"line_number":164,"context_line":"                             verbose\u003d0,"},{"line_number":165,"context_line":"                             minimum_shard_size\u003d100000)"},{"line_number":166,"context_line":"        mocked.assert_called_once_with(mock.ANY, expected)"}],"source_content_type":"text/x-python","patch_set":12,"id":"257cbd97_10b6581d","line":163,"in_reply_to":"8d65c7c1_7ade28cc","updated":"2025-10-08 18:21:01.000000000","message":"Thanks for noticing that! it was a remnant from me copying changes from vsaio","commit_id":"32369bedf2eb93e2fbb57c067d4a4cf06c143207"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"73b9c57f48b9f23cefcdcc299a84e284cd6cb43b","unresolved":true,"context_lines":[{"line_number":2931,"context_line":"            \"argument --yes/-y: not allowed with argument --dry-run/-n\","},{"line_number":2932,"context_line":"            err_lines[-2], err_lines)"},{"line_number":2933,"context_line":""},{"line_number":2934,"context_line":"    def test_skip_commits_and_force_commits_args(self):"},{"line_number":2935,"context_line":"        broker \u003d self._make_broker()"},{"line_number":2936,"context_line":""},{"line_number":2937,"context_line":"        with mock.patch(\u0027swift.cli.manage_shard_ranges.find_ranges\u0027,"}],"source_content_type":"text/x-python","patch_set":12,"id":"62fbab79_f43bfd27","line":2934,"updated":"2025-10-08 16:36:00.000000000","message":"ditto","commit_id":"32369bedf2eb93e2fbb57c067d4a4cf06c143207"},{"author":{"_account_id":38368,"name":"Christian Ohanaja","display_name":"Christian Ohanaja","email":"cohanaja@nvidia.com","username":"cohanaja"},"change_message_id":"0e65b21e6c708a08cdbeccceb5cc2645849019e4","unresolved":false,"context_lines":[{"line_number":2931,"context_line":"            \"argument --yes/-y: not allowed with argument --dry-run/-n\","},{"line_number":2932,"context_line":"            err_lines[-2], err_lines)"},{"line_number":2933,"context_line":""},{"line_number":2934,"context_line":"    def test_skip_commits_and_force_commits_args(self):"},{"line_number":2935,"context_line":"        broker \u003d self._make_broker()"},{"line_number":2936,"context_line":""},{"line_number":2937,"context_line":"        with mock.patch(\u0027swift.cli.manage_shard_ranges.find_ranges\u0027,"}],"source_content_type":"text/x-python","patch_set":12,"id":"9d1ab336_9e29048a","line":2934,"in_reply_to":"62fbab79_f43bfd27","updated":"2025-10-08 18:21:01.000000000","message":"Acknowledged","commit_id":"32369bedf2eb93e2fbb57c067d4a4cf06c143207"}],"test/unit/container/test_sharder.py":[{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"d9ccef0a44e8dd065c20991b77348db915ee993e","unresolved":true,"context_lines":[{"line_number":649,"context_line":"                        fake_time.return_value +\u003d next(fake_periods_iter)"},{"line_number":650,"context_line":"                    except StopIteration:"},{"line_number":651,"context_line":"                        # bail out"},{"line_number":652,"context_line":"                        mock_sleep.side_effect \u003d Exception(\u0027Test over\u0027)"},{"line_number":653,"context_line":"                    sharder.stats[\u0027sharding\u0027].update(fake_stats)"},{"line_number":654,"context_line":""},{"line_number":655,"context_line":"                with mock.patch("}],"source_content_type":"text/x-python","patch_set":16,"id":"4c1f0145_bb94b1d1","line":652,"updated":"2025-10-10 03:28:56.000000000","message":"this test case failed because this patch added a few ``time.time()`` calls around _process_broker() to calculate processing time and others, but the original test\u0027s mock setup causes an exception to be raised immediately when the fake periods run out. This happens in the middle of processing, causing error logs and incomplete cycle logs.\n\nI guess a simple way to fix this is to add more time periods into ``fake_periods``? but that\u0027ll be a lot of time periods, which make this test more brittle if later someone decides to add more ``time.time()``. we probably need a new way to mock.","commit_id":"a67a64d0de25e6c42ae7d957da2db2f66996b4e5"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"254c9071998c517e7b9d57dfc82daec13cfbeb79","unresolved":false,"context_lines":[{"line_number":649,"context_line":"                        fake_time.return_value +\u003d next(fake_periods_iter)"},{"line_number":650,"context_line":"                    except StopIteration:"},{"line_number":651,"context_line":"                        # bail out"},{"line_number":652,"context_line":"                        mock_sleep.side_effect \u003d Exception(\u0027Test over\u0027)"},{"line_number":653,"context_line":"                    sharder.stats[\u0027sharding\u0027].update(fake_stats)"},{"line_number":654,"context_line":""},{"line_number":655,"context_line":"                with mock.patch("}],"source_content_type":"text/x-python","patch_set":16,"id":"bb4d5432_26b295b2","line":652,"in_reply_to":"4bdc6de6_625906ef","updated":"2025-10-11 03:24:35.000000000","message":"Done","commit_id":"a67a64d0de25e6c42ae7d957da2db2f66996b4e5"},{"author":{"_account_id":38368,"name":"Christian Ohanaja","display_name":"Christian Ohanaja","email":"cohanaja@nvidia.com","username":"cohanaja"},"change_message_id":"9e429f4bf023d48b922e9de20ebcbb4576a0046d","unresolved":true,"context_lines":[{"line_number":649,"context_line":"                        fake_time.return_value +\u003d next(fake_periods_iter)"},{"line_number":650,"context_line":"                    except StopIteration:"},{"line_number":651,"context_line":"                        # bail out"},{"line_number":652,"context_line":"                        mock_sleep.side_effect \u003d Exception(\u0027Test over\u0027)"},{"line_number":653,"context_line":"                    sharder.stats[\u0027sharding\u0027].update(fake_stats)"},{"line_number":654,"context_line":""},{"line_number":655,"context_line":"                with mock.patch("}],"source_content_type":"text/x-python","patch_set":16,"id":"4bdc6de6_625906ef","line":652,"in_reply_to":"4c1f0145_bb94b1d1","updated":"2025-10-10 17:18:14.000000000","message":"I\u0027ve refactored the test a bit to rely on a hard \"number of times to run process_broker\" instead of the time tracking thing","commit_id":"a67a64d0de25e6c42ae7d957da2db2f66996b4e5"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"d9ccef0a44e8dd065c20991b77348db915ee993e","unresolved":true,"context_lines":[{"line_number":8448,"context_line":"        broker \u003d self._make_old_style_sharding_broker()"},{"line_number":8449,"context_line":"        old_db_id \u003d broker.get_brokers()[0].get_info()[\u0027id\u0027]"},{"line_number":8450,"context_line":"        last_mod \u003d Timestamp.now()"},{"line_number":8451,"context_line":"        ctx \u003d CleavingContext(old_db_id, \u0027curs\u0027, 12, 11, 2, True, True, 2, 4,"},{"line_number":8452,"context_line":"                              2, 0.5)"},{"line_number":8453,"context_line":"        with mock_timestamp_now(last_mod):"},{"line_number":8454,"context_line":"            ctx.store(broker)"}],"source_content_type":"text/x-python","patch_set":16,"id":"df582351_8b7e6e11","line":8451,"updated":"2025-10-10 03:28:56.000000000","message":"indentation looks better to be:\n```\n        ctx \u003d CleavingContext(\n            old_db_id, \u0027curs\u0027, 12, 11, 2, True, True, 2, 4, 2, 0.5)\n```","commit_id":"a67a64d0de25e6c42ae7d957da2db2f66996b4e5"},{"author":{"_account_id":38368,"name":"Christian Ohanaja","display_name":"Christian Ohanaja","email":"cohanaja@nvidia.com","username":"cohanaja"},"change_message_id":"9e429f4bf023d48b922e9de20ebcbb4576a0046d","unresolved":false,"context_lines":[{"line_number":8448,"context_line":"        broker \u003d self._make_old_style_sharding_broker()"},{"line_number":8449,"context_line":"        old_db_id \u003d broker.get_brokers()[0].get_info()[\u0027id\u0027]"},{"line_number":8450,"context_line":"        last_mod \u003d Timestamp.now()"},{"line_number":8451,"context_line":"        ctx \u003d CleavingContext(old_db_id, \u0027curs\u0027, 12, 11, 2, True, True, 2, 4,"},{"line_number":8452,"context_line":"                              2, 0.5)"},{"line_number":8453,"context_line":"        with mock_timestamp_now(last_mod):"},{"line_number":8454,"context_line":"            ctx.store(broker)"}],"source_content_type":"text/x-python","patch_set":16,"id":"bb0413d3_1c01ee8e","line":8451,"in_reply_to":"df582351_8b7e6e11","updated":"2025-10-10 17:18:14.000000000","message":"Done","commit_id":"a67a64d0de25e6c42ae7d957da2db2f66996b4e5"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"d9ccef0a44e8dd065c20991b77348db915ee993e","unresolved":true,"context_lines":[{"line_number":8681,"context_line":"            self.assertFalse(ctx.misplaced_done)"},{"line_number":8682,"context_line":"            self.assertFalse(ctx.cleaving_done)"},{"line_number":8683,"context_line":"            self.assertEqual(0, ctx.ranges_done)"},{"line_number":8684,"context_line":"            self.assertEqual(0, ctx.ranges_todo)"},{"line_number":8685,"context_line":"        ctx.reset()"},{"line_number":8686,"context_line":"        check_context()"},{"line_number":8687,"context_line":"        # check idempotency"}],"source_content_type":"text/x-python","patch_set":16,"id":"39f9074b_0fbf1f6c","line":8684,"updated":"2025-10-10 03:28:56.000000000","message":"add\n```\n   self.assertEqual(0, ctx.replication_count)\n   self.assertEqual(0, ctx.replication_time)\n```","commit_id":"a67a64d0de25e6c42ae7d957da2db2f66996b4e5"},{"author":{"_account_id":38368,"name":"Christian Ohanaja","display_name":"Christian Ohanaja","email":"cohanaja@nvidia.com","username":"cohanaja"},"change_message_id":"9e429f4bf023d48b922e9de20ebcbb4576a0046d","unresolved":false,"context_lines":[{"line_number":8681,"context_line":"            self.assertFalse(ctx.misplaced_done)"},{"line_number":8682,"context_line":"            self.assertFalse(ctx.cleaving_done)"},{"line_number":8683,"context_line":"            self.assertEqual(0, ctx.ranges_done)"},{"line_number":8684,"context_line":"            self.assertEqual(0, ctx.ranges_todo)"},{"line_number":8685,"context_line":"        ctx.reset()"},{"line_number":8686,"context_line":"        check_context()"},{"line_number":8687,"context_line":"        # check idempotency"}],"source_content_type":"text/x-python","patch_set":16,"id":"510c632c_9565f06d","line":8684,"in_reply_to":"39f9074b_0fbf1f6c","updated":"2025-10-10 17:18:14.000000000","message":"Done","commit_id":"a67a64d0de25e6c42ae7d957da2db2f66996b4e5"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"d9ccef0a44e8dd065c20991b77348db915ee993e","unresolved":true,"context_lines":[{"line_number":8700,"context_line":"            self.assertTrue(ctx.misplaced_done)  # *not* reset here"},{"line_number":8701,"context_line":"            self.assertFalse(ctx.cleaving_done)"},{"line_number":8702,"context_line":"            self.assertEqual(0, ctx.ranges_done)"},{"line_number":8703,"context_line":"            self.assertEqual(0, ctx.ranges_todo)"},{"line_number":8704,"context_line":"        ctx.start()"},{"line_number":8705,"context_line":"        check_context()"},{"line_number":8706,"context_line":"        # check idempotency"}],"source_content_type":"text/x-python","patch_set":16,"id":"f39aaa91_b8c2b5d2","line":8703,"updated":"2025-10-10 03:28:56.000000000","message":"add\n```\n        self.assertEqual(0, ctx.replication_count)\n        self.assertEqual(0, ctx.replication_time)\n```","commit_id":"a67a64d0de25e6c42ae7d957da2db2f66996b4e5"},{"author":{"_account_id":38368,"name":"Christian Ohanaja","display_name":"Christian Ohanaja","email":"cohanaja@nvidia.com","username":"cohanaja"},"change_message_id":"9e429f4bf023d48b922e9de20ebcbb4576a0046d","unresolved":false,"context_lines":[{"line_number":8700,"context_line":"            self.assertTrue(ctx.misplaced_done)  # *not* reset here"},{"line_number":8701,"context_line":"            self.assertFalse(ctx.cleaving_done)"},{"line_number":8702,"context_line":"            self.assertEqual(0, ctx.ranges_done)"},{"line_number":8703,"context_line":"            self.assertEqual(0, ctx.ranges_todo)"},{"line_number":8704,"context_line":"        ctx.start()"},{"line_number":8705,"context_line":"        check_context()"},{"line_number":8706,"context_line":"        # check idempotency"}],"source_content_type":"text/x-python","patch_set":16,"id":"020def01_b4c6663e","line":8703,"in_reply_to":"f39aaa91_b8c2b5d2","updated":"2025-10-10 17:18:14.000000000","message":"Done","commit_id":"a67a64d0de25e6c42ae7d957da2db2f66996b4e5"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"ac1c9680d8db1fc81ed41c73031d10e94ea260fd","unresolved":true,"context_lines":[{"line_number":3650,"context_line":"                   \u0027container\u0027: \u0027c000\u0027,"},{"line_number":3651,"context_line":"                   \u0027root\u0027: \u0027a/c\u0027,"},{"line_number":3652,"context_line":"                   \u0027object_count\u0027: 100,"},{"line_number":3653,"context_line":"                   \u0027tombstones\u0027: -1,"},{"line_number":3654,"context_line":"                   \u0027meta_timestamp\u0027: now.internal,"},{"line_number":3655,"context_line":"                   \u0027file_size\u0027: os.stat(brokers[0].db_file).st_size}"},{"line_number":3656,"context_line":"        self.assertEqual([stats_0], sharder.sharding_candidates)"}],"source_content_type":"text/x-python","patch_set":18,"id":"71147710_7eb10835","line":3653,"updated":"2025-10-13 15:42:04.000000000","message":"The current changes in test cases all have ``\u0027tombstones\u0027: -1`` which is the default unknown value. We need add/modify some test cases to show a valid value of ``\u0027tombstones\u0027``. For example, during a probe test run on my vasio, I saw ``\"tombstones\": 0``.\n\n\n```\n        {\n          \"account\": \".shards_AUTH_test\",\n          \"active\": 0,\n          \"cleaved\": 1,\n          \"container\": \"container-c054918e-2997-4ca6-bab2-1289d3b5ec08-185b3229b3ee36100a47d990a9b9d1df-1760132226.74811-1\",\n          \"created\": 1,\n          \"db_sharding_total_elapsed\": 11.642561435699463,\n          \"db_state\": \"sharded\",\n          \"error\": null,\n          \"file_size\": 53248,\n          \"found\": 0,\n          \"meta_timestamp\": \"1760132249.83083\",\n          \"node_index\": 1,\n          \"object_count\": 0,\n          \"path\": \"/srv/node4/sdb4/containers/151/784/25c813b4010b152462b443208e1fd784/25c813b4010b152462b443208e1fd784_1760132238.18828.db\",\n          \"processing_time\": 0.047776222229003906,\n          \"root\": \"AUTH_test/container-c054918e-2997-4ca6-bab2-1289d3b5ec08\",\n          \"state\": \"shrunk\",\n          \"tombstones\": 0,\n          \"total_replicate_calls\": 2,\n          \"total_replicate_time\": 0.10363459587097168\n        },\n```\n\nAnd all the new recon stats showed up, nice!","commit_id":"117cd99c31fb220ebc02f1440bef15724b4a9ff3"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"9bc59668d3dd7c2623fe9cad579a085fd5d2457e","unresolved":false,"context_lines":[{"line_number":3650,"context_line":"                   \u0027container\u0027: \u0027c000\u0027,"},{"line_number":3651,"context_line":"                   \u0027root\u0027: \u0027a/c\u0027,"},{"line_number":3652,"context_line":"                   \u0027object_count\u0027: 100,"},{"line_number":3653,"context_line":"                   \u0027tombstones\u0027: -1,"},{"line_number":3654,"context_line":"                   \u0027meta_timestamp\u0027: now.internal,"},{"line_number":3655,"context_line":"                   \u0027file_size\u0027: os.stat(brokers[0].db_file).st_size}"},{"line_number":3656,"context_line":"        self.assertEqual([stats_0], sharder.sharding_candidates)"}],"source_content_type":"text/x-python","patch_set":18,"id":"95052433_f66258b2","line":3653,"in_reply_to":"71147710_7eb10835","updated":"2025-10-13 23:32:46.000000000","message":"Done","commit_id":"117cd99c31fb220ebc02f1440bef15724b4a9ff3"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"c99e25185d780b7ebef8a12d01ddc20c68dbdadd","unresolved":true,"context_lines":[{"line_number":927,"context_line":"            self._assert_recon_stats(expected_stats, sharder, \u0027visited\u0027)"},{"line_number":928,"context_line":"            expected_candidate_stats \u003d {"},{"line_number":929,"context_line":"                \u0027found\u0027: 1,"},{"line_number":930,"context_line":"                \u0027top\u0027: [{\u0027object_count\u0027: 11, \u0027tombstones\u0027: 5, \u0027account\u0027: \u0027a\u0027,"},{"line_number":931,"context_line":"                         \u0027container\u0027: \u0027c2\u0027, \u0027meta_timestamp\u0027: mock.ANY,"},{"line_number":932,"context_line":"                         \u0027file_size\u0027: os.stat(brokers[1].db_file).st_size,"},{"line_number":933,"context_line":"                         \u0027path\u0027: brokers[2].db_file, \u0027root\u0027: \u0027a/c2\u0027,"}],"source_content_type":"text/x-python","patch_set":19,"id":"346596c0_ce770829","line":930,"updated":"2025-10-13 18:28:16.000000000","message":"I see. it\u0027s only shows up in the ``top`` section, not ``in_progress``. I think some probe tests might have \u0027tombstones\u0027 in recon files, but we haven\u0027t made assertions on them yet.","commit_id":"f49bd133afd4d6d1bef370b0927a8c4744c63d9e"},{"author":{"_account_id":38368,"name":"Christian Ohanaja","display_name":"Christian Ohanaja","email":"cohanaja@nvidia.com","username":"cohanaja"},"change_message_id":"8c9f2fa628f4c2aee58e7747da986b1221f27165","unresolved":false,"context_lines":[{"line_number":927,"context_line":"            self._assert_recon_stats(expected_stats, sharder, \u0027visited\u0027)"},{"line_number":928,"context_line":"            expected_candidate_stats \u003d {"},{"line_number":929,"context_line":"                \u0027found\u0027: 1,"},{"line_number":930,"context_line":"                \u0027top\u0027: [{\u0027object_count\u0027: 11, \u0027tombstones\u0027: 5, \u0027account\u0027: \u0027a\u0027,"},{"line_number":931,"context_line":"                         \u0027container\u0027: \u0027c2\u0027, \u0027meta_timestamp\u0027: mock.ANY,"},{"line_number":932,"context_line":"                         \u0027file_size\u0027: os.stat(brokers[1].db_file).st_size,"},{"line_number":933,"context_line":"                         \u0027path\u0027: brokers[2].db_file, \u0027root\u0027: \u0027a/c2\u0027,"}],"source_content_type":"text/x-python","patch_set":19,"id":"a29fb4c7_0449a347","line":930,"in_reply_to":"346596c0_ce770829","updated":"2025-10-13 20:16:20.000000000","message":"Done","commit_id":"f49bd133afd4d6d1bef370b0927a8c4744c63d9e"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"773fa0ef4a9ef9aa90c0af269c7d329625962e32","unresolved":true,"context_lines":[{"line_number":8480,"context_line":"        self.assertEqual(\u0027curs\u0027, ctx.cursor)"},{"line_number":8481,"context_line":"        self.assertTrue(ctx.misplaced_done)"},{"line_number":8482,"context_line":""},{"line_number":8483,"context_line":"    def test_store(self):"},{"line_number":8484,"context_line":"        broker \u003d self._make_sharding_broker()"},{"line_number":8485,"context_line":"        old_db_id \u003d broker.get_brokers()[0].get_info()[\u0027id\u0027]"},{"line_number":8486,"context_line":"        last_mod \u003d Timestamp.now()"}],"source_content_type":"text/x-python","patch_set":35,"id":"bc144342_f22fe842","line":8483,"updated":"2025-11-03 16:51:01.000000000","message":"I think this test case is same as ``test_store_old_style`` except that the initial value of ``replication_count`` is 4","commit_id":"565917f6126148c43814f4b5bd7a9615d20a14d3"}]}