)]}'
{"/PATCHSET_LEVEL":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a84a5c859b832d601c9e3f5de3deb6bc1a742a3b","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":1,"id":"8d33a8f8_5afb3c18","updated":"2025-06-09 14:39:34.000000000","message":"I tested this in prod and all it really told me was `_get_part_jobs` and `_get_suffixes_to_sync` are about 10x slower than `ssync`\n\n```\nobject-reconstructor: [worker 1/1 pid\u003d68393] Built 1 jobs for partition /srv/node/d48136/objects-1/242403 in 1080.171 seconds\n...\nobject-reconstructor: [worker 1/1 pid\u003d68393] _get_suffixes_to_sync completed successfully in 882.5246 seconds for node 10.52.131.166:6003/d59822\nobject-reconstructor: [worker 1/1 pid\u003d68393] Completed ssync_sender to 10.52.131.166:6003/d59822 in 114.6247 seconds with success for 100 suffixes on partition 242403 policy 1 (SYNC job)\n...\nobject-reconstructor: [worker 1/1 pid\u003d68393] _get_suffixes_to_sync completed successfully in 790.0534 seconds for node 10.150.37.43:6003/d9316\nobject-reconstructor: [worker 1/1 pid\u003d68393] Completed ssync_sender to 10.150.37.43:6003/d9316 in 43.4905 seconds with success for 50 suffixes on partition 242403 policy 1 (SYNC job)\n...\nobject-reconstructor: [worker 1/1 pid\u003d68393] Unable to get remote suffix hashes from \u002710.150.17.182:6003/d55593/242403 policy#1\u0027: Timeout (900s)\nobject-reconstructor: [worker 1/1 pid\u003d68393] _get_suffixes_to_sync completed with error in 900.0021 seconds for node 10.150.17.182:6003/d55593\n```","commit_id":"dfd364a158a65dbdfd34afdf5adcc0bf574c7d42"}],"swift/obj/reconstructor.py":[{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"e20b5fd3d9b4546ab8562312c82e29b2c1631848","unresolved":true,"context_lines":[{"line_number":1080,"context_line":"            # job, we don\u0027t purge any objects so start with the same set each"},{"line_number":1081,"context_line":"            # cycle"},{"line_number":1082,"context_line":"            ssync_start_time \u003d time.time()"},{"line_number":1083,"context_line":"            success, _ \u003d ssync_sender("},{"line_number":1084,"context_line":"                self, node, job, suffixes, include_non_durable\u003dFalse,"},{"line_number":1085,"context_line":"                max_objects\u003d0)()"},{"line_number":1086,"context_line":"            ssync_time \u003d time.time() - ssync_start_time"}],"source_content_type":"text/x-python","patch_set":1,"id":"0ef53acb_b550763d","line":1083,"updated":"2025-06-06 19:10:21.000000000","message":"we throw away ``can_delete_objs`` returned by calling ``ssync_sender()()``","commit_id":"dfd364a158a65dbdfd34afdf5adcc0bf574c7d42"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a84a5c859b832d601c9e3f5de3deb6bc1a742a3b","unresolved":true,"context_lines":[{"line_number":1080,"context_line":"            # job, we don\u0027t purge any objects so start with the same set each"},{"line_number":1081,"context_line":"            # cycle"},{"line_number":1082,"context_line":"            ssync_start_time \u003d time.time()"},{"line_number":1083,"context_line":"            success, _ \u003d ssync_sender("},{"line_number":1084,"context_line":"                self, node, job, suffixes, include_non_durable\u003dFalse,"},{"line_number":1085,"context_line":"                max_objects\u003d0)()"},{"line_number":1086,"context_line":"            ssync_time \u003d time.time() - ssync_start_time"}],"source_content_type":"text/x-python","patch_set":1,"id":"b6c03ea4_5f389bd6","line":1083,"in_reply_to":"0ef53acb_b550763d","updated":"2025-06-09 14:39:34.000000000","message":"I think it\u0027s only relevant to revert jobs?  Since we\u0027re a primary we don\u0027t want to delete anything?","commit_id":"dfd364a158a65dbdfd34afdf5adcc0bf574c7d42"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"e20b5fd3d9b4546ab8562312c82e29b2c1631848","unresolved":true,"context_lines":[{"line_number":1084,"context_line":"                self, node, job, suffixes, include_non_durable\u003dFalse,"},{"line_number":1085,"context_line":"                max_objects\u003d0)()"},{"line_number":1086,"context_line":"            ssync_time \u003d time.time() - ssync_start_time"},{"line_number":1087,"context_line":"            self.logger.debug("},{"line_number":1088,"context_line":"                \u0027Completed ssync_sender to %s in %.4f seconds with %s \u0027"},{"line_number":1089,"context_line":"                \u0027for %d suffixes on partition %d policy %d (SYNC job)\u0027,"},{"line_number":1090,"context_line":"                node_to_string(node, replication\u003dTrue), ssync_time,"}],"source_content_type":"text/x-python","patch_set":1,"id":"8aa9645e_6f63727b","line":1087,"updated":"2025-06-06 19:10:21.000000000","message":"that\u0027ll be great to print out how many objects were synced during this SYNC operation, ideally how many bytes as well, but I am not sure if there is an easy way to get that info.\n\n```\nsuccess, synced_objects \u003d ssync_sender(...)()\nobject_count \u003d len(synced_objects) if synced_objects else 0\nself.logger.debug(\n    \u0027Completed ssync_sender to %s in %.4f seconds with %s \u0027\n    \u0027for %d suffixes, %d objects on partition %d policy %d (SYNC job)\u0027,\n    node_to_string(node, replication\u003dTrue), ssync_time,\n    \u0027success\u0027 if success else \u0027failure\u0027, len(suffixes), object_count,\n    job[\u0027partition\u0027], job[\u0027policy\u0027].idx)\n```","commit_id":"dfd364a158a65dbdfd34afdf5adcc0bf574c7d42"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a84a5c859b832d601c9e3f5de3deb6bc1a742a3b","unresolved":true,"context_lines":[{"line_number":1084,"context_line":"                self, node, job, suffixes, include_non_durable\u003dFalse,"},{"line_number":1085,"context_line":"                max_objects\u003d0)()"},{"line_number":1086,"context_line":"            ssync_time \u003d time.time() - ssync_start_time"},{"line_number":1087,"context_line":"            self.logger.debug("},{"line_number":1088,"context_line":"                \u0027Completed ssync_sender to %s in %.4f seconds with %s \u0027"},{"line_number":1089,"context_line":"                \u0027for %d suffixes on partition %d policy %d (SYNC job)\u0027,"},{"line_number":1090,"context_line":"                node_to_string(node, replication\u003dTrue), ssync_time,"}],"source_content_type":"text/x-python","patch_set":1,"id":"2f9d0f36_6bf62a87","line":1087,"in_reply_to":"8aa9645e_6f63727b","updated":"2025-06-09 14:39:34.000000000","message":"```\nobject-reconstructor: [worker 1/1 pid\u003d68393] Completed ssync_sender to 10.52.131.166:6003/d59822 in 114.6247 seconds with success for 100 suffixes on partition 242403 policy 1 (SYNC job)\n```","commit_id":"dfd364a158a65dbdfd34afdf5adcc0bf574c7d42"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"e20b5fd3d9b4546ab8562312c82e29b2c1631848","unresolved":true,"context_lines":[{"line_number":1095,"context_line":"            self.logger.update_stats(\u0027suffix.syncs\u0027, len(suffixes))"},{"line_number":1096,"context_line":"        self.logger.timing_since(\u0027partition.update.timing\u0027, begin)"},{"line_number":1097,"context_line":""},{"line_number":1098,"context_line":"    def _revert(self, job, begin):"},{"line_number":1099,"context_line":"        \"\"\""},{"line_number":1100,"context_line":"        Process a REVERT job."},{"line_number":1101,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":1,"id":"dc4727dd_ff4febac","line":1098,"updated":"2025-06-06 19:10:21.000000000","message":"for the completeness of this patch, that\u0027ll be great to have timing breakdown for REVERT as well.","commit_id":"dfd364a158a65dbdfd34afdf5adcc0bf574c7d42"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a84a5c859b832d601c9e3f5de3deb6bc1a742a3b","unresolved":true,"context_lines":[{"line_number":1095,"context_line":"            self.logger.update_stats(\u0027suffix.syncs\u0027, len(suffixes))"},{"line_number":1096,"context_line":"        self.logger.timing_since(\u0027partition.update.timing\u0027, begin)"},{"line_number":1097,"context_line":""},{"line_number":1098,"context_line":"    def _revert(self, job, begin):"},{"line_number":1099,"context_line":"        \"\"\""},{"line_number":1100,"context_line":"        Process a REVERT job."},{"line_number":1101,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":1,"id":"a35ce76f_e472f152","line":1098,"in_reply_to":"dc4727dd_ff4febac","updated":"2025-06-09 14:39:34.000000000","message":"but that\u0027s not a problem for us right now; I\u0027m not sure yet I even need to carry this patch; much less merge it.","commit_id":"dfd364a158a65dbdfd34afdf5adcc0bf574c7d42"},{"author":{"_account_id":34930,"name":"Jianjian Huo","email":"jhuo@nvidia.com","username":"jhuo"},"change_message_id":"e20b5fd3d9b4546ab8562312c82e29b2c1631848","unresolved":true,"context_lines":[{"line_number":1525,"context_line":"            with Timeout(self.lockup_timeout):"},{"line_number":1526,"context_line":"                self.run_pool.waitall()"},{"line_number":1527,"context_line":"            execution_time \u003d time.time() - job_collection_start"},{"line_number":1528,"context_line":"            self.logger.debug("},{"line_number":1529,"context_line":"                \"Job execution completed: %.2f seconds, average %.3f \""},{"line_number":1530,"context_line":"                \"seconds per task\","},{"line_number":1531,"context_line":"                execution_time, execution_time / max(1, total_spawned))"}],"source_content_type":"text/x-python","patch_set":1,"id":"0e6377a0_9064a6d7","line":1528,"updated":"2025-06-06 19:10:21.000000000","message":"we can also print out the average process time for each kind of jobs:\n\n```\nJob execution completed: 45.2 seconds...\n  - SYNC jobs: 15 completed, average 2.3 seconds each...\n  - REVERT jobs: 8 completed, average 0.9 seconds each...\n```","commit_id":"dfd364a158a65dbdfd34afdf5adcc0bf574c7d42"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a84a5c859b832d601c9e3f5de3deb6bc1a742a3b","unresolved":true,"context_lines":[{"line_number":1525,"context_line":"            with Timeout(self.lockup_timeout):"},{"line_number":1526,"context_line":"                self.run_pool.waitall()"},{"line_number":1527,"context_line":"            execution_time \u003d time.time() - job_collection_start"},{"line_number":1528,"context_line":"            self.logger.debug("},{"line_number":1529,"context_line":"                \"Job execution completed: %.2f seconds, average %.3f \""},{"line_number":1530,"context_line":"                \"seconds per task\","},{"line_number":1531,"context_line":"                execution_time, execution_time / max(1, total_spawned))"}],"source_content_type":"text/x-python","patch_set":1,"id":"ae6728fa_bfc76f39","line":1528,"in_reply_to":"0e6377a0_9064a6d7","updated":"2025-06-09 14:39:34.000000000","message":"that could be useful; IME the runtime will tend to be dominated by one or the other - in our case it\u0027s currently SYNC; when we have lots of revert jobs we tend to run in handoffs_only (which doesn\u0027t cause the DELETE/expirer slow down)","commit_id":"dfd364a158a65dbdfd34afdf5adcc0bf574c7d42"}]}