)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"15abbceef29b7ceb7e79521048632c4643933ee4","unresolved":false,"context_lines":[{"line_number":6,"context_line":""},{"line_number":7,"context_line":"Fix a race condition in case of cross-replication"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"In a situation where two nodes does not have the same version of a ring"},{"line_number":10,"context_line":"and they both think the other node is the primary node of a partition,"},{"line_number":11,"context_line":"a race condition can lead to the loss of some of the objects of the"},{"line_number":12,"context_line":"partition."}],"source_content_type":"text/x-gerrit-commit-message","patch_set":6,"id":"7f6b1bfe_2bb89fc6","line":9,"updated":"2020-10-21 04:10:38.000000000","message":"\"nodes do not have\"","commit_id":"8c0a1abf744a11b5c289239e3ac830786a9de4e9"}],"swift/obj/diskfile.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"5241112dc7a069f409eb32952648e99b9df3d514","unresolved":false,"context_lines":[{"line_number":1352,"context_line":"                               limit\u003dself.replication_concurrency_per_device):"},{"line_number":1353,"context_line":"                    yield True"},{"line_number":1354,"context_line":"            else:"},{"line_number":1355,"context_line":"                yield True"},{"line_number":1356,"context_line":""},{"line_number":1357,"context_line":"    @contextmanager"},{"line_number":1358,"context_line":"    def partition_lock(self, device, policy, partition, name\u003dNone,"}],"source_content_type":"text/x-python","patch_set":4,"id":"9f560f44_cbce4c8e","line":1355,"updated":"2020-09-30 18:59:04.000000000","message":"i can\u0027t think of a good reason why the original order mattered - obviously we\u0027d hold the partition lock waiting to get a slot for the device, but it\u0027s not obvious to me that\u0027s a problem","commit_id":"7d1135a0c5051a8c1f1f22b61196b88e62d2f7e3"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"5241112dc7a069f409eb32952648e99b9df3d514","unresolved":false,"context_lines":[{"line_number":1359,"context_line":"                       timeout\u003dNone):"},{"line_number":1360,"context_line":"        \"\"\""},{"line_number":1361,"context_line":"        A context manager that will lock on the device given, if"},{"line_number":1362,"context_line":"        configured to do so."},{"line_number":1363,"context_line":""},{"line_number":1364,"context_line":"        :param device: name of target device"},{"line_number":1365,"context_line":"        :param policy: policy targeted by the replication request"}],"source_content_type":"text/x-python","patch_set":4,"id":"9f560f44_ab2e783f","line":1362,"updated":"2020-09-30 18:59:04.000000000","message":"lock on the *partition* given (and I don\u0027t think it\u0027s conditional)","commit_id":"7d1135a0c5051a8c1f1f22b61196b88e62d2f7e3"},{"author":{"_account_id":13852,"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","username":"rledisez"},"change_message_id":"dc750202503bfb60f05592d5290969b5d7fcdef7","unresolved":false,"context_lines":[{"line_number":1359,"context_line":"                       timeout\u003dNone):"},{"line_number":1360,"context_line":"        \"\"\""},{"line_number":1361,"context_line":"        A context manager that will lock on the device given, if"},{"line_number":1362,"context_line":"        configured to do so."},{"line_number":1363,"context_line":""},{"line_number":1364,"context_line":"        :param device: name of target device"},{"line_number":1365,"context_line":"        :param policy: policy targeted by the replication request"}],"source_content_type":"text/x-python","patch_set":4,"id":"9f560f44_d8956ee5","line":1362,"in_reply_to":"9f560f44_ab2e783f","updated":"2020-10-01 22:06:20.000000000","message":"Done","commit_id":"7d1135a0c5051a8c1f1f22b61196b88e62d2f7e3"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"5241112dc7a069f409eb32952648e99b9df3d514","unresolved":false,"context_lines":[{"line_number":1365,"context_line":"        :param policy: policy targeted by the replication request"},{"line_number":1366,"context_line":"        :param partition: partition targeted by the replication request"},{"line_number":1367,"context_line":"        :raises PartitionLockTimeout: If the lock on the partition"},{"line_number":1368,"context_line":"            cannot be granted within the configured timeout."},{"line_number":1369,"context_line":"        \"\"\""},{"line_number":1370,"context_line":"        if timeout is None:"},{"line_number":1371,"context_line":"            timeout \u003d self.replication_lock_timeout"}],"source_content_type":"text/x-python","patch_set":4,"id":"9f560f44_cbd80c26","line":1368,"updated":"2020-09-30 18:59:04.000000000","message":"Do we want to pull \"name\" out of the signature and assume our partition_lock is always a replication lock?  or is there some plan to converge partition lock with the call to lock_path used for suffix hash invalidation?","commit_id":"7d1135a0c5051a8c1f1f22b61196b88e62d2f7e3"},{"author":{"_account_id":13852,"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","username":"rledisez"},"change_message_id":"dc750202503bfb60f05592d5290969b5d7fcdef7","unresolved":false,"context_lines":[{"line_number":1365,"context_line":"        :param policy: policy targeted by the replication request"},{"line_number":1366,"context_line":"        :param partition: partition targeted by the replication request"},{"line_number":1367,"context_line":"        :raises PartitionLockTimeout: If the lock on the partition"},{"line_number":1368,"context_line":"            cannot be granted within the configured timeout."},{"line_number":1369,"context_line":"        \"\"\""},{"line_number":1370,"context_line":"        if timeout is None:"},{"line_number":1371,"context_line":"            timeout \u003d self.replication_lock_timeout"}],"source_content_type":"text/x-python","patch_set":4,"id":"9f560f44_18b9a678","line":1368,"in_reply_to":"9f560f44_cbd80c26","updated":"2020-10-01 22:06:20.000000000","message":"I\u0027m hesitant on that. I think it could be re-used later, that\u0027s why I kept the parameter. It can be removed if it seems appropriate, but we may want to change the function name then. Something like partition_replication_lock?","commit_id":"7d1135a0c5051a8c1f1f22b61196b88e62d2f7e3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"82d789805d6b31134c4f3abf3c2343b4a634ec23","unresolved":false,"context_lines":[{"line_number":1346,"context_line":"        with self.partition_lock(device, policy, partition, name\u003d\u0027replication\u0027,"},{"line_number":1347,"context_line":"                                 timeout\u003dself.replication_lock_timeout):"},{"line_number":1348,"context_line":"            if self.replication_concurrency_per_device:"},{"line_number":1349,"context_line":"                with lock_path(self.get_dev_path(device),"},{"line_number":1350,"context_line":"                               timeout\u003dlimit_time - time.time(),"},{"line_number":1351,"context_line":"                               timeout_class\u003dReplicationLockTimeout,"},{"line_number":1352,"context_line":"                               limit\u003dself.replication_concurrency_per_device):"}],"source_content_type":"text/x-python","patch_set":6,"id":"1f621f24_9e2a988d","line":1349,"updated":"2020-11-04 23:04:26.000000000","message":"Is the order of these lock acquisitions significant? I understand how it could deadlock if different lockers acquired them in different orders, but it seems like this is the only place that we acquire both... right?\n\nI suppose this order has a maintainability advantage given the replication_concurrency_per_device logic... plus we aren\u0027t left holding the device lock for a while while waiting on the partition lock...","commit_id":"8c0a1abf744a11b5c289239e3ac830786a9de4e9"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"bb766ab0e289b9aa35325f07823b5ecb877418ee","unresolved":false,"context_lines":[{"line_number":1346,"context_line":"        with self.partition_lock(device, policy, partition, name\u003d\u0027replication\u0027,"},{"line_number":1347,"context_line":"                                 timeout\u003dself.replication_lock_timeout):"},{"line_number":1348,"context_line":"            if self.replication_concurrency_per_device:"},{"line_number":1349,"context_line":"                with lock_path(self.get_dev_path(device),"},{"line_number":1350,"context_line":"                               timeout\u003dlimit_time - time.time(),"},{"line_number":1351,"context_line":"                               timeout_class\u003dReplicationLockTimeout,"},{"line_number":1352,"context_line":"                               limit\u003dself.replication_concurrency_per_device):"}],"source_content_type":"text/x-python","patch_set":6,"id":"1f621f24_8c753635","line":1349,"in_reply_to":"1f621f24_9e2a988d","updated":"2020-11-07 00:59:50.000000000","message":"I considered this at a certain length too, but I\u0027m unable to construct a scenario where this change poses a hazard of a deadlock (at least within the same node).","commit_id":"8c0a1abf744a11b5c289239e3ac830786a9de4e9"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"82d789805d6b31134c4f3abf3c2343b4a634ec23","unresolved":false,"context_lines":[{"line_number":1362,"context_line":""},{"line_number":1363,"context_line":"        :param device: device targeted by the lock request"},{"line_number":1364,"context_line":"        :param policy: policy targeted by the lock request"},{"line_number":1365,"context_line":"        :param partition: partition targeted by the lock request"},{"line_number":1366,"context_line":"        :raises PartitionLockTimeout: If the lock on the partition"},{"line_number":1367,"context_line":"            cannot be granted within the configured timeout."},{"line_number":1368,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":6,"id":"1f621f24_c11d5567","line":1365,"updated":"2020-11-04 23:04:26.000000000","message":"nit: could use :param:s for timeout (and name, if we keep it)","commit_id":"8c0a1abf744a11b5c289239e3ac830786a9de4e9"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"82d789805d6b31134c4f3abf3c2343b4a634ec23","unresolved":false,"context_lines":[{"line_number":1366,"context_line":"        :raises PartitionLockTimeout: If the lock on the partition"},{"line_number":1367,"context_line":"            cannot be granted within the configured timeout."},{"line_number":1368,"context_line":"        \"\"\""},{"line_number":1369,"context_line":"        if timeout is None:"},{"line_number":1370,"context_line":"            timeout \u003d self.replication_lock_timeout"},{"line_number":1371,"context_line":"        part_path \u003d os.path.join(self.get_dev_path(device),"},{"line_number":1372,"context_line":"                                 get_data_dir(policy), str(partition))"}],"source_content_type":"text/x-python","patch_set":6,"id":"1f621f24_7e25fcc1","line":1369,"updated":"2020-11-04 23:04:26.000000000","message":"Is timeout ever None in practice? It looks like we\u0027ve just got the two callers, in replication_lock (where we\u0027re already using self.replication_lock_timeout) and in reconstructor (where we hard-code it to 0.2)...","commit_id":"8c0a1abf744a11b5c289239e3ac830786a9de4e9"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"3e8aaf1c1867777dbf34651b4a057f7338adfd1b","unresolved":false,"context_lines":[{"line_number":1366,"context_line":"        :raises PartitionLockTimeout: If the lock on the partition"},{"line_number":1367,"context_line":"            cannot be granted within the configured timeout."},{"line_number":1368,"context_line":"        \"\"\""},{"line_number":1369,"context_line":"        if timeout is None:"},{"line_number":1370,"context_line":"            timeout \u003d self.replication_lock_timeout"},{"line_number":1371,"context_line":"        part_path \u003d os.path.join(self.get_dev_path(device),"},{"line_number":1372,"context_line":"                                 get_data_dir(policy), str(partition))"}],"source_content_type":"text/x-python","patch_set":6,"id":"1f621f24_e23c8530","line":1369,"in_reply_to":"1f621f24_2c7ce21d","updated":"2020-11-11 18:49:41.000000000","message":"I was thinking about making it required, yeah.","commit_id":"8c0a1abf744a11b5c289239e3ac830786a9de4e9"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"bb766ab0e289b9aa35325f07823b5ecb877418ee","unresolved":false,"context_lines":[{"line_number":1366,"context_line":"        :raises PartitionLockTimeout: If the lock on the partition"},{"line_number":1367,"context_line":"            cannot be granted within the configured timeout."},{"line_number":1368,"context_line":"        \"\"\""},{"line_number":1369,"context_line":"        if timeout is None:"},{"line_number":1370,"context_line":"            timeout \u003d self.replication_lock_timeout"},{"line_number":1371,"context_line":"        part_path \u003d os.path.join(self.get_dev_path(device),"},{"line_number":1372,"context_line":"                                 get_data_dir(policy), str(partition))"}],"source_content_type":"text/x-python","patch_set":6,"id":"1f621f24_2c7ce21d","line":1369,"in_reply_to":"1f621f24_7e25fcc1","updated":"2020-11-07 00:59:50.000000000","message":"Not sure what you\u0027re implying here. A keyword parameter must have a default. Do you want to convert timeout to a positional parameter? Or use 0.0 instead of None?","commit_id":"8c0a1abf744a11b5c289239e3ac830786a9de4e9"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"82d789805d6b31134c4f3abf3c2343b4a634ec23","unresolved":false,"context_lines":[{"line_number":1371,"context_line":"        part_path \u003d os.path.join(self.get_dev_path(device),"},{"line_number":1372,"context_line":"                                 get_data_dir(policy), str(partition))"},{"line_number":1373,"context_line":"        with lock_path(part_path, timeout\u003dtimeout,"},{"line_number":1374,"context_line":"                       timeout_class\u003dPartitionLockTimeout, limit\u003d1, name\u003dname):"},{"line_number":1375,"context_line":"            yield True"},{"line_number":1376,"context_line":""},{"line_number":1377,"context_line":"    def pickle_async_update(self, device, account, container, obj, data,"}],"source_content_type":"text/x-python","patch_set":6,"id":"1f621f24_610e0133","line":1374,"range":{"start_line":1374,"start_character":73,"end_line":1374,"end_character":77},"updated":"2020-11-04 23:04:26.000000000","message":"Should we hard-code this to be \u0027replication\u0027 here, instead of up in replication_lock? That much less argument plumbing required...","commit_id":"8c0a1abf744a11b5c289239e3ac830786a9de4e9"}],"swift/obj/reconstructor.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"077b107937c9434daf98f33c8ee9923c8272cf00","unresolved":false,"context_lines":[{"line_number":841,"context_line":"        if job[\u0027job_type\u0027] \u003d\u003d REVERT:"},{"line_number":842,"context_line":"            self._revert(job, begin)"},{"line_number":843,"context_line":"        else:"},{"line_number":844,"context_line":"            self._sync(job, begin)"},{"line_number":845,"context_line":"        self.partition_times.append(time.time() - begin)"},{"line_number":846,"context_line":"        self.reconstruction_count +\u003d 1"},{"line_number":847,"context_line":""}],"source_content_type":"text/x-python","patch_set":1,"id":"9f560f44_45bc32b4","line":844,"updated":"2020-09-25 15:17:02.000000000","message":"maybe here would be a good layer for a pokemon exception handler","commit_id":"5787c9f92a7df55473d5c074dd085fdae21d72e3"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"077b107937c9434daf98f33c8ee9923c8272cf00","unresolved":false,"context_lines":[{"line_number":895,"context_line":"            self.delete_reverted_objs("},{"line_number":896,"context_line":"                job, reverted_objs, job[\u0027frag_index\u0027])"},{"line_number":897,"context_line":"        else:"},{"line_number":898,"context_line":"            self.handoffs_remaining +\u003d 1"},{"line_number":899,"context_line":"        self.logger.timing_since(\u0027partition.delete.timing\u0027, begin)"},{"line_number":900,"context_line":""},{"line_number":901,"context_line":"    def _get_part_jobs(self, local_dev, part_path, partition, policy):"}],"source_content_type":"text/x-python","patch_set":1,"id":"9f560f44_e5946638","line":898,"updated":"2020-09-25 15:17:02.000000000","message":"we may loose some accounting if that lock pops","commit_id":"5787c9f92a7df55473d5c074dd085fdae21d72e3"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"077b107937c9434daf98f33c8ee9923c8272cf00","unresolved":false,"context_lines":[{"line_number":1235,"context_line":"                    self.run_pool.spawn(self.delete_partition,"},{"line_number":1236,"context_line":"                                        part_info[\u0027part_path\u0027])"},{"line_number":1237,"context_line":"                for job in jobs:"},{"line_number":1238,"context_line":"                    self.run_pool.spawn(self.process_job, job)"},{"line_number":1239,"context_line":"            with Timeout(self.lockup_timeout):"},{"line_number":1240,"context_line":"                self.run_pool.waitall()"},{"line_number":1241,"context_line":"        except (Exception, Timeout):"}],"source_content_type":"text/x-python","patch_set":1,"id":"9f560f44_65a85677","line":1238,"updated":"2020-09-25 15:17:02.000000000","message":"I think we just let unhanded exceptions in process_job fail in the background","commit_id":"5787c9f92a7df55473d5c074dd085fdae21d72e3"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"cb87b8eb77e256a13cb076b409d338508bd5ad94","unresolved":false,"context_lines":[{"line_number":881,"context_line":"        reverted_objs \u003d {}"},{"line_number":882,"context_line":"        try:"},{"line_number":883,"context_line":"            with lock_path(job[\u0027path\u0027],"},{"line_number":884,"context_line":"                           timeout\u003d1,"},{"line_number":885,"context_line":"                           timeout_class\u003dPartitionLockTimeout,"},{"line_number":886,"context_line":"                           name\u003d\u0027replication\u0027):"},{"line_number":887,"context_line":"                for node in job[\u0027sync_to\u0027]:"}],"source_content_type":"text/x-python","patch_set":2,"id":"9f560f44_ddc8e94e","line":884,"updated":"2020-09-29 14:50:05.000000000","message":"Do you have any justification for this value; i.e. should it be configurable?  (e.g. replication_lock_timeout)","commit_id":"5c313607336299bd46be67523d3036df7a4614ee"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"5241112dc7a069f409eb32952648e99b9df3d514","unresolved":false,"context_lines":[{"line_number":881,"context_line":"        reverted_objs \u003d {}"},{"line_number":882,"context_line":"        try:"},{"line_number":883,"context_line":"            with lock_path(job[\u0027path\u0027],"},{"line_number":884,"context_line":"                           timeout\u003d1,"},{"line_number":885,"context_line":"                           timeout_class\u003dPartitionLockTimeout,"},{"line_number":886,"context_line":"                           name\u003d\u0027replication\u0027):"},{"line_number":887,"context_line":"                for node in job[\u0027sync_to\u0027]:"}],"source_content_type":"text/x-python","patch_set":2,"id":"9f560f44_0b8044f7","line":884,"in_reply_to":"9f560f44_4b6fdc92","updated":"2020-09-30 18:59:04.000000000","message":"i\u0027d guess 1-30s are all reasonable, default 15s is probably middle of the road\n\ndefinitely worth it to keep it relatively low, 300s or something would seem nuts to me, but I\u0027ve never tested it","commit_id":"5c313607336299bd46be67523d3036df7a4614ee"},{"author":{"_account_id":13852,"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","username":"rledisez"},"change_message_id":"41b17cef6730091e67273e60e745dc3332620a79","unresolved":false,"context_lines":[{"line_number":881,"context_line":"        reverted_objs \u003d {}"},{"line_number":882,"context_line":"        try:"},{"line_number":883,"context_line":"            with lock_path(job[\u0027path\u0027],"},{"line_number":884,"context_line":"                           timeout\u003d1,"},{"line_number":885,"context_line":"                           timeout_class\u003dPartitionLockTimeout,"},{"line_number":886,"context_line":"                           name\u003d\u0027replication\u0027):"},{"line_number":887,"context_line":"                for node in job[\u0027sync_to\u0027]:"}],"source_content_type":"text/x-python","patch_set":2,"id":"9f560f44_4b6fdc92","line":884,"in_reply_to":"9f560f44_ddc8e94e","updated":"2020-09-30 18:42:53.000000000","message":"No justifications. I put back replication_lock_timeout (I\u0027m always torned on that parameter: is it best to wait or fail fast and come back later)","commit_id":"5c313607336299bd46be67523d3036df7a4614ee"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"cb87b8eb77e256a13cb076b409d338508bd5ad94","unresolved":false,"context_lines":[{"line_number":883,"context_line":"            with lock_path(job[\u0027path\u0027],"},{"line_number":884,"context_line":"                           timeout\u003d1,"},{"line_number":885,"context_line":"                           timeout_class\u003dPartitionLockTimeout,"},{"line_number":886,"context_line":"                           name\u003d\u0027replication\u0027):"},{"line_number":887,"context_line":"                for node in job[\u0027sync_to\u0027]:"},{"line_number":888,"context_line":"                    node[\u0027backend_index\u0027] \u003d job[\u0027policy\u0027].get_backend_index("},{"line_number":889,"context_line":"                        node[\u0027index\u0027])"}],"source_content_type":"text/x-python","patch_set":2,"id":"9f560f44_3d9ac56e","line":886,"updated":"2020-09-29 14:50:05.000000000","message":"it seems like path + the *name* here guarantees that it synchronizes with the incoming ssync_receiver lock (see diskfile.BaseDiskFileManager.replication_lock common.utils.lock_path and obj.ssync_reciever.Receiver.__call__","commit_id":"5c313607336299bd46be67523d3036df7a4614ee"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"5241112dc7a069f409eb32952648e99b9df3d514","unresolved":false,"context_lines":[{"line_number":902,"context_line":"                else:"},{"line_number":903,"context_line":"                    self.handoffs_remaining +\u003d 1"},{"line_number":904,"context_line":"        except PartitionLockTimeout:"},{"line_number":905,"context_line":"            self.logger.info(\"Unable to lock handoff partition for revert\")"},{"line_number":906,"context_line":"            self.logger.increment(\u0027partition.lock-failure.count\u0027)"},{"line_number":907,"context_line":"            self.handoffs_remaining +\u003d 1"},{"line_number":908,"context_line":"        self.logger.timing_since(\u0027partition.delete.timing\u0027, begin)"}],"source_content_type":"text/x-python","patch_set":4,"id":"9f560f44_ebb6b0f9","line":905,"updated":"2020-09-30 18:59:04.000000000","message":"extra info in the log line is free tho, part and device would be great!","commit_id":"7d1135a0c5051a8c1f1f22b61196b88e62d2f7e3"},{"author":{"_account_id":13852,"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","username":"rledisez"},"change_message_id":"dc750202503bfb60f05592d5290969b5d7fcdef7","unresolved":false,"context_lines":[{"line_number":902,"context_line":"                else:"},{"line_number":903,"context_line":"                    self.handoffs_remaining +\u003d 1"},{"line_number":904,"context_line":"        except PartitionLockTimeout:"},{"line_number":905,"context_line":"            self.logger.info(\"Unable to lock handoff partition for revert\")"},{"line_number":906,"context_line":"            self.logger.increment(\u0027partition.lock-failure.count\u0027)"},{"line_number":907,"context_line":"            self.handoffs_remaining +\u003d 1"},{"line_number":908,"context_line":"        self.logger.timing_since(\u0027partition.delete.timing\u0027, begin)"}],"source_content_type":"text/x-python","patch_set":4,"id":"9f560f44_a25968fe","line":905,"in_reply_to":"9f560f44_ebb6b0f9","updated":"2020-10-01 22:06:20.000000000","message":"Done","commit_id":"7d1135a0c5051a8c1f1f22b61196b88e62d2f7e3"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"5241112dc7a069f409eb32952648e99b9df3d514","unresolved":false,"context_lines":[{"line_number":903,"context_line":"                    self.handoffs_remaining +\u003d 1"},{"line_number":904,"context_line":"        except PartitionLockTimeout:"},{"line_number":905,"context_line":"            self.logger.info(\"Unable to lock handoff partition for revert\")"},{"line_number":906,"context_line":"            self.logger.increment(\u0027partition.lock-failure.count\u0027)"},{"line_number":907,"context_line":"            self.handoffs_remaining +\u003d 1"},{"line_number":908,"context_line":"        self.logger.timing_since(\u0027partition.delete.timing\u0027, begin)"},{"line_number":909,"context_line":""}],"source_content_type":"text/x-python","patch_set":4,"id":"9f560f44_6be900e2","line":906,"updated":"2020-09-30 18:59:04.000000000","message":"trying to decide if I want metrics per-device - our metrics storage cost us like \"metric names x nodes\" so maybe it\u0027s better to start here.","commit_id":"7d1135a0c5051a8c1f1f22b61196b88e62d2f7e3"}],"swift/obj/replicator.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"077b107937c9434daf98f33c8ee9923c8272cf00","unresolved":false,"context_lines":[{"line_number":573,"context_line":"                    self.delete_partition(job[\u0027path\u0027])"},{"line_number":574,"context_line":"                    handoff_partition_deleted \u003d True"},{"line_number":575,"context_line":"        except (Exception, Timeout):"},{"line_number":576,"context_line":"            self.logger.exception(_(\"Error syncing handoff partition\"))"},{"line_number":577,"context_line":"            stats.add_failure_stats(failure_devs_info)"},{"line_number":578,"context_line":"        finally:"},{"line_number":579,"context_line":"            target_devs_info \u003d set([(target_dev[\u0027replication_ip\u0027],"}],"source_content_type":"text/x-python","patch_set":1,"id":"9f560f44_e5b10603","line":576,"updated":"2020-09-25 15:17:02.000000000","message":"looks like ReplicationLockTimeout would be caught here","commit_id":"5787c9f92a7df55473d5c074dd085fdae21d72e3"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"cb87b8eb77e256a13cb076b409d338508bd5ad94","unresolved":false,"context_lines":[{"line_number":502,"context_line":"            with lock_path(job[\u0027path\u0027],"},{"line_number":503,"context_line":"                           timeout\u003d1,"},{"line_number":504,"context_line":"                           timeout_class\u003dPartitionLockTimeout,"},{"line_number":505,"context_line":"                           name\u003d\u0027replication\u0027):"},{"line_number":506,"context_line":"                responses \u003d []"},{"line_number":507,"context_line":"                suffixes \u003d tpool.execute(tpool_get_suffixes, job[\u0027path\u0027])"},{"line_number":508,"context_line":"                synced_remote_regions \u003d {}"}],"source_content_type":"text/x-python","patch_set":2,"id":"9f560f44_5dddf911","line":505,"updated":"2020-09-29 14:50:05.000000000","message":"both ssync \u0026 rsync replication will take the lock - only ssync lock will ever encounter the object-server already holding it\n\nis that difference worth a comment in code?","commit_id":"5c313607336299bd46be67523d3036df7a4614ee"},{"author":{"_account_id":13852,"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","username":"rledisez"},"change_message_id":"41b17cef6730091e67273e60e745dc3332620a79","unresolved":false,"context_lines":[{"line_number":502,"context_line":"            with lock_path(job[\u0027path\u0027],"},{"line_number":503,"context_line":"                           timeout\u003d1,"},{"line_number":504,"context_line":"                           timeout_class\u003dPartitionLockTimeout,"},{"line_number":505,"context_line":"                           name\u003d\u0027replication\u0027):"},{"line_number":506,"context_line":"                responses \u003d []"},{"line_number":507,"context_line":"                suffixes \u003d tpool.execute(tpool_get_suffixes, job[\u0027path\u0027])"},{"line_number":508,"context_line":"                synced_remote_regions \u003d {}"}],"source_content_type":"text/x-python","patch_set":2,"id":"9f560f44_6b6620be","line":505,"in_reply_to":"9f560f44_5dddf911","updated":"2020-09-30 18:42:53.000000000","message":"Done","commit_id":"5c313607336299bd46be67523d3036df7a4614ee"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"cb87b8eb77e256a13cb076b409d338508bd5ad94","unresolved":false,"context_lines":[{"line_number":576,"context_line":"                    handoff_partition_deleted \u003d True"},{"line_number":577,"context_line":"        except PartitionLockTimeout:"},{"line_number":578,"context_line":"            self.logger.info("},{"line_number":579,"context_line":"                \"Unable to lock handoff partition for replication\")"},{"line_number":580,"context_line":"        except (Exception, Timeout):"},{"line_number":581,"context_line":"            self.logger.exception(_(\"Error syncing handoff partition\"))"},{"line_number":582,"context_line":"            stats.add_failure_stats(failure_devs_info)"}],"source_content_type":"text/x-python","patch_set":2,"id":"9f560f44_3d4be5b1","line":579,"updated":"2020-09-29 14:50:05.000000000","message":"this is nice, I think as an improvement we could add a count to stats down the road","commit_id":"5c313607336299bd46be67523d3036df7a4614ee"},{"author":{"_account_id":13852,"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","username":"rledisez"},"change_message_id":"41b17cef6730091e67273e60e745dc3332620a79","unresolved":false,"context_lines":[{"line_number":576,"context_line":"                    handoff_partition_deleted \u003d True"},{"line_number":577,"context_line":"        except PartitionLockTimeout:"},{"line_number":578,"context_line":"            self.logger.info("},{"line_number":579,"context_line":"                \"Unable to lock handoff partition for replication\")"},{"line_number":580,"context_line":"        except (Exception, Timeout):"},{"line_number":581,"context_line":"            self.logger.exception(_(\"Error syncing handoff partition\"))"},{"line_number":582,"context_line":"            stats.add_failure_stats(failure_devs_info)"}],"source_content_type":"text/x-python","patch_set":2,"id":"9f560f44_0b0b24d5","line":579,"in_reply_to":"9f560f44_3d4be5b1","updated":"2020-09-30 18:42:53.000000000","message":"Done","commit_id":"5c313607336299bd46be67523d3036df7a4614ee"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"5241112dc7a069f409eb32952648e99b9df3d514","unresolved":false,"context_lines":[{"line_number":503,"context_line":"            # Only object-server can take this lock if an incoming SSYNC is"},{"line_number":504,"context_line":"            # running on the same partition. Taking the lock here ensure we"},{"line_number":505,"context_line":"            # won\u0027t enter a race condition where both nodes try to"},{"line_number":506,"context_line":"            # cross-replicate the same partition and both delete it."},{"line_number":507,"context_line":"            with df_mgr.partition_lock(job[\u0027device\u0027], job[\u0027policy\u0027],"},{"line_number":508,"context_line":"                                       job[\u0027partition\u0027], name\u003d\u0027replication\u0027):"},{"line_number":509,"context_line":"                responses \u003d []"}],"source_content_type":"text/x-python","patch_set":4,"id":"9f560f44_4b799c2e","line":506,"updated":"2020-09-30 18:59:04.000000000","message":"thanks for the comment.","commit_id":"7d1135a0c5051a8c1f1f22b61196b88e62d2f7e3"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"5241112dc7a069f409eb32952648e99b9df3d514","unresolved":false,"context_lines":[{"line_number":580,"context_line":"        except PartitionLockTimeout:"},{"line_number":581,"context_line":"            self.logger.info("},{"line_number":582,"context_line":"                \"Unable to lock handoff partition for replication\")"},{"line_number":583,"context_line":"            self.logger.increment(\u0027partition.lock-failure.count\u0027)"},{"line_number":584,"context_line":"        except (Exception, Timeout):"},{"line_number":585,"context_line":"            self.logger.exception(_(\"Error syncing handoff partition\"))"},{"line_number":586,"context_line":"            stats.add_failure_stats(failure_devs_info)"}],"source_content_type":"text/x-python","patch_set":4,"id":"9f560f44_4b907c50","line":583,"updated":"2020-09-30 18:59:04.000000000","message":"seems about the same as the reconstructor","commit_id":"7d1135a0c5051a8c1f1f22b61196b88e62d2f7e3"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"5241112dc7a069f409eb32952648e99b9df3d514","unresolved":false,"context_lines":[{"line_number":590,"context_line":"                                    for target_dev in job[\u0027nodes\u0027]])"},{"line_number":591,"context_line":"            stats.success +\u003d len(target_devs_info - failure_devs_info)"},{"line_number":592,"context_line":"            if not handoff_partition_deleted:"},{"line_number":593,"context_line":"                self.handoffs_remaining +\u003d 1"},{"line_number":594,"context_line":"            self.partition_times.append(time.time() - begin)"},{"line_number":595,"context_line":"            self.logger.timing_since(\u0027partition.delete.timing\u0027, begin)"},{"line_number":596,"context_line":""}],"source_content_type":"text/x-python","patch_set":4,"id":"9f560f44_2b8d4835","line":593,"updated":"2020-09-30 18:59:04.000000000","message":"ok, so this is all in a finally - should be fine.","commit_id":"7d1135a0c5051a8c1f1f22b61196b88e62d2f7e3"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"82d789805d6b31134c4f3abf3c2343b4a634ec23","unresolved":false,"context_lines":[{"line_number":506,"context_line":"            # cross-replicate the same partition and both delete it."},{"line_number":507,"context_line":"            with df_mgr.partition_lock(job[\u0027device\u0027], job[\u0027policy\u0027],"},{"line_number":508,"context_line":"                                       job[\u0027partition\u0027], name\u003d\u0027replication\u0027,"},{"line_number":509,"context_line":"                                       timeout\u003d0.2):"},{"line_number":510,"context_line":"                responses \u003d []"},{"line_number":511,"context_line":"                suffixes \u003d tpool.execute(tpool_get_suffixes, job[\u0027path\u0027])"},{"line_number":512,"context_line":"                synced_remote_regions \u003d {}"}],"source_content_type":"text/x-python","patch_set":6,"id":"1f621f24_1ea32851","line":509,"updated":"2020-11-04 23:04:26.000000000","message":"Right -- we only need the lock in update_deleted because it\u0027s the delete we\u0027re guarding against.","commit_id":"8c0a1abf744a11b5c289239e3ac830786a9de4e9"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"82d789805d6b31134c4f3abf3c2343b4a634ec23","unresolved":false,"context_lines":[{"line_number":590,"context_line":"            target_devs_info \u003d set([(target_dev[\u0027replication_ip\u0027],"},{"line_number":591,"context_line":"                                     target_dev[\u0027device\u0027])"},{"line_number":592,"context_line":"                                    for target_dev in job[\u0027nodes\u0027]])"},{"line_number":593,"context_line":"            stats.success +\u003d len(target_devs_info - failure_devs_info)"},{"line_number":594,"context_line":"            if not handoff_partition_deleted:"},{"line_number":595,"context_line":"                self.handoffs_remaining +\u003d 1"},{"line_number":596,"context_line":"            self.partition_times.append(time.time() - begin)"}],"source_content_type":"text/x-python","patch_set":6,"id":"1f621f24_9ea73850","line":593,"updated":"2020-11-04 23:04:26.000000000","message":"So if there\u0027s a lock timeout... the success stats go up?","commit_id":"8c0a1abf744a11b5c289239e3ac830786a9de4e9"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"82d789805d6b31134c4f3abf3c2343b4a634ec23","unresolved":false,"context_lines":[{"line_number":592,"context_line":"                                    for target_dev in job[\u0027nodes\u0027]])"},{"line_number":593,"context_line":"            stats.success +\u003d len(target_devs_info - failure_devs_info)"},{"line_number":594,"context_line":"            if not handoff_partition_deleted:"},{"line_number":595,"context_line":"                self.handoffs_remaining +\u003d 1"},{"line_number":596,"context_line":"            self.partition_times.append(time.time() - begin)"},{"line_number":597,"context_line":"            self.logger.timing_since(\u0027partition.delete.timing\u0027, begin)"},{"line_number":598,"context_line":""}],"source_content_type":"text/x-python","patch_set":6,"id":"1f621f24_9e7c58b6","line":595,"updated":"2020-11-04 23:04:26.000000000","message":"At least handoffs_remaining goes up, too...","commit_id":"8c0a1abf744a11b5c289239e3ac830786a9de4e9"}],"swift/obj/ssync_receiver.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"5241112dc7a069f409eb32952648e99b9df3d514","unresolved":false,"context_lines":[{"line_number":174,"context_line":"                    if self.app.replication_semaphore:"},{"line_number":175,"context_line":"                        self.app.replication_semaphore.release()"},{"line_number":176,"context_line":"            except (exceptions.ReplicationLockTimeout,"},{"line_number":177,"context_line":"                    exceptions.PartitionLockTimeout) as err:"},{"line_number":178,"context_line":"                self.app.logger.debug("},{"line_number":179,"context_line":"                    \u0027%s/%s/%s SSYNC LOCK TIMEOUT: %s\u0027 % ("},{"line_number":180,"context_line":"                        self.request.remote_addr, self.device, self.partition,"}],"source_content_type":"text/x-python","patch_set":4,"id":"9f560f44_8b9a946c","line":177,"updated":"2020-09-30 18:59:04.000000000","message":"maybe just LockTimeout?","commit_id":"7d1135a0c5051a8c1f1f22b61196b88e62d2f7e3"},{"author":{"_account_id":13852,"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","username":"rledisez"},"change_message_id":"dc750202503bfb60f05592d5290969b5d7fcdef7","unresolved":false,"context_lines":[{"line_number":174,"context_line":"                    if self.app.replication_semaphore:"},{"line_number":175,"context_line":"                        self.app.replication_semaphore.release()"},{"line_number":176,"context_line":"            except (exceptions.ReplicationLockTimeout,"},{"line_number":177,"context_line":"                    exceptions.PartitionLockTimeout) as err:"},{"line_number":178,"context_line":"                self.app.logger.debug("},{"line_number":179,"context_line":"                    \u0027%s/%s/%s SSYNC LOCK TIMEOUT: %s\u0027 % ("},{"line_number":180,"context_line":"                        self.request.remote_addr, self.device, self.partition,"}],"source_content_type":"text/x-python","patch_set":4,"id":"9f560f44_620470c6","line":177,"in_reply_to":"9f560f44_8b9a946c","updated":"2020-10-01 22:06:20.000000000","message":"Done","commit_id":"7d1135a0c5051a8c1f1f22b61196b88e62d2f7e3"}],"test/unit/obj/test_diskfile.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"5241112dc7a069f409eb32952648e99b9df3d514","unresolved":false,"context_lines":[{"line_number":1231,"context_line":"        success \u003d False"},{"line_number":1232,"context_line":"        with self.df_mgr.partition_lock(self.existing_device,"},{"line_number":1233,"context_line":"                                        POLICIES.legacy, \u00271\u0027, name\u003d\u0027foo\u0027):"},{"line_number":1234,"context_line":"            with self.assertRaises(PartitionLockTimeout):"},{"line_number":1235,"context_line":"                with self.df_mgr.partition_lock(self.existing_device,"},{"line_number":1236,"context_line":"                                                POLICIES.legacy, \u00271\u0027,"},{"line_number":1237,"context_line":"                                                name\u003d\u0027bar\u0027):"}],"source_content_type":"text/x-python","patch_set":4,"id":"9f560f44_eb99906d","line":1234,"updated":"2020-09-30 18:59:04.000000000","message":"i don\u0027t know how this ctx manager works...","commit_id":"7d1135a0c5051a8c1f1f22b61196b88e62d2f7e3"},{"author":{"_account_id":13852,"name":"Romain LE DISEZ","email":"romain.le-disez@corp.ovh.com","username":"rledisez"},"change_message_id":"dc750202503bfb60f05592d5290969b5d7fcdef7","unresolved":false,"context_lines":[{"line_number":1231,"context_line":"        success \u003d False"},{"line_number":1232,"context_line":"        with self.df_mgr.partition_lock(self.existing_device,"},{"line_number":1233,"context_line":"                                        POLICIES.legacy, \u00271\u0027, name\u003d\u0027foo\u0027):"},{"line_number":1234,"context_line":"            with self.assertRaises(PartitionLockTimeout):"},{"line_number":1235,"context_line":"                with self.df_mgr.partition_lock(self.existing_device,"},{"line_number":1236,"context_line":"                                                POLICIES.legacy, \u00271\u0027,"},{"line_number":1237,"context_line":"                                                name\u003d\u0027bar\u0027):"}],"source_content_type":"text/x-python","patch_set":4,"id":"9f560f44_a2e96880","line":1234,"in_reply_to":"9f560f44_eb99906d","updated":"2020-10-01 22:06:20.000000000","message":"Right. The test is wrong (and not passing by the way). Fixed.","commit_id":"7d1135a0c5051a8c1f1f22b61196b88e62d2f7e3"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"5241112dc7a069f409eb32952648e99b9df3d514","unresolved":false,"context_lines":[{"line_number":1236,"context_line":"                                                POLICIES.legacy, \u00271\u0027,"},{"line_number":1237,"context_line":"                                                name\u003d\u0027bar\u0027):"},{"line_number":1238,"context_line":"                    success \u003d True"},{"line_number":1239,"context_line":"        self.assertTrue(success)"},{"line_number":1240,"context_line":""},{"line_number":1241,"context_line":"    def test_partition_lock_and_replication_lock_same_partition(self):"},{"line_number":1242,"context_line":"        # Double check settings"}],"source_content_type":"text/x-python","patch_set":4,"id":"9f560f44_2b9b8864","line":1239,"updated":"2020-09-30 18:59:04.000000000","message":"if we\u0027re acctually getting into the lock?  (they *do* use different \"names\")","commit_id":"7d1135a0c5051a8c1f1f22b61196b88e62d2f7e3"}]}