)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"fa490b36aa0c0987dbe87c7bc917c3af9f78d817","unresolved":false,"context_lines":[{"line_number":25,"context_line":"based on the list. These changes are needed because current"},{"line_number":26,"context_line":"swift can not handle the existence check in object-level."},{"line_number":27,"context_line":""},{"line_number":28,"context_line":"Note that this feature is enabled only when ssync enabled."},{"line_number":29,"context_line":""},{"line_number":30,"context_line":"- kota, please squash and remove debug"},{"line_number":31,"context_line":""}],"source_content_type":"text/x-gerrit-commit-message","patch_set":15,"id":"da86d52c_73a2a24c","line":28,"updated":"2015-02-10 01:39:10.000000000","message":"(parts still work even with rsync though)","commit_id":"54ba45d862203dc622b753e6e29ffdd9bbf9aaf9"}],"etc/object-server.conf-sample":[{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"be86116c32ed24a6e47f9b4e862d891d1325aab8","unresolved":false,"context_lines":[{"line_number":208,"context_line":"# object-replicator push only one remote node in another region with expecting"},{"line_number":209,"context_line":"# asynchronous replication in the region. This option affects only when using"},{"line_number":210,"context_line":"# ssync."},{"line_number":211,"context_line":"# copy_once \u003d False"},{"line_number":212,"context_line":""},{"line_number":213,"context_line":"[object-updater]"},{"line_number":214,"context_line":"# You can override the default log routing for this app here (don\u0027t use set!):"}],"source_content_type":"application/octet-stream","patch_set":4,"id":"1ae5cdf2_c101fd05","line":211,"updated":"2014-06-24 03:09:39.000000000","message":"I think this sentence needs to be reworked a little, maybe something like:\n\ncopy_once is a flag for replication efficiency. When set to True, the object-replicator pushes to only one remote node in another region, and assumes that asynchronous replication will do the rest. This option is only available when ssync is used.","commit_id":"6a18587937e1fbad40933c4e058a3459648b56ff"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"4f92913ceef369190eb7d5a84fc121caa0039752","unresolved":false,"context_lines":[{"line_number":208,"context_line":"# object-replicator push only one remote node in another region with expecting"},{"line_number":209,"context_line":"# asynchronous replication in the region. This option affects only when using"},{"line_number":210,"context_line":"# ssync."},{"line_number":211,"context_line":"# copy_once \u003d False"},{"line_number":212,"context_line":""},{"line_number":213,"context_line":"[object-updater]"},{"line_number":214,"context_line":"# You can override the default log routing for this app here (don\u0027t use set!):"}],"source_content_type":"application/octet-stream","patch_set":4,"id":"1ae5cdf2_baa86b22","line":211,"in_reply_to":"1ae5cdf2_c101fd05","updated":"2014-06-25 01:57:24.000000000","message":"Great! I appreciate your polite review. I will change this document  according to your comment.","commit_id":"6a18587937e1fbad40933c4e058a3459648b56ff"}],"swift/obj/replicator.py":[{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"be86116c32ed24a6e47f9b4e862d891d1325aab8","unresolved":false,"context_lines":[{"line_number":95,"context_line":"            conf.get(\u0027handoff_delete\u0027, \u0027auto\u0027), 0)"},{"line_number":96,"context_line":"        self._diskfile_mgr \u003d DiskFileManager(conf, self.logger)"},{"line_number":97,"context_line":"        self.region_copy_once \u003d config_true_value("},{"line_number":98,"context_line":"            conf.get(\u0027region_copy_once\u0027, \u0027no\u0027))"},{"line_number":99,"context_line":""},{"line_number":100,"context_line":"    def sync(self, node, job, suffixes):  # Just exists for doc anchor point"},{"line_number":101,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":4,"id":"1ae5cdf2_01a845da","line":98,"updated":"2014-06-24 03:09:39.000000000","message":"In the config file you call the option copy_once and here you are using region_copy_once.","commit_id":"6a18587937e1fbad40933c4e058a3459648b56ff"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"4f92913ceef369190eb7d5a84fc121caa0039752","unresolved":false,"context_lines":[{"line_number":95,"context_line":"            conf.get(\u0027handoff_delete\u0027, \u0027auto\u0027), 0)"},{"line_number":96,"context_line":"        self._diskfile_mgr \u003d DiskFileManager(conf, self.logger)"},{"line_number":97,"context_line":"        self.region_copy_once \u003d config_true_value("},{"line_number":98,"context_line":"            conf.get(\u0027region_copy_once\u0027, \u0027no\u0027))"},{"line_number":99,"context_line":""},{"line_number":100,"context_line":"    def sync(self, node, job, suffixes):  # Just exists for doc anchor point"},{"line_number":101,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":4,"id":"1ae5cdf2_da4617c0","line":98,"in_reply_to":"1ae5cdf2_01a845da","updated":"2014-06-25 01:57:24.000000000","message":"Oh, I had a mistake in the config file. I will change the option name as \"region_copy_once\" in the config. Thanks!","commit_id":"6a18587937e1fbad40933c4e058a3459648b56ff"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"4f177d8139683d9d2df3bbf16efaa6e241065256","unresolved":false,"context_lines":[{"line_number":252,"context_line":"                            conn.getresponse().read()"},{"line_number":253,"context_line":"                        # add only remote region when replicate succeeded"},{"line_number":254,"context_line":"                        if copy_once and not cand_objs \\"},{"line_number":255,"context_line":"                                and self.sync_method \u003d\u003d \u0027ssync\u0027:"},{"line_number":256,"context_line":"                            synced_remote_regions.add(node[\u0027region\u0027])"},{"line_number":257,"context_line":"                        job[\u0027obj_list\u0027] \u003d delete_objs \u003d cand_objs"},{"line_number":258,"context_line":"                    responses.append(success)"}],"source_content_type":"text/x-python","patch_set":6,"id":"baada198_a28da637","line":255,"updated":"2014-07-31 06:34:20.000000000","message":"self.sync_method is a method not a string.. shouldn\u0027t this be:\n\n  self.sync_method.func_name \u003d\u003d \"ssync\"\n\nOr pull it out of the config (or store the value out of the config).","commit_id":"3228029c92e9e4b722fd96bed7b4140be0e1f436"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"cc08155f4c8b378c2487f84f060683a7a5b84770","unresolved":false,"context_lines":[{"line_number":252,"context_line":"                            conn.getresponse().read()"},{"line_number":253,"context_line":"                        # add only remote region when replicate succeeded"},{"line_number":254,"context_line":"                        if copy_once and not cand_objs \\"},{"line_number":255,"context_line":"                                and self.sync_method \u003d\u003d \u0027ssync\u0027:"},{"line_number":256,"context_line":"                            synced_remote_regions.add(node[\u0027region\u0027])"},{"line_number":257,"context_line":"                        job[\u0027obj_list\u0027] \u003d delete_objs \u003d cand_objs"},{"line_number":258,"context_line":"                    responses.append(success)"}],"source_content_type":"text/x-python","patch_set":6,"id":"baada198_0541d509","line":255,"in_reply_to":"baada198_a28da637","updated":"2014-08-01 03:58:41.000000000","message":"Right! It\u0027s my mistake. I will fix it.","commit_id":"3228029c92e9e4b722fd96bed7b4140be0e1f436"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"e05cff2d5d76fdb2b1e0ba262aea1c4723876988","unresolved":false,"context_lines":[{"line_number":195,"context_line":"                args.append(spath)"},{"line_number":196,"context_line":"                had_any \u003d True"},{"line_number":197,"context_line":"        if not had_any:"},{"line_number":198,"context_line":"            return False"},{"line_number":199,"context_line":"        data_dir \u003d get_data_dir(job[\u0027policy_idx\u0027])"},{"line_number":200,"context_line":"        args.append(join(rsync_module, node[\u0027device\u0027],"},{"line_number":201,"context_line":"                    data_dir, job[\u0027partition\u0027]))"}],"source_content_type":"text/x-python","patch_set":7,"id":"baa201ad_e895d040","line":198,"updated":"2014-10-01 15:07:18.000000000","message":"I think return False needs to return False, []. There is one more above too.","commit_id":"cf615115c80185ddd44068000c7ad5647eb4fc1c"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"f2edd44ac411ac80b5fc18e644547328ebe3e1c0","unresolved":false,"context_lines":[{"line_number":195,"context_line":"                args.append(spath)"},{"line_number":196,"context_line":"                had_any \u003d True"},{"line_number":197,"context_line":"        if not had_any:"},{"line_number":198,"context_line":"            return False"},{"line_number":199,"context_line":"        data_dir \u003d get_data_dir(job[\u0027policy_idx\u0027])"},{"line_number":200,"context_line":"        args.append(join(rsync_module, node[\u0027device\u0027],"},{"line_number":201,"context_line":"                    data_dir, job[\u0027partition\u0027]))"}],"source_content_type":"text/x-python","patch_set":7,"id":"baa201ad_7438d5a0","line":198,"in_reply_to":"baa201ad_e895d040","updated":"2014-10-02 02:27:14.000000000","message":"Right! Thanks!","commit_id":"cf615115c80185ddd44068000c7ad5647eb4fc1c"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"9eecb676d6e0ae7f36b5fefadf803f77c12ea549","unresolved":false,"context_lines":[{"line_number":253,"context_line":"                        # add only remote region when replicate succeeded"},{"line_number":254,"context_line":"                        if copy_once and not cand_objs \\"},{"line_number":255,"context_line":"                                and self.sync_method.__name__ \u003d\u003d \u0027ssync\u0027:"},{"line_number":256,"context_line":"                            synced_remote_regions.add(node[\u0027region\u0027])"},{"line_number":257,"context_line":"                        job[\u0027obj_list\u0027] \u003d delete_objs \u003d cand_objs"},{"line_number":258,"context_line":"                    responses.append(success)"},{"line_number":259,"context_line":"            if self.handoff_delete:"}],"source_content_type":"text/x-python","patch_set":7,"id":"baa201ad_a884a8d4","line":256,"updated":"2014-10-01 15:13:45.000000000","message":"Is it just me or is synced_remote_regions not used at all in update_deleted()? It is set, but never referenced in any conditions or computations. Variable with the same name is used productively in update(), but not here. If that is correct, we can delete the whole special-casing for ssync.","commit_id":"cf615115c80185ddd44068000c7ad5647eb4fc1c"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"f2edd44ac411ac80b5fc18e644547328ebe3e1c0","unresolved":false,"context_lines":[{"line_number":253,"context_line":"                        # add only remote region when replicate succeeded"},{"line_number":254,"context_line":"                        if copy_once and not cand_objs \\"},{"line_number":255,"context_line":"                                and self.sync_method.__name__ \u003d\u003d \u0027ssync\u0027:"},{"line_number":256,"context_line":"                            synced_remote_regions.add(node[\u0027region\u0027])"},{"line_number":257,"context_line":"                        job[\u0027obj_list\u0027] \u003d delete_objs \u003d cand_objs"},{"line_number":258,"context_line":"                    responses.append(success)"},{"line_number":259,"context_line":"            if self.handoff_delete:"}],"source_content_type":"text/x-python","patch_set":7,"id":"baa201ad_b43cfde3","line":256,"in_reply_to":"baa201ad_a884a8d4","updated":"2014-10-02 02:27:14.000000000","message":"Oh, yes. we might be able to rid of synced_remote_region because update_deleted() have to \"sync\" (even if the \"sync\" does only missing check) to whole primary node to make a decision to delete own local object.","commit_id":"cf615115c80185ddd44068000c7ad5647eb4fc1c"},{"author":{"_account_id":330,"name":"John Dickinson","email":"me@not.mn","username":"notmyname"},"change_message_id":"a1de9b09cd690db3b5f2ed3b2758a08138c67714","unresolved":false,"context_lines":[{"line_number":304,"context_line":"            attempts_left \u003d len(job[\u0027nodes\u0027])"},{"line_number":305,"context_line":"            synced_remote_regions \u003d set()"},{"line_number":306,"context_line":"            nodes \u003d itertools.chain("},{"line_number":307,"context_line":"                job[\u0027nodes\u0027],"},{"line_number":308,"context_line":"                job[\u0027object_ring\u0027].get_more_nodes(int(job[\u0027partition\u0027])))"},{"line_number":309,"context_line":"            while attempts_left \u003e 0:"},{"line_number":310,"context_line":"                # If this throws StopIterator it will be caught way below"}],"source_content_type":"text/x-python","patch_set":7,"id":"baa201ad_28e6f8b4","line":307,"updated":"2014-10-01 15:15:55.000000000","message":"shuffle this so that we don\u0027t only replicate to the same parts all the time","commit_id":"cf615115c80185ddd44068000c7ad5647eb4fc1c"},{"author":{"_account_id":330,"name":"John Dickinson","email":"me@not.mn","username":"notmyname"},"change_message_id":"a1de9b09cd690db3b5f2ed3b2758a08138c67714","unresolved":false,"context_lines":[{"line_number":310,"context_line":"                # If this throws StopIterator it will be caught way below"},{"line_number":311,"context_line":"                node \u003d next(nodes)"},{"line_number":312,"context_line":"                attempts_left -\u003d 1"},{"line_number":313,"context_line":"                copy_once \u003d self.region_copy_once and \\"},{"line_number":314,"context_line":"                    node[\u0027region\u0027] !\u003d job[\u0027local\u0027]"},{"line_number":315,"context_line":"                try:"},{"line_number":316,"context_line":"                    with Timeout(self.http_timeout):"}],"source_content_type":"text/x-python","patch_set":7,"id":"baa201ad_a53cd1e3","line":313,"updated":"2014-10-01 15:15:55.000000000","message":"get rid of config option","commit_id":"cf615115c80185ddd44068000c7ad5647eb4fc1c"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"f2edd44ac411ac80b5fc18e644547328ebe3e1c0","unresolved":false,"context_lines":[{"line_number":310,"context_line":"                # If this throws StopIterator it will be caught way below"},{"line_number":311,"context_line":"                node \u003d next(nodes)"},{"line_number":312,"context_line":"                attempts_left -\u003d 1"},{"line_number":313,"context_line":"                copy_once \u003d self.region_copy_once and \\"},{"line_number":314,"context_line":"                    node[\u0027region\u0027] !\u003d job[\u0027local\u0027]"},{"line_number":315,"context_line":"                try:"},{"line_number":316,"context_line":"                    with Timeout(self.http_timeout):"}],"source_content_type":"text/x-python","patch_set":7,"id":"baa201ad_1abdf368","line":313,"in_reply_to":"baa201ad_a53cd1e3","updated":"2014-10-02 02:27:14.000000000","message":"ok :)","commit_id":"cf615115c80185ddd44068000c7ad5647eb4fc1c"},{"author":{"_account_id":6968,"name":"Christian Schwede","email":"cschwede@nvidia.com","username":"cschwede"},"change_message_id":"1ecdd225d34ea3b7bf37cc518b3f43910080776d","unresolved":false,"context_lines":[{"line_number":311,"context_line":"                node \u003d next(nodes)"},{"line_number":312,"context_line":"                attempts_left -\u003d 1"},{"line_number":313,"context_line":"                copy_once \u003d self.region_copy_once and \\"},{"line_number":314,"context_line":"                    node[\u0027region\u0027] !\u003d job[\u0027local\u0027]"},{"line_number":315,"context_line":"                try:"},{"line_number":316,"context_line":"                    with Timeout(self.http_timeout):"},{"line_number":317,"context_line":"                        resp \u003d http_connect("}],"source_content_type":"text/x-python","patch_set":7,"id":"baa201ad_c5a9d5e0","line":314,"updated":"2014-10-01 14:26:37.000000000","message":"Could you rename the key from \u0027local\u0027 to \u0027region\u0027? This might be misleading I think.","commit_id":"cf615115c80185ddd44068000c7ad5647eb4fc1c"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"f2edd44ac411ac80b5fc18e644547328ebe3e1c0","unresolved":false,"context_lines":[{"line_number":311,"context_line":"                node \u003d next(nodes)"},{"line_number":312,"context_line":"                attempts_left -\u003d 1"},{"line_number":313,"context_line":"                copy_once \u003d self.region_copy_once and \\"},{"line_number":314,"context_line":"                    node[\u0027region\u0027] !\u003d job[\u0027local\u0027]"},{"line_number":315,"context_line":"                try:"},{"line_number":316,"context_line":"                    with Timeout(self.http_timeout):"},{"line_number":317,"context_line":"                        resp \u003d http_connect("}],"source_content_type":"text/x-python","patch_set":7,"id":"baa201ad_9ab0038d","line":314,"in_reply_to":"baa201ad_c5a9d5e0","updated":"2014-10-02 02:27:14.000000000","message":"+1 I\u0027ll do so.","commit_id":"cf615115c80185ddd44068000c7ad5647eb4fc1c"},{"author":{"_account_id":330,"name":"John Dickinson","email":"me@not.mn","username":"notmyname"},"change_message_id":"a1de9b09cd690db3b5f2ed3b2758a08138c67714","unresolved":false,"context_lines":[{"line_number":347,"context_line":"                                remote_hash.get(suffix, -1)]"},{"line_number":348,"context_line":"                    # only check the region of the node has been synced"},{"line_number":349,"context_line":"                    # or not because the local region must not be in"},{"line_number":350,"context_line":"                    # synced_remote_regions"},{"line_number":351,"context_line":"                    if node[\u0027region\u0027] in synced_remote_regions:"},{"line_number":352,"context_line":"                        continue"},{"line_number":353,"context_line":"                    success, _junk \u003d self.sync(node, job, suffixes)"}],"source_content_type":"text/x-python","patch_set":7,"id":"baa201ad_281ef863","line":350,"updated":"2014-10-01 15:15:55.000000000","message":"reword:\n\nif we have already synced to this remote region, don\u0027t sync again on this replication pass","commit_id":"cf615115c80185ddd44068000c7ad5647eb4fc1c"},{"author":{"_account_id":330,"name":"John Dickinson","email":"me@not.mn","username":"notmyname"},"change_message_id":"a1de9b09cd690db3b5f2ed3b2758a08138c67714","unresolved":false,"context_lines":[{"line_number":359,"context_line":"                            headers\u003dself.headers)"},{"line_number":360,"context_line":"                        conn.getresponse().read()"},{"line_number":361,"context_line":"                    # add only remote region when replicate succeeded"},{"line_number":362,"context_line":"                    if success and copy_once:"},{"line_number":363,"context_line":"                        synced_remote_regions.add(node[\u0027region\u0027])"},{"line_number":364,"context_line":"                    self.suffix_sync +\u003d len(suffixes)"},{"line_number":365,"context_line":"                    self.logger.update_stats(\u0027suffix.syncs\u0027, len(suffixes))"}],"source_content_type":"text/x-python","patch_set":7,"id":"baa201ad_c85bac2c","line":362,"updated":"2014-10-01 15:15:55.000000000","message":"instead of copy_once variable, replace with\n\n\"node[\u0027region\u0027] !\u003d job[\u0027local\u0027]\"\n\n(also, as Christian commented, change \"local\" to \"region\"","commit_id":"cf615115c80185ddd44068000c7ad5647eb4fc1c"},{"author":{"_account_id":330,"name":"John Dickinson","email":"me@not.mn","username":"notmyname"},"change_message_id":"a1de9b09cd690db3b5f2ed3b2758a08138c67714","unresolved":false,"context_lines":[{"line_number":360,"context_line":"                        conn.getresponse().read()"},{"line_number":361,"context_line":"                    # add only remote region when replicate succeeded"},{"line_number":362,"context_line":"                    if success and copy_once:"},{"line_number":363,"context_line":"                        synced_remote_regions.add(node[\u0027region\u0027])"},{"line_number":364,"context_line":"                    self.suffix_sync +\u003d len(suffixes)"},{"line_number":365,"context_line":"                    self.logger.update_stats(\u0027suffix.syncs\u0027, len(suffixes))"},{"line_number":366,"context_line":"                except (Exception, Timeout):"}],"source_content_type":"text/x-python","patch_set":7,"id":"baa201ad_454a45ad","line":363,"updated":"2014-10-01 15:15:55.000000000","message":"move this added block to above the try on line 315","commit_id":"cf615115c80185ddd44068000c7ad5647eb4fc1c"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"f2edd44ac411ac80b5fc18e644547328ebe3e1c0","unresolved":false,"context_lines":[{"line_number":360,"context_line":"                        conn.getresponse().read()"},{"line_number":361,"context_line":"                    # add only remote region when replicate succeeded"},{"line_number":362,"context_line":"                    if success and copy_once:"},{"line_number":363,"context_line":"                        synced_remote_regions.add(node[\u0027region\u0027])"},{"line_number":364,"context_line":"                    self.suffix_sync +\u003d len(suffixes)"},{"line_number":365,"context_line":"                    self.logger.update_stats(\u0027suffix.syncs\u0027, len(suffixes))"},{"line_number":366,"context_line":"                except (Exception, Timeout):"}],"source_content_type":"text/x-python","patch_set":7,"id":"baa201ad_d76339db","line":363,"in_reply_to":"baa201ad_454a45ad","updated":"2014-10-02 02:27:14.000000000","message":"Did you mean we should add the remote region even though the sync fails? We should not add it if the sync fails, right?\n\nI think we discussed the \"continue\" bloack starts at 351 in hackathon.","commit_id":"cf615115c80185ddd44068000c7ad5647eb4fc1c"},{"author":{"_account_id":6968,"name":"Christian Schwede","email":"cschwede@nvidia.com","username":"cschwede"},"change_message_id":"1ecdd225d34ea3b7bf37cc518b3f43910080776d","unresolved":false,"context_lines":[{"line_number":485,"context_line":"                             policy_idx\u003dpolicy.idx,"},{"line_number":486,"context_line":"                             partition\u003dpartition,"},{"line_number":487,"context_line":"                             object_ring\u003dobj_ring,"},{"line_number":488,"context_line":"                             local\u003dlocal_dev[\u0027region\u0027]))"},{"line_number":489,"context_line":"                except (ValueError, OSError):"},{"line_number":490,"context_line":"                    continue"},{"line_number":491,"context_line":""}],"source_content_type":"text/x-python","patch_set":7,"id":"baa201ad_6583c956","line":488,"updated":"2014-10-01 14:26:37.000000000","message":"Same as above, the key should be named region IMO.","commit_id":"cf615115c80185ddd44068000c7ad5647eb4fc1c"},{"author":{"_account_id":330,"name":"John Dickinson","email":"me@not.mn","username":"notmyname"},"change_message_id":"c47bc0deadaf65064ba415bcc6042d2291365c85","unresolved":false,"context_lines":[{"line_number":350,"context_line":"                    # if we have already synced to this remote region,"},{"line_number":351,"context_line":"                    # don\u0027t sync again on this replication pass"},{"line_number":352,"context_line":"                    if node[\u0027region\u0027] in synced_remote_regions:"},{"line_number":353,"context_line":"                        continue"},{"line_number":354,"context_line":"                    success, _junk \u003d self.sync(node, job, suffixes)"},{"line_number":355,"context_line":"                    with Timeout(self.http_timeout):"},{"line_number":356,"context_line":"                        conn \u003d http_connect("}],"source_content_type":"text/x-python","patch_set":8,"id":"baa201ad_eb7daee8","line":353,"updated":"2014-10-02 15:40:35.000000000","message":"this check needs to move up to line 316","commit_id":"53d0cf832ab9b861dc302b19b59d6af5c9bc850e"},{"author":{"_account_id":330,"name":"John Dickinson","email":"me@not.mn","username":"notmyname"},"change_message_id":"2ef91a7b9837378bf031d86a1980fe040e5b6ac4","unresolved":false,"context_lines":[{"line_number":239,"context_line":"                    if node[\u0027region\u0027] in synced_remote_regions:"},{"line_number":240,"context_line":"                        job[\u0027obj_list\u0027] \u003d \\"},{"line_number":241,"context_line":"                            synced_remote_regions[node[\u0027region\u0027]]"},{"line_number":242,"context_line":"                    success, cand_objs \u003d self.sync(node, job, suffixes)"},{"line_number":243,"context_line":"                    if success:"},{"line_number":244,"context_line":"                        with Timeout(self.http_timeout):"},{"line_number":245,"context_line":"                            conn \u003d http_connect("}],"source_content_type":"text/x-python","patch_set":12,"id":"3a961159_b19692ee","line":242,"updated":"2014-12-19 00:54:47.000000000","message":"cand_objects is the list of objects that local asked about minus the objects that the remote asked to send. ie the objects locally that are either older than the remote or just fine","commit_id":"19a1a18da8c8a82c410423ff9fbc3d94b2bdc979"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"b6927cb23697dc0902744d364a649c32c8d0704d","unresolved":false,"context_lines":[{"line_number":239,"context_line":"                    if node[\u0027region\u0027] in synced_remote_regions:"},{"line_number":240,"context_line":"                        job[\u0027obj_list\u0027] \u003d \\"},{"line_number":241,"context_line":"                            synced_remote_regions[node[\u0027region\u0027]]"},{"line_number":242,"context_line":"                    success, cand_objs \u003d self.sync(node, job, suffixes)"},{"line_number":243,"context_line":"                    if success:"},{"line_number":244,"context_line":"                        with Timeout(self.http_timeout):"},{"line_number":245,"context_line":"                            conn \u003d http_connect("}],"source_content_type":"text/x-python","patch_set":12,"id":"3a961159_2e7a3977","line":242,"in_reply_to":"3a961159_b19692ee","updated":"2014-12-19 09:54:29.000000000","message":"Yes it is. cand_objects is a list of candidate objects to delete at the handloff node.","commit_id":"19a1a18da8c8a82c410423ff9fbc3d94b2bdc979"},{"author":{"_account_id":330,"name":"John Dickinson","email":"me@not.mn","username":"notmyname"},"change_message_id":"2ef91a7b9837378bf031d86a1980fe040e5b6ac4","unresolved":false,"context_lines":[{"line_number":250,"context_line":"                            conn.getresponse().read()"},{"line_number":251,"context_line":"                        # add only remote region when replicate succeeded"},{"line_number":252,"context_line":"                        if node[\u0027region\u0027] !\u003d job[\u0027region\u0027] and cand_objs:"},{"line_number":253,"context_line":"                            synced_remote_regions[node[\u0027region\u0027]] \u003d cand_objs"},{"line_number":254,"context_line":"                    responses.append(success)"},{"line_number":255,"context_line":"                    if \u0027obj_list\u0027 in job:"},{"line_number":256,"context_line":"                        del job[\u0027obj_list\u0027]"}],"source_content_type":"text/x-python","patch_set":12,"id":"3a961159_51000e34","line":253,"updated":"2014-12-19 00:54:47.000000000","message":"ones that are successfully synced. as in didn\u0027t need to be sent because either local is older or because they are the same.","commit_id":"19a1a18da8c8a82c410423ff9fbc3d94b2bdc979"},{"author":{"_account_id":330,"name":"John Dickinson","email":"me@not.mn","username":"notmyname"},"change_message_id":"2ef91a7b9837378bf031d86a1980fe040e5b6ac4","unresolved":false,"context_lines":[{"line_number":256,"context_line":"                        del job[\u0027obj_list\u0027]"},{"line_number":257,"context_line":"                for region, cand_objs in synced_remote_regions.iteritems():"},{"line_number":258,"context_line":"                    if delete_objs is None:"},{"line_number":259,"context_line":"                        delete_objs \u003d cand_objs"},{"line_number":260,"context_line":"                    else:"},{"line_number":261,"context_line":"                        delete_objs \u003d list(set(delete_objs) \u0026 set(cand_objs))"},{"line_number":262,"context_line":"            if self.handoff_delete:"}],"source_content_type":"text/x-python","patch_set":12,"id":"3a961159_b14bf285","line":259,"updated":"2014-12-19 00:54:47.000000000","message":"stuff that\u0027s either locally older or the same as the remote\n\nie doesn\u0027t need to be sent","commit_id":"19a1a18da8c8a82c410423ff9fbc3d94b2bdc979"},{"author":{"_account_id":330,"name":"John Dickinson","email":"me@not.mn","username":"notmyname"},"change_message_id":"2ef91a7b9837378bf031d86a1980fe040e5b6ac4","unresolved":false,"context_lines":[{"line_number":258,"context_line":"                    if delete_objs is None:"},{"line_number":259,"context_line":"                        delete_objs \u003d cand_objs"},{"line_number":260,"context_line":"                    else:"},{"line_number":261,"context_line":"                        delete_objs \u003d list(set(delete_objs) \u0026 set(cand_objs))"},{"line_number":262,"context_line":"            if self.handoff_delete:"},{"line_number":263,"context_line":"                # delete handoff if we have had handoff_delete successes"},{"line_number":264,"context_line":"                delete_handoff \u003d len([resp for resp in responses if resp]) \u003e\u003d \\"}],"source_content_type":"text/x-python","patch_set":12,"id":"3a961159_310d423f","line":261,"updated":"2014-12-19 00:54:47.000000000","message":"the cand_objs that are common across all remote regions\n\nie the object locally that every remote region has not asked for","commit_id":"19a1a18da8c8a82c410423ff9fbc3d94b2bdc979"},{"author":{"_account_id":330,"name":"John Dickinson","email":"me@not.mn","username":"notmyname"},"change_message_id":"2ef91a7b9837378bf031d86a1980fe040e5b6ac4","unresolved":false,"context_lines":[{"line_number":270,"context_line":"            if not suffixes or delete_handoff:"},{"line_number":271,"context_line":"                if delete_objs:"},{"line_number":272,"context_line":"                    delete_objs \u003d map("},{"line_number":273,"context_line":"                        lambda obj: join(job[\u0027path\u0027], obj[-3:]), delete_objs)"},{"line_number":274,"context_line":"                    self.logger.info(_(\"Removing partition suffixes: %s\"),"},{"line_number":275,"context_line":"                                     \u0027,\u0027.join(delete_objs))"},{"line_number":276,"context_line":"                else:"}],"source_content_type":"text/x-python","patch_set":12,"id":"3a961159_c34ef56e","line":273,"updated":"2014-12-19 00:54:47.000000000","message":"delete_objs \u003d [join(job[\u0027path\u0027], obj[-3:]) for obj in delete_objs]","commit_id":"19a1a18da8c8a82c410423ff9fbc3d94b2bdc979"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"b6927cb23697dc0902744d364a649c32c8d0704d","unresolved":false,"context_lines":[{"line_number":270,"context_line":"            if not suffixes or delete_handoff:"},{"line_number":271,"context_line":"                if delete_objs:"},{"line_number":272,"context_line":"                    delete_objs \u003d map("},{"line_number":273,"context_line":"                        lambda obj: join(job[\u0027path\u0027], obj[-3:]), delete_objs)"},{"line_number":274,"context_line":"                    self.logger.info(_(\"Removing partition suffixes: %s\"),"},{"line_number":275,"context_line":"                                     \u0027,\u0027.join(delete_objs))"},{"line_number":276,"context_line":"                else:"}],"source_content_type":"text/x-python","patch_set":12,"id":"3a961159_ceeb75da","line":273,"in_reply_to":"3a961159_c34ef56e","updated":"2014-12-19 09:54:29.000000000","message":"ya, it looks better for mine. I\u0027ll fix it.","commit_id":"19a1a18da8c8a82c410423ff9fbc3d94b2bdc979"},{"author":{"_account_id":330,"name":"John Dickinson","email":"me@not.mn","username":"notmyname"},"change_message_id":"2ef91a7b9837378bf031d86a1980fe040e5b6ac4","unresolved":false,"context_lines":[{"line_number":272,"context_line":"                    delete_objs \u003d map("},{"line_number":273,"context_line":"                        lambda obj: join(job[\u0027path\u0027], obj[-3:]), delete_objs)"},{"line_number":274,"context_line":"                    self.logger.info(_(\"Removing partition suffixes: %s\"),"},{"line_number":275,"context_line":"                                     \u0027,\u0027.join(delete_objs))"},{"line_number":276,"context_line":"                else:"},{"line_number":277,"context_line":"                    delete_objs \u003d [job[\u0027path\u0027]]"},{"line_number":278,"context_line":"                    self.logger.info(_(\"Removing partition: %s\"), job[\u0027path\u0027])"}],"source_content_type":"text/x-python","patch_set":12,"id":"3a961159_03a39d00","line":275,"updated":"2014-12-19 00:54:47.000000000","message":"I don\u0027t yet get why this specific case for suffix dirs is needed. maybe because we\u0027ll only delete some suffix dirs each run?","commit_id":"19a1a18da8c8a82c410423ff9fbc3d94b2bdc979"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"97811804539207911ee8e97f70eb507af083c469","unresolved":false,"context_lines":[{"line_number":272,"context_line":"                    delete_objs \u003d map("},{"line_number":273,"context_line":"                        lambda obj: join(job[\u0027path\u0027], obj[-3:]), delete_objs)"},{"line_number":274,"context_line":"                    self.logger.info(_(\"Removing partition suffixes: %s\"),"},{"line_number":275,"context_line":"                                     \u0027,\u0027.join(delete_objs))"},{"line_number":276,"context_line":"                else:"},{"line_number":277,"context_line":"                    delete_objs \u003d [job[\u0027path\u0027]]"},{"line_number":278,"context_line":"                    self.logger.info(_(\"Removing partition: %s\"), job[\u0027path\u0027])"}],"source_content_type":"text/x-python","patch_set":12,"id":"3a961159_7cb6be3f","line":275,"in_reply_to":"3a961159_03a39d00","updated":"2014-12-19 03:43:50.000000000","message":"Actually... If the remote is not intrested in the objects because they are okay, they end in the returned \"cand_list\", don\u0027t they? If yes, they get deleted, don\u0027t they? But that can\u0027t be right.","commit_id":"19a1a18da8c8a82c410423ff9fbc3d94b2bdc979"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"b6927cb23697dc0902744d364a649c32c8d0704d","unresolved":false,"context_lines":[{"line_number":272,"context_line":"                    delete_objs \u003d map("},{"line_number":273,"context_line":"                        lambda obj: join(job[\u0027path\u0027], obj[-3:]), delete_objs)"},{"line_number":274,"context_line":"                    self.logger.info(_(\"Removing partition suffixes: %s\"),"},{"line_number":275,"context_line":"                                     \u0027,\u0027.join(delete_objs))"},{"line_number":276,"context_line":"                else:"},{"line_number":277,"context_line":"                    delete_objs \u003d [job[\u0027path\u0027]]"},{"line_number":278,"context_line":"                    self.logger.info(_(\"Removing partition: %s\"), job[\u0027path\u0027])"}],"source_content_type":"text/x-python","patch_set":12,"id":"3a961159_117bdc8c","line":275,"in_reply_to":"3a961159_7cb6be3f","updated":"2014-12-19 09:54:29.000000000","message":"@John\n\nThere might be objects which is not okay in the partition. delete_objs shows only objects to be deleted.\n\nThis suffix dirs are care about the case that swift has a lot of handoffs (e.g. using write affinity!) In that case, handoff objects  will be created continuously. If a replicator deletes the pertition only after whole objects are synced successfully in the pertition, they will never deleted until incoming requests of PUT stopped.\n\n\n@Pete\n\nThe remote will return \"cand_list\" shows objects which are okay. And then, update_deleted deletes only them.","commit_id":"19a1a18da8c8a82c410423ff9fbc3d94b2bdc979"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"c04346c924e801984967dc1d17e13e718c18d674","unresolved":false,"context_lines":[{"line_number":196,"context_line":"        data_dir \u003d get_data_dir(job[\u0027policy_idx\u0027])"},{"line_number":197,"context_line":"        args.append(join(rsync_module, node[\u0027device\u0027],"},{"line_number":198,"context_line":"                    data_dir, job[\u0027partition\u0027]))"},{"line_number":199,"context_line":"        return self._rsync(args) \u003d\u003d 0, []"},{"line_number":200,"context_line":""},{"line_number":201,"context_line":"    def ssync(self, node, job, suffixes):"},{"line_number":202,"context_line":"        return ssync_sender.Sender(self, node, job, suffixes)()"}],"source_content_type":"text/x-python","patch_set":14,"id":"da86d52c_24d627c4","line":199,"updated":"2015-02-09 23:40:47.000000000","message":"Is this expected maybe to not work with rsync?","commit_id":"d52d661628fc92b14e8d8322d5959687384f8037"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"c04346c924e801984967dc1d17e13e718c18d674","unresolved":false,"context_lines":[{"line_number":238,"context_line":"                for node in job[\u0027nodes\u0027]:"},{"line_number":239,"context_line":"                    if node[\u0027region\u0027] in synced_remote_regions:"},{"line_number":240,"context_line":"                        job[\u0027obj_list\u0027] \u003d \\"},{"line_number":241,"context_line":"                            synced_remote_regions[node[\u0027region\u0027]]"},{"line_number":242,"context_line":"                    success, cand_objs \u003d self.sync(node, job, suffixes)"},{"line_number":243,"context_line":"                    if success:"},{"line_number":244,"context_line":"                        with Timeout(self.http_timeout):"}],"source_content_type":"text/x-python","patch_set":14,"id":"da86d52c_3fd39160","line":241,"updated":"2015-02-09 23:40:47.000000000","message":"so if we\u0027ve already synced to a remote region once, we add the obj_list key to the job so that it won\u0027t send remote objects?\n\nIt seems like the idea is to make sure enough nodes have the parts that we can delete our local copies...","commit_id":"d52d661628fc92b14e8d8322d5959687384f8037"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"083ab02578d1e500aaa8f44da92f949182aa58d5","unresolved":false,"context_lines":[{"line_number":238,"context_line":"                for node in job[\u0027nodes\u0027]:"},{"line_number":239,"context_line":"                    if node[\u0027region\u0027] in synced_remote_regions:"},{"line_number":240,"context_line":"                        job[\u0027obj_list\u0027] \u003d \\"},{"line_number":241,"context_line":"                            synced_remote_regions[node[\u0027region\u0027]]"},{"line_number":242,"context_line":"                    success, cand_objs \u003d self.sync(node, job, suffixes)"},{"line_number":243,"context_line":"                    if success:"},{"line_number":244,"context_line":"                        with Timeout(self.http_timeout):"}],"source_content_type":"text/x-python","patch_set":14,"id":"da86d52c_2e927d4b","line":241,"in_reply_to":"da86d52c_3fd39160","updated":"2015-02-10 02:32:49.000000000","message":"I think this is all true, if obj_list was renamed to \"remote_check_objs\" it may have made it more immediately obvious to me, but I\u0027m sorta slow.","commit_id":"d52d661628fc92b14e8d8322d5959687384f8037"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"c04346c924e801984967dc1d17e13e718c18d674","unresolved":false,"context_lines":[{"line_number":249,"context_line":"                                \u0027/\u0027 + \u0027-\u0027.join(suffixes), headers\u003dself.headers)"},{"line_number":250,"context_line":"                            conn.getresponse().read()"},{"line_number":251,"context_line":"                        # add only remote region when replicate succeeded"},{"line_number":252,"context_line":"                        if node[\u0027region\u0027] !\u003d job[\u0027region\u0027] and cand_objs:"},{"line_number":253,"context_line":"                            synced_remote_regions[node[\u0027region\u0027]] \u003d cand_objs"},{"line_number":254,"context_line":"                    responses.append(success)"},{"line_number":255,"context_line":"                    if \u0027obj_list\u0027 in job:"}],"source_content_type":"text/x-python","patch_set":14,"id":"da86d52c_5f41250a","line":252,"updated":"2015-02-09 23:40:47.000000000","message":"is cand_objs always non-empty on success?","commit_id":"d52d661628fc92b14e8d8322d5959687384f8037"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"f9b741b1c444a0f37209d874bbfcd467fbbda936","unresolved":false,"context_lines":[{"line_number":249,"context_line":"                                \u0027/\u0027 + \u0027-\u0027.join(suffixes), headers\u003dself.headers)"},{"line_number":250,"context_line":"                            conn.getresponse().read()"},{"line_number":251,"context_line":"                        # add only remote region when replicate succeeded"},{"line_number":252,"context_line":"                        if node[\u0027region\u0027] !\u003d job[\u0027region\u0027] and cand_objs:"},{"line_number":253,"context_line":"                            synced_remote_regions[node[\u0027region\u0027]] \u003d cand_objs"},{"line_number":254,"context_line":"                    responses.append(success)"},{"line_number":255,"context_line":"                    if \u0027obj_list\u0027 in job:"}],"source_content_type":"text/x-python","patch_set":14,"id":"da86d52c_0e8f7924","line":252,"in_reply_to":"da86d52c_5f41250a","updated":"2015-02-10 02:07:51.000000000","message":"No (rsync returns empty on success)","commit_id":"d52d661628fc92b14e8d8322d5959687384f8037"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"083ab02578d1e500aaa8f44da92f949182aa58d5","unresolved":false,"context_lines":[{"line_number":249,"context_line":"                                \u0027/\u0027 + \u0027-\u0027.join(suffixes), headers\u003dself.headers)"},{"line_number":250,"context_line":"                            conn.getresponse().read()"},{"line_number":251,"context_line":"                        # add only remote region when replicate succeeded"},{"line_number":252,"context_line":"                        if node[\u0027region\u0027] !\u003d job[\u0027region\u0027] and cand_objs:"},{"line_number":253,"context_line":"                            synced_remote_regions[node[\u0027region\u0027]] \u003d cand_objs"},{"line_number":254,"context_line":"                    responses.append(success)"},{"line_number":255,"context_line":"                    if \u0027obj_list\u0027 in job:"}],"source_content_type":"text/x-python","patch_set":14,"id":"da86d52c_cec72163","line":252,"in_reply_to":"da86d52c_5f41250a","updated":"2015-02-10 02:32:49.000000000","message":"no, because rsync can be success and return empty.  But all that means is that we don\u0027t update synced_remote_regions, not that we failed.  I think i\u0027ll have trouble getting this lodged into my head.  Maybe the comment should just away, and it should say \"if the sync engine returned delete candidates we update our list of delete candidates for the remote region\"","commit_id":"d52d661628fc92b14e8d8322d5959687384f8037"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"c04346c924e801984967dc1d17e13e718c18d674","unresolved":false,"context_lines":[{"line_number":258,"context_line":"                    if delete_objs is None:"},{"line_number":259,"context_line":"                        delete_objs \u003d cand_objs"},{"line_number":260,"context_line":"                    else:"},{"line_number":261,"context_line":"                        delete_objs \u003d list(set(delete_objs) \u0026 set(cand_objs))"},{"line_number":262,"context_line":"            if self.handoff_delete:"},{"line_number":263,"context_line":"                # delete handoff if we have had handoff_delete successes"},{"line_number":264,"context_line":"                delete_handoff \u003d len([resp for resp in responses if resp]) \u003e\u003d \\"}],"source_content_type":"text/x-python","patch_set":14,"id":"da86d52c_9f1f1d09","line":261,"updated":"2015-02-09 23:40:47.000000000","message":"does delete_objs need to be ordered?\n\nhow about just initialize delete_objs to set(), and then always:\n\n    delete_objs.intersection(cand_objs)","commit_id":"d52d661628fc92b14e8d8322d5959687384f8037"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"083ab02578d1e500aaa8f44da92f949182aa58d5","unresolved":false,"context_lines":[{"line_number":258,"context_line":"                    if delete_objs is None:"},{"line_number":259,"context_line":"                        delete_objs \u003d cand_objs"},{"line_number":260,"context_line":"                    else:"},{"line_number":261,"context_line":"                        delete_objs \u003d list(set(delete_objs) \u0026 set(cand_objs))"},{"line_number":262,"context_line":"            if self.handoff_delete:"},{"line_number":263,"context_line":"                # delete handoff if we have had handoff_delete successes"},{"line_number":264,"context_line":"                delete_handoff \u003d len([resp for resp in responses if resp]) \u003e\u003d \\"}],"source_content_type":"text/x-python","patch_set":14,"id":"da86d52c_4efed16c","line":261,"in_reply_to":"da86d52c_9f1f1d09","updated":"2015-02-10 02:32:49.000000000","message":"yes, this would probably be better - need to avoid casting to sets and then lists and then sets and then lists.\n\n    delete_objs \u003d delete_objs.intersection(cand_objs)","commit_id":"d52d661628fc92b14e8d8322d5959687384f8037"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"083ab02578d1e500aaa8f44da92f949182aa58d5","unresolved":false,"context_lines":[{"line_number":272,"context_line":"                    delete_objs \u003d ["},{"line_number":273,"context_line":"                        join(job[\u0027path\u0027], obj[-3:]) for obj in delete_objs]"},{"line_number":274,"context_line":"                    self.logger.info(_(\"Removing partition suffixes: %s\"),"},{"line_number":275,"context_line":"                                     \u0027,\u0027.join(delete_objs))"},{"line_number":276,"context_line":"                else:"},{"line_number":277,"context_line":"                    delete_objs \u003d [job[\u0027path\u0027]]"},{"line_number":278,"context_line":"                    self.logger.info(_(\"Removing partition: %s\"), job[\u0027path\u0027])"}],"source_content_type":"text/x-python","patch_set":14,"id":"da86d52c_8e7609d7","line":275,"updated":"2015-02-10 02:32:49.000000000","message":"instead of rmtree on each of the suffixes, just delete the hashdir maybe?  Either way, if delete_objs has multiple hashes with the same suffix this will call rmtree on the same suffix multiple times - or worse if there is another obj in the suffix that isn\u0027t in the delete_obj list it could be deleted.\n\nPete: \"This is wrong\"","commit_id":"d52d661628fc92b14e8d8322d5959687384f8037"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"c04346c924e801984967dc1d17e13e718c18d674","unresolved":false,"context_lines":[{"line_number":276,"context_line":"                else:"},{"line_number":277,"context_line":"                    delete_objs \u003d [job[\u0027path\u0027]]"},{"line_number":278,"context_line":"                    self.logger.info(_(\"Removing partition: %s\"), job[\u0027path\u0027])"},{"line_number":279,"context_line":"                self.delete_handoff_paths(delete_objs)"},{"line_number":280,"context_line":"        except (Exception, Timeout):"},{"line_number":281,"context_line":"            self.logger.exception(_(\"Error syncing handoff partition\"))"},{"line_number":282,"context_line":"        finally:"}],"source_content_type":"text/x-python","patch_set":14,"id":"da86d52c_a4bf775f","line":279,"updated":"2015-02-09 23:40:47.000000000","message":"so we either delete the whole partition, or we delete a subset of suffixes which have successfully synced?","commit_id":"d52d661628fc92b14e8d8322d5959687384f8037"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"c04346c924e801984967dc1d17e13e718c18d674","unresolved":false,"context_lines":[{"line_number":317,"context_line":"                # if we have already synced to this remote region,"},{"line_number":318,"context_line":"                # don\u0027t sync again on this replication pass"},{"line_number":319,"context_line":"                if node[\u0027region\u0027] in synced_remote_regions:"},{"line_number":320,"context_line":"                    continue"},{"line_number":321,"context_line":"                try:"},{"line_number":322,"context_line":"                    with Timeout(self.http_timeout):"},{"line_number":323,"context_line":"                        resp \u003d http_connect("}],"source_content_type":"text/x-python","patch_set":14,"id":"da86d52c_ff29f99b","line":320,"updated":"2015-02-09 23:40:47.000000000","message":"this seems simple enough, only talk to a remote region once per replication pass.\n\nIs the nodes list stable?  I feel like if I have two replicas in r1 and one in r2 - the r2 node is always going to talk to the same node in r1.  Also if I had three replicas and three regions every node still talks to everyone...","commit_id":"d52d661628fc92b14e8d8322d5959687384f8037"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"c04346c924e801984967dc1d17e13e718c18d674","unresolved":false,"context_lines":[{"line_number":351,"context_line":"                    suffixes \u003d [suffix for suffix in local_hash if"},{"line_number":352,"context_line":"                                local_hash[suffix] !\u003d"},{"line_number":353,"context_line":"                                remote_hash.get(suffix, -1)]"},{"line_number":354,"context_line":"                    success, _junk \u003d self.sync(node, job, suffixes)"},{"line_number":355,"context_line":"                    with Timeout(self.http_timeout):"},{"line_number":356,"context_line":"                        conn \u003d http_connect("},{"line_number":357,"context_line":"                            node[\u0027replication_ip\u0027], node[\u0027replication_port\u0027],"}],"source_content_type":"text/x-python","patch_set":14,"id":"da86d52c_9fa2bd98","line":354,"updated":"2015-02-09 23:40:47.000000000","message":"so we only care about cand_objs in update_deleted?  Is _junk always empty here because we never send in obj_list?","commit_id":"d52d661628fc92b14e8d8322d5959687384f8037"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"083ab02578d1e500aaa8f44da92f949182aa58d5","unresolved":false,"context_lines":[{"line_number":351,"context_line":"                    suffixes \u003d [suffix for suffix in local_hash if"},{"line_number":352,"context_line":"                                local_hash[suffix] !\u003d"},{"line_number":353,"context_line":"                                remote_hash.get(suffix, -1)]"},{"line_number":354,"context_line":"                    success, _junk \u003d self.sync(node, job, suffixes)"},{"line_number":355,"context_line":"                    with Timeout(self.http_timeout):"},{"line_number":356,"context_line":"                        conn \u003d http_connect("},{"line_number":357,"context_line":"                            node[\u0027replication_ip\u0027], node[\u0027replication_port\u0027],"}],"source_content_type":"text/x-python","patch_set":14,"id":"da86d52c_b94eb9c0","line":354,"in_reply_to":"da86d52c_9fa2bd98","updated":"2015-02-10 02:32:49.000000000","message":"so we *do* only care about the delete candidates in update_deleted because we only delete in update_deleted.  But _junk is *not* always empty - in the next patch a sync job will return the full available list as _junk - some amount of wasted effort because sync doesn\u0027t have a understanding of when it\u0027s a sync job and a poke job.  The point is this is mostly fine just like it is.","commit_id":"d52d661628fc92b14e8d8322d5959687384f8037"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"c04346c924e801984967dc1d17e13e718c18d674","unresolved":false,"context_lines":[{"line_number":361,"context_line":"                        conn.getresponse().read()"},{"line_number":362,"context_line":"                    # add only remote region when replicate succeeded"},{"line_number":363,"context_line":"                    if success and node[\u0027region\u0027] !\u003d job[\u0027region\u0027]:"},{"line_number":364,"context_line":"                        synced_remote_regions.add(node[\u0027region\u0027])"},{"line_number":365,"context_line":"                    self.suffix_sync +\u003d len(suffixes)"},{"line_number":366,"context_line":"                    self.logger.update_stats(\u0027suffix.syncs\u0027, len(suffixes))"},{"line_number":367,"context_line":"                except (Exception, Timeout):"}],"source_content_type":"text/x-python","patch_set":14,"id":"da86d52c_ff99f979","line":364,"updated":"2015-02-09 23:40:47.000000000","message":"this part effects rsync too","commit_id":"d52d661628fc92b14e8d8322d5959687384f8037"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"d914fdcc6251e966b72fe0b236002264d42cfc7d","unresolved":false,"context_lines":[{"line_number":255,"context_line":"                                \u0027/\u0027 + \u0027-\u0027.join(suffixes), headers\u003dself.headers)"},{"line_number":256,"context_line":"                            conn.getresponse().read()"},{"line_number":257,"context_line":"                        # add only remote region when replicate succeeded"},{"line_number":258,"context_line":"                        if node[\u0027region\u0027] !\u003d job[\u0027region\u0027] and cand_objs:"},{"line_number":259,"context_line":"                            synced_remote_regions[node[\u0027region\u0027]] \u003d cand_objs"},{"line_number":260,"context_line":"                    responses.append(success)"},{"line_number":261,"context_line":"                for region, cand_objs in synced_remote_regions.iteritems():"}],"source_content_type":"text/x-python","patch_set":16,"id":"da86d52c_3df370a0","line":258,"updated":"2015-02-10 19:16:43.000000000","message":"I still struggle to reconcile this comment with this conditional.  The conditional does not evaluate \"success\", and success does not imply a non-empty cand_objs.  w/o the comment the conditional just reads \"when remote region and delete candidates, keep track of delete candidates in synced remote regions\" which I don\u0027t find particularly dubious - I would suggest the comment is removed.","commit_id":"228d5407943bec9c5d3a204693fa421456b2d0d5"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"d914fdcc6251e966b72fe0b236002264d42cfc7d","unresolved":false,"context_lines":[{"line_number":277,"context_line":"                                     \u0027,\u0027.join(delete_objs))"},{"line_number":278,"context_line":"                    delete_objs \u003d ["},{"line_number":279,"context_line":"                        (join(job[\u0027path\u0027], object_hash[-3:]),"},{"line_number":280,"context_line":"                         join(job[\u0027path\u0027], object_hash[-3:], object_hash))"},{"line_number":281,"context_line":"                        for object_hash in delete_objs]"},{"line_number":282,"context_line":"                    self.delete_handoff_paths(delete_objs)"},{"line_number":283,"context_line":"                else:"}],"source_content_type":"text/x-python","patch_set":16,"id":"da86d52c_9dbc5c6e","line":280,"updated":"2015-02-10 19:16:43.000000000","message":"can you use common.utils.storage_directory?  I think it\u0027d be easiest to just add obj_path to the job dict next to the \u0027path\u0027 and \u0027partition\u0027 keys.","commit_id":"228d5407943bec9c5d3a204693fa421456b2d0d5"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"d914fdcc6251e966b72fe0b236002264d42cfc7d","unresolved":false,"context_lines":[{"line_number":283,"context_line":"                else:"},{"line_number":284,"context_line":"                    tpool.execute("},{"line_number":285,"context_line":"                        shutil.rmtree, job[\u0027path\u0027], ignore_errors\u003dTrue)"},{"line_number":286,"context_line":"                    self.logger.info(_(\"Removing partition: %s\"), job[\u0027path\u0027])"},{"line_number":287,"context_line":"        except (Exception, Timeout):"},{"line_number":288,"context_line":"            self.logger.exception(_(\"Error syncing handoff partition\"))"},{"line_number":289,"context_line":"        finally:"}],"source_content_type":"text/x-python","patch_set":16,"id":"da86d52c_5d81048e","line":286,"updated":"2015-02-10 19:16:43.000000000","message":"s/removing/removed/ or put them back in the original order.","commit_id":"228d5407943bec9c5d3a204693fa421456b2d0d5"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"588c5a2b0fbfc66de06d7912598ed0192f83e972","unresolved":false,"context_lines":[{"line_number":294,"context_line":"        for suffix_dir, object_path in paths:"},{"line_number":295,"context_line":"            tpool.execute(shutil.rmtree, object_path, ignore_errors\u003dTrue)"},{"line_number":296,"context_line":"            try:"},{"line_number":297,"context_line":"                os.rmdir(suffix_dir)"},{"line_number":298,"context_line":"            except OSError:"},{"line_number":299,"context_line":"                pass"},{"line_number":300,"context_line":""}],"source_content_type":"text/x-python","patch_set":16,"id":"da86d52c_9db61cdb","line":297,"updated":"2015-02-10 18:20:57.000000000","message":"This is not sufficient for me to withhold +2, but don\u0027t do this. You are racing someone who\u0027s creating objects. Just leave suffix directories around. Someone doing lock_parent_directory will eventially collect them (I forgot who does it, I think I saw code for it). Here, I suspect, you\u0027re increasing a probability that somebody\u0027s PUT fails, although by a very small amount.","commit_id":"228d5407943bec9c5d3a204693fa421456b2d0d5"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"e66cae8ee592b38acc2de7a128e4ce53206a7c80","unresolved":false,"context_lines":[{"line_number":275,"context_line":"            if not suffixes or delete_handoff:"},{"line_number":276,"context_line":"                if delete_objs:"},{"line_number":277,"context_line":"                    self.logger.info(_(\"Removing objecs: %s\"),"},{"line_number":278,"context_line":"                                     \u0027,\u0027.join(delete_objs))"},{"line_number":279,"context_line":"                    delete_objs \u003d ["},{"line_number":280,"context_line":"                        (join(job[\u0027path\u0027], object_hash[-3:]),"},{"line_number":281,"context_line":"                         storage_directory(job[\u0027obj_path\u0027],"}],"source_content_type":"text/x-python","patch_set":17,"id":"da86d52c_09cf1780","line":278,"updated":"2015-02-11 19:52:21.000000000","message":"I think this is a bit too heavy for info level - needs to be either removing %d len(delete_objs) or move to a debug log line - probably just the count at info is better... dunno.","commit_id":"f038febfeea837e04ef6ed745eddd12809970ce3"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"e66cae8ee592b38acc2de7a128e4ce53206a7c80","unresolved":false,"context_lines":[{"line_number":280,"context_line":"                        (join(job[\u0027path\u0027], object_hash[-3:]),"},{"line_number":281,"context_line":"                         storage_directory(job[\u0027obj_path\u0027],"},{"line_number":282,"context_line":"                                           job[\u0027partition\u0027],"},{"line_number":283,"context_line":"                                           object_hash))"},{"line_number":284,"context_line":"                        for object_hash in delete_objs]"},{"line_number":285,"context_line":"                    self.delete_handoff_paths(delete_objs)"},{"line_number":286,"context_line":"                else:"}],"source_content_type":"text/x-python","patch_set":17,"id":"da86d52c_097d571c","line":283,"updated":"2015-02-11 19:52:21.000000000","message":"wait, is this right?  I thought storage_directory returned the full path - the whole idea with storage_directory was that you wouldn\u0027t have to do the object_hash[-3:] thing.","commit_id":"f038febfeea837e04ef6ed745eddd12809970ce3"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"2c1490fa83ffbdf35501b87a787f404b72e9b9d4","unresolved":false,"context_lines":[{"line_number":296,"context_line":"    def delete_handoff_paths(self, paths):"},{"line_number":297,"context_line":"        for suffix_dir, object_path in paths:"},{"line_number":298,"context_line":"            tpool.execute(shutil.rmtree, object_path, ignore_errors\u003dTrue)"},{"line_number":299,"context_line":"            try:"},{"line_number":300,"context_line":"                with lock_parent_directory(suffix_dir):"},{"line_number":301,"context_line":"                    os.rmdir(suffix_dir)"},{"line_number":302,"context_line":"            except OSError:"}],"source_content_type":"text/x-python","patch_set":17,"id":"da86d52c_6d6a64ee","line":299,"updated":"2015-02-11 17:47:05.000000000","message":"I think this is inside-out. I imagine one locks then catches rmdir in case of non-empty. It still works, I just don\u0027t like catching too much, in case.","commit_id":"f038febfeea837e04ef6ed745eddd12809970ce3"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"65d0e96f69ea84f577daf02ea2faf902066f4acc","unresolved":false,"context_lines":[{"line_number":296,"context_line":"    def delete_handoff_paths(self, paths):"},{"line_number":297,"context_line":"        for suffix_dir, object_path in paths:"},{"line_number":298,"context_line":"            tpool.execute(shutil.rmtree, object_path, ignore_errors\u003dTrue)"},{"line_number":299,"context_line":"            try:"},{"line_number":300,"context_line":"                with lock_parent_directory(suffix_dir):"},{"line_number":301,"context_line":"                    os.rmdir(suffix_dir)"},{"line_number":302,"context_line":"            except OSError:"}],"source_content_type":"text/x-python","patch_set":17,"id":"da86d52c_232ddd46","line":299,"in_reply_to":"da86d52c_6d6a64ee","updated":"2015-02-11 18:19:29.000000000","message":"you just mean with/try instead of try/with?\n\n    with lock_parent_directory(suffix_dir):\n        try:\n            os.rmdir(suffix_dir)\n        except OSError as e:\n            if e.errno !\u003d errno.ENOENT:\n                self.logger.exception(\"Unexpected error trying to cleanup suffix dir: %r\", suffix_dir)\n\nWhat about the lock_parent_directory - it can raise some sort of timeout too right - does it inherit from OSError?","commit_id":"f038febfeea837e04ef6ed745eddd12809970ce3"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"e66cae8ee592b38acc2de7a128e4ce53206a7c80","unresolved":false,"context_lines":[{"line_number":300,"context_line":"                with lock_parent_directory(suffix_dir):"},{"line_number":301,"context_line":"                    os.rmdir(suffix_dir)"},{"line_number":302,"context_line":"            except OSError:"},{"line_number":303,"context_line":"                pass"},{"line_number":304,"context_line":""},{"line_number":305,"context_line":"    def update(self, job):"},{"line_number":306,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":17,"id":"da86d52c_293dbb2b","line":303,"updated":"2015-02-11 19:52:21.000000000","message":"i want to check if any tests fail when we remove the lock_parent_dir line","commit_id":"f038febfeea837e04ef6ed745eddd12809970ce3"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"e66cae8ee592b38acc2de7a128e4ce53206a7c80","unresolved":false,"context_lines":[{"line_number":321,"context_line":"            self.logger.update_stats(\u0027suffix.hashes\u0027, hashed)"},{"line_number":322,"context_line":"            attempts_left \u003d len(job[\u0027nodes\u0027])"},{"line_number":323,"context_line":"            synced_remote_regions \u003d set()"},{"line_number":324,"context_line":"            random.shuffle(job[\u0027nodes\u0027])"},{"line_number":325,"context_line":"            nodes \u003d itertools.chain("},{"line_number":326,"context_line":"                job[\u0027nodes\u0027],"},{"line_number":327,"context_line":"                job[\u0027object_ring\u0027].get_more_nodes(int(job[\u0027partition\u0027])))"}],"source_content_type":"text/x-python","patch_set":17,"id":"da86d52c_096e372f","line":324,"updated":"2015-02-11 19:52:21.000000000","message":"seems reasonable... I also thought about re-ordering the nodes based on locality - local nodes first?","commit_id":"f038febfeea837e04ef6ed745eddd12809970ce3"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"3f3d7833a583ec6ce7f731f80ec65f4cc2c5deb5","unresolved":false,"context_lines":[{"line_number":257,"context_line":"                                \u0027/\u0027 + \u0027-\u0027.join(suffixes), headers\u003dself.headers)"},{"line_number":258,"context_line":"                            conn.getresponse().read()"},{"line_number":259,"context_line":"                        if node[\u0027region\u0027] !\u003d job[\u0027region\u0027] and cand_objs:"},{"line_number":260,"context_line":"                            synced_remote_regions[node[\u0027region\u0027]] \u003d cand_objs"},{"line_number":261,"context_line":"                    responses.append(success)"},{"line_number":262,"context_line":"                for region, cand_objs in synced_remote_regions.iteritems():"},{"line_number":263,"context_line":"                    if delete_objs is None:"}],"source_content_type":"text/x-python","patch_set":19,"id":"da86d52c_5aa35842","line":260,"updated":"2015-02-12 22:42:39.000000000","message":"because cand_objs returned from ssync is a subset of remote_check_objs (which was the previous value of synced_remote_regions[node[\u0027region\u0027]]) it\u0027s always safe to overwrite.","commit_id":"20ca279d747523f5cb942567515f85d03779eb8a"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"3f3d7833a583ec6ce7f731f80ec65f4cc2c5deb5","unresolved":false,"context_lines":[{"line_number":274,"context_line":"                    all(responses)"},{"line_number":275,"context_line":"            if not suffixes or delete_handoff:"},{"line_number":276,"context_line":"                if delete_objs:"},{"line_number":277,"context_line":"                    self.logger.info(_(\"Removing %s objecs\"),"},{"line_number":278,"context_line":"                                     len(delete_objs))"},{"line_number":279,"context_line":"                    delete_objs \u003d ["},{"line_number":280,"context_line":"                        storage_directory(job[\u0027obj_path\u0027],"}],"source_content_type":"text/x-python","patch_set":19,"id":"da86d52c_7aebbc12","line":277,"updated":"2015-02-12 22:42:39.000000000","message":"unfortunate typo :\u0027(\n\ns/objecs/objects","commit_id":"20ca279d747523f5cb942567515f85d03779eb8a"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"3f3d7833a583ec6ce7f731f80ec65f4cc2c5deb5","unresolved":false,"context_lines":[{"line_number":501,"context_line":"                    jobs.append("},{"line_number":502,"context_line":"                        dict(path\u003djob_path,"},{"line_number":503,"context_line":"                             device\u003dlocal_dev[\u0027device\u0027],"},{"line_number":504,"context_line":"                             obj_path\u003dobj_path,"},{"line_number":505,"context_line":"                             nodes\u003dnodes,"},{"line_number":506,"context_line":"                             delete\u003dlen(nodes) \u003e len(part_nodes) - 1,"},{"line_number":507,"context_line":"                             policy_idx\u003dpolicy.idx,"}],"source_content_type":"text/x-python","patch_set":19,"id":"da86d52c_7a973cec","line":504,"updated":"2015-02-12 22:42:39.000000000","message":"yes very helpful!","commit_id":"20ca279d747523f5cb942567515f85d03779eb8a"}],"swift/obj/ssync_sender.py":[{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"5eb2f55b833fb2330293f0183a57c528847789fc","unresolved":false,"context_lines":[{"line_number":41,"context_line":"        # When obj_list is given in job, ssync_sender trys only to make sure"},{"line_number":42,"context_line":"        # those objects exist or not in remote."},{"line_number":43,"context_line":"        self.obj_list \u003d job.get(\u0027obj_list\u0027, None) \\"},{"line_number":44,"context_line":"            if job is not None else None"},{"line_number":45,"context_line":"        self.send_list \u003d []"},{"line_number":46,"context_line":"        self.failures \u003d 0"},{"line_number":47,"context_line":""}],"source_content_type":"text/x-python","patch_set":7,"id":"baa201ad_25bf01fd","line":44,"updated":"2014-10-01 14:52:24.000000000","message":"When is job ever None here? I only found calls from update_deleted and update, and both supply job.","commit_id":"cf615115c80185ddd44068000c7ad5647eb4fc1c"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"f2edd44ac411ac80b5fc18e644547328ebe3e1c0","unresolved":false,"context_lines":[{"line_number":41,"context_line":"        # When obj_list is given in job, ssync_sender trys only to make sure"},{"line_number":42,"context_line":"        # those objects exist or not in remote."},{"line_number":43,"context_line":"        self.obj_list \u003d job.get(\u0027obj_list\u0027, None) \\"},{"line_number":44,"context_line":"            if job is not None else None"},{"line_number":45,"context_line":"        self.send_list \u003d []"},{"line_number":46,"context_line":"        self.failures \u003d 0"},{"line_number":47,"context_line":""}],"source_content_type":"text/x-python","patch_set":7,"id":"baa201ad_b796b592","line":44,"in_reply_to":"baa201ad_25bf01fd","updated":"2014-10-02 02:27:14.000000000","message":"This None is from existing unit test method. As you mentioned we don\u0027t have to check whether it is None or not, actually. I\u0027d like to cleanup the unit test by another patch as a code cleanup. Is it prefer for you?","commit_id":"cf615115c80185ddd44068000c7ad5647eb4fc1c"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"b59ced8c6cbfec264eb08240506aa5b55d2f5823","unresolved":false,"context_lines":[{"line_number":41,"context_line":"        # When obj_list is given in job, ssync_sender trys only to make sure"},{"line_number":42,"context_line":"        # those objects exist or not in remote."},{"line_number":43,"context_line":"        self.obj_list \u003d job.get(\u0027obj_list\u0027, None) \\"},{"line_number":44,"context_line":"            if job is not None else None"},{"line_number":45,"context_line":"        self.send_list \u003d []"},{"line_number":46,"context_line":"        self.failures \u003d 0"},{"line_number":47,"context_line":""}],"source_content_type":"text/x-python","patch_set":9,"id":"7a8c0949_44222ea5","line":44,"updated":"2014-11-06 09:39:09.000000000","message":"Nice, I\u0027ll ignore this for now then :)","commit_id":"875b049e4e392bf8c62c844339197f06df8c521b"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"f193b854c10f5394a9fdb0ed19a019ed1c4f403e","unresolved":false,"context_lines":[{"line_number":41,"context_line":"        # When obj_list is given in job, ssync_sender trys only to make sure"},{"line_number":42,"context_line":"        # those objects exist or not in remote."},{"line_number":43,"context_line":"        self.obj_list \u003d job.get(\u0027obj_list\u0027, None) \\"},{"line_number":44,"context_line":"            if job is not None else None"},{"line_number":45,"context_line":"        self.send_list \u003d []"},{"line_number":46,"context_line":"        self.failures \u003d 0"},{"line_number":47,"context_line":""}],"source_content_type":"text/x-python","patch_set":9,"id":"9aa7fdbe_be46e51f","line":44,"updated":"2014-10-29 04:17:14.000000000","message":"None is the default value to return on get method so you don\u0027t need to specify the None. Further below you check for \u0027if not self.obj_list\u0027 so the check for None is probably not needed, so you could simply this as:\n\n  self.obj_list \u003d job.get(\u0027obj_list\u0027) if job else None","commit_id":"875b049e4e392bf8c62c844339197f06df8c521b"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"3668c9272874e1d07451e50c4b6e1dec8402481e","unresolved":false,"context_lines":[{"line_number":41,"context_line":"        # When obj_list is given in job, ssync_sender trys only to make sure"},{"line_number":42,"context_line":"        # those objects exist or not in remote."},{"line_number":43,"context_line":"        self.obj_list \u003d job.get(\u0027obj_list\u0027, None) \\"},{"line_number":44,"context_line":"            if job is not None else None"},{"line_number":45,"context_line":"        self.send_list \u003d []"},{"line_number":46,"context_line":"        self.failures \u003d 0"},{"line_number":47,"context_line":""}],"source_content_type":"text/x-python","patch_set":9,"id":"7a8c0949_3f9bc8ca","line":44,"in_reply_to":"9aa7fdbe_b5451619","updated":"2014-10-30 11:39:42.000000000","message":"Thanks to answer, Pete!\n\nI thought the stuff to get rid of unnecessary \"None\" checking should be discussed in another place (I am going to work soon for it after this merged) because it is a code cleanup which is outside of this stuff. \n\nIs that OK, Pete?  Or should I work for it in this patch?","commit_id":"875b049e4e392bf8c62c844339197f06df8c521b"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"18b6cd293b9a7376dd923a44994e6776c27f3580","unresolved":false,"context_lines":[{"line_number":41,"context_line":"        # When obj_list is given in job, ssync_sender trys only to make sure"},{"line_number":42,"context_line":"        # those objects exist or not in remote."},{"line_number":43,"context_line":"        self.obj_list \u003d job.get(\u0027obj_list\u0027, None) \\"},{"line_number":44,"context_line":"            if job is not None else None"},{"line_number":45,"context_line":"        self.send_list \u003d []"},{"line_number":46,"context_line":"        self.failures \u003d 0"},{"line_number":47,"context_line":""}],"source_content_type":"text/x-python","patch_set":9,"id":"9aa7fdbe_b5451619","line":44,"in_reply_to":"9aa7fdbe_be46e51f","updated":"2014-10-29 14:47:57.000000000","message":"Matthew: it\u0027s actually even better - the None checking is unnecessary here, but it\u0027s used in tests. Kota promised to fix them up once we get this in and get rid of it entirely.","commit_id":"875b049e4e392bf8c62c844339197f06df8c521b"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"03277ac605a3eff6d08b5c435604b4ac4f9dec4d","unresolved":false,"context_lines":[{"line_number":178,"context_line":"            self.job[\u0027device\u0027], self.job[\u0027partition\u0027],"},{"line_number":179,"context_line":"            self.policy_idx, self.suffixes)"},{"line_number":180,"context_line":"        if self.obj_list:"},{"line_number":181,"context_line":"            hash_gen \u003d iter("},{"line_number":182,"context_line":"                filter(lambda (p, o, t): o in self.obj_list, hash_gen))"},{"line_number":183,"context_line":"        for path, object_hash, timestamp in hash_gen:"},{"line_number":184,"context_line":"            self.candidates.append(object_hash)"}],"source_content_type":"text/x-python","patch_set":9,"id":"9aa7fdbe_c307561c","line":181,"updated":"2014-10-29 03:45:29.000000000","message":"stylistic: why do you think iter() is needed here? hash_gen goes right into a for-in loop. It would work with the unrolled list that filter() returns in Python 2, right?","commit_id":"875b049e4e392bf8c62c844339197f06df8c521b"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"3668c9272874e1d07451e50c4b6e1dec8402481e","unresolved":false,"context_lines":[{"line_number":178,"context_line":"            self.job[\u0027device\u0027], self.job[\u0027partition\u0027],"},{"line_number":179,"context_line":"            self.policy_idx, self.suffixes)"},{"line_number":180,"context_line":"        if self.obj_list:"},{"line_number":181,"context_line":"            hash_gen \u003d iter("},{"line_number":182,"context_line":"                filter(lambda (p, o, t): o in self.obj_list, hash_gen))"},{"line_number":183,"context_line":"        for path, object_hash, timestamp in hash_gen:"},{"line_number":184,"context_line":"            self.candidates.append(object_hash)"}],"source_content_type":"text/x-python","patch_set":9,"id":"7a8c0949_444a8784","line":181,"in_reply_to":"9aa7fdbe_c307561c","updated":"2014-10-30 11:39:42.000000000","message":"yup, maybe you are right. I forgot my intention to wrap by iter() here so I\u0027ll try to get rid of it. Thanks :)","commit_id":"875b049e4e392bf8c62c844339197f06df8c521b"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"6cf7cbafd2e032b2b512437d97e83e671be0b6b4","unresolved":false,"context_lines":[{"line_number":219,"context_line":"                self.send_list.append(line)"},{"line_number":220,"context_line":"        # fail when all hashes required by receiver"},{"line_number":221,"context_line":"        if self.send_list \u003d\u003d self.obj_list:"},{"line_number":222,"context_line":"            self.failure +\u003d 1"},{"line_number":223,"context_line":""},{"line_number":224,"context_line":"    def updates(self):"},{"line_number":225,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":10,"id":"7a8c0949_b2bdc66a","line":222,"updated":"2014-11-07 03:34:04.000000000","message":"is this self.falure or self.failures ?","commit_id":"33b06e2e776d97c7681a50cd3dcb2906e67cb967"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"91730a748c3b1a6cbbef2dc0f9afefae2a3624c1","unresolved":false,"context_lines":[{"line_number":219,"context_line":"                self.send_list.append(line)"},{"line_number":220,"context_line":"        # fail when all hashes required by receiver"},{"line_number":221,"context_line":"        if self.send_list \u003d\u003d self.obj_list:"},{"line_number":222,"context_line":"            self.failure +\u003d 1"},{"line_number":223,"context_line":""},{"line_number":224,"context_line":"    def updates(self):"},{"line_number":225,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":10,"id":"7a8c0949_ffce36b3","line":222,"in_reply_to":"7a8c0949_b2bdc66a","updated":"2014-11-07 07:51:33.000000000","message":"Oh, yes. I guess, you are right. I will recheck my implementation.","commit_id":"33b06e2e776d97c7681a50cd3dcb2906e67cb967"},{"author":{"_account_id":330,"name":"John Dickinson","email":"me@not.mn","username":"notmyname"},"change_message_id":"2ef91a7b9837378bf031d86a1980fe040e5b6ac4","unresolved":false,"context_lines":[{"line_number":92,"context_line":"            # would only get called if the above except Exception handler"},{"line_number":93,"context_line":"            # failed (bad node or job data)."},{"line_number":94,"context_line":"            self.daemon.logger.exception(\u0027EXCEPTION in replication.Sender\u0027)"},{"line_number":95,"context_line":"        return False, list(set(self.candidates) - set(self.send_list))"},{"line_number":96,"context_line":""},{"line_number":97,"context_line":"    def connect(self):"},{"line_number":98,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":12,"id":"3a961159_11a2064b","line":95,"updated":"2014-12-19 00:54:47.000000000","message":"the objects that we asked about but the remote side didn\u0027t want","commit_id":"19a1a18da8c8a82c410423ff9fbc3d94b2bdc979"},{"author":{"_account_id":330,"name":"John Dickinson","email":"me@not.mn","username":"notmyname"},"change_message_id":"2ef91a7b9837378bf031d86a1980fe040e5b6ac4","unresolved":false,"context_lines":[{"line_number":221,"context_line":"                self.send_list.append(line)"},{"line_number":222,"context_line":"        # fail when all hashes required by receiver"},{"line_number":223,"context_line":"        if self.send_list \u003d\u003d self.obj_list:"},{"line_number":224,"context_line":"            self.failures +\u003d 1"},{"line_number":225,"context_line":""},{"line_number":226,"context_line":"    def updates(self):"},{"line_number":227,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":12,"id":"3a961159_31cf0281","line":224,"updated":"2014-12-19 00:54:47.000000000","message":"I don\u0027t get this yet...","commit_id":"19a1a18da8c8a82c410423ff9fbc3d94b2bdc979"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"97811804539207911ee8e97f70eb507af083c469","unresolved":false,"context_lines":[{"line_number":221,"context_line":"                self.send_list.append(line)"},{"line_number":222,"context_line":"        # fail when all hashes required by receiver"},{"line_number":223,"context_line":"        if self.send_list \u003d\u003d self.obj_list:"},{"line_number":224,"context_line":"            self.failures +\u003d 1"},{"line_number":225,"context_line":""},{"line_number":226,"context_line":"    def updates(self):"},{"line_number":227,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":12,"id":"3a961159_fc79ce82","line":224,"in_reply_to":"3a961159_31cf0281","updated":"2014-12-19 03:43:50.000000000","message":"So, if _everything_ has to be sent, the replicatior in check_ring() sees a failure of sender failure and sends _nothing_? The one in update() seems to work as expected.","commit_id":"19a1a18da8c8a82c410423ff9fbc3d94b2bdc979"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"b6927cb23697dc0902744d364a649c32c8d0704d","unresolved":false,"context_lines":[{"line_number":221,"context_line":"                self.send_list.append(line)"},{"line_number":222,"context_line":"        # fail when all hashes required by receiver"},{"line_number":223,"context_line":"        if self.send_list \u003d\u003d self.obj_list:"},{"line_number":224,"context_line":"            self.failures +\u003d 1"},{"line_number":225,"context_line":""},{"line_number":226,"context_line":"    def updates(self):"},{"line_number":227,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":12,"id":"3a961159_51856405","line":224,"in_reply_to":"3a961159_fc79ce82","updated":"2014-12-19 09:54:29.000000000","message":"If assuming this as a success, update/update_deleted will pass empty list via the \"synced_remote_region\" variable in the next call. It will cause unexpected syncing because currently we assumes an empty list as an order that \"SYNC WHOLE THINGS\".\n\nI also think, it might be complex bit. However, I\u0027d like to avoid to add something like a response value as possible. This is why I made this as a failure.","commit_id":"19a1a18da8c8a82c410423ff9fbc3d94b2bdc979"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"083ab02578d1e500aaa8f44da92f949182aa58d5","unresolved":false,"context_lines":[{"line_number":28,"context_line":"    process is there."},{"line_number":29,"context_line":"    \"\"\""},{"line_number":30,"context_line":""},{"line_number":31,"context_line":"    def __init__(self, daemon, node, job, suffixes):"},{"line_number":32,"context_line":"        self.daemon \u003d daemon"},{"line_number":33,"context_line":"        self.node \u003d node"},{"line_number":34,"context_line":"        self.job \u003d job"}],"source_content_type":"text/x-python","patch_set":14,"id":"da86d52c_ee178543","side":"PARENT","line":31,"updated":"2015-02-10 02:32:49.000000000","message":"suffixes, remote_check_objs\u003dNone","commit_id":"b45b83fb00917b729253804443094d97d39ef78f"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"c04346c924e801984967dc1d17e13e718c18d674","unresolved":false,"context_lines":[{"line_number":39,"context_line":"        self.response_chunk_left \u003d 0"},{"line_number":40,"context_line":"        self.candidates \u003d []"},{"line_number":41,"context_line":"        # When obj_list is given in job, ssync_sender trys only to make sure"},{"line_number":42,"context_line":"        # those objects exist or not in remote."},{"line_number":43,"context_line":"        self.obj_list \u003d job.get(\u0027obj_list\u0027, None) \\"},{"line_number":44,"context_line":"            if job is not None else None"},{"line_number":45,"context_line":"        self.send_list \u003d []"}],"source_content_type":"text/x-python","patch_set":14,"id":"da86d52c_c9aba487","line":42,"updated":"2015-02-09 23:40:47.000000000","message":"I don\u0027t understand.","commit_id":"d52d661628fc92b14e8d8322d5959687384f8037"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"083ab02578d1e500aaa8f44da92f949182aa58d5","unresolved":false,"context_lines":[{"line_number":39,"context_line":"        self.response_chunk_left \u003d 0"},{"line_number":40,"context_line":"        self.candidates \u003d []"},{"line_number":41,"context_line":"        # When obj_list is given in job, ssync_sender trys only to make sure"},{"line_number":42,"context_line":"        # those objects exist or not in remote."},{"line_number":43,"context_line":"        self.obj_list \u003d job.get(\u0027obj_list\u0027, None) \\"},{"line_number":44,"context_line":"            if job is not None else None"},{"line_number":45,"context_line":"        self.send_list \u003d []"}],"source_content_type":"text/x-python","patch_set":14,"id":"da86d52c_331bca81","line":42,"in_reply_to":"da86d52c_c9aba487","updated":"2015-02-10 02:32:49.000000000","message":"don\u0027t put obj_list in the job (we have to delete it from the job later!) - this should be kwarg that indicates the job does not do updates, and instead the work to do is to only *check* which objects the remote end needs.\n\nremote_check_objs maybe?","commit_id":"d52d661628fc92b14e8d8322d5959687384f8037"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"f9b741b1c444a0f37209d874bbfcd467fbbda936","unresolved":false,"context_lines":[{"line_number":41,"context_line":"        # When obj_list is given in job, ssync_sender trys only to make sure"},{"line_number":42,"context_line":"        # those objects exist or not in remote."},{"line_number":43,"context_line":"        self.obj_list \u003d job.get(\u0027obj_list\u0027, None) \\"},{"line_number":44,"context_line":"            if job is not None else None"},{"line_number":45,"context_line":"        self.send_list \u003d []"},{"line_number":46,"context_line":"        self.failures \u003d 0"},{"line_number":47,"context_line":""}],"source_content_type":"text/x-python","patch_set":14,"id":"da86d52c_ae458d16","line":44,"updated":"2015-02-10 02:07:51.000000000","message":"this \"job is None\" is conditionally accepted with Kota\u0027s promise to fix tests later so we don\u0027t mix the two","commit_id":"d52d661628fc92b14e8d8322d5959687384f8037"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"c04346c924e801984967dc1d17e13e718c18d674","unresolved":false,"context_lines":[{"line_number":63,"context_line":"                self.connect()"},{"line_number":64,"context_line":"                self.missing_check()"},{"line_number":65,"context_line":"                if not self.obj_list:"},{"line_number":66,"context_line":"                    self.updates()"},{"line_number":67,"context_line":"                self.disconnect()"},{"line_number":68,"context_line":"                return self.failures \u003d\u003d 0, \\"},{"line_number":69,"context_line":"                    list(set(self.candidates) - set(self.send_list))"}],"source_content_type":"text/x-python","patch_set":14,"id":"da86d52c_8988cc23","line":66,"updated":"2015-02-09 23:40:47.000000000","message":"we only send our objects/updates over (sync) if not initialized with obj_list.","commit_id":"d52d661628fc92b14e8d8322d5959687384f8037"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"083ab02578d1e500aaa8f44da92f949182aa58d5","unresolved":false,"context_lines":[{"line_number":63,"context_line":"                self.connect()"},{"line_number":64,"context_line":"                self.missing_check()"},{"line_number":65,"context_line":"                if not self.obj_list:"},{"line_number":66,"context_line":"                    self.updates()"},{"line_number":67,"context_line":"                self.disconnect()"},{"line_number":68,"context_line":"                return self.failures \u003d\u003d 0, \\"},{"line_number":69,"context_line":"                    list(set(self.candidates) - set(self.send_list))"}],"source_content_type":"text/x-python","patch_set":14,"id":"da86d52c_2e299d89","line":66,"in_reply_to":"da86d52c_8988cc23","updated":"2015-02-10 02:32:49.000000000","message":"if not self.remote_check_objs:\n   self.updates()","commit_id":"d52d661628fc92b14e8d8322d5959687384f8037"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"c04346c924e801984967dc1d17e13e718c18d674","unresolved":false,"context_lines":[{"line_number":66,"context_line":"                    self.updates()"},{"line_number":67,"context_line":"                self.disconnect()"},{"line_number":68,"context_line":"                return self.failures \u003d\u003d 0, \\"},{"line_number":69,"context_line":"                    list(set(self.candidates) - set(self.send_list))"},{"line_number":70,"context_line":"            except (exceptions.MessageTimeout,"},{"line_number":71,"context_line":"                    exceptions.ReplicationException) as err:"},{"line_number":72,"context_line":"                self.daemon.logger.error("}],"source_content_type":"text/x-python","patch_set":14,"id":"da86d52c_84c7d311","line":69,"updated":"2015-02-09 23:40:47.000000000","message":"so we\u0027re returning a list of all the objects we have - but that we didn\u0027t send?  Do we know if that\u0027s because we didn\u0027t send updates at all (because this job was initialized with obj_list) or if the remote end just didn\u0027t end up needing anything we have (maybe the suffix hashes looked out of sync, but by the time we got into the data dirs another node has pushed over the object and now all the timestamps match?)","commit_id":"d52d661628fc92b14e8d8322d5959687384f8037"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"083ab02578d1e500aaa8f44da92f949182aa58d5","unresolved":false,"context_lines":[{"line_number":66,"context_line":"                    self.updates()"},{"line_number":67,"context_line":"                self.disconnect()"},{"line_number":68,"context_line":"                return self.failures \u003d\u003d 0, \\"},{"line_number":69,"context_line":"                    list(set(self.candidates) - set(self.send_list))"},{"line_number":70,"context_line":"            except (exceptions.MessageTimeout,"},{"line_number":71,"context_line":"                    exceptions.ReplicationException) as err:"},{"line_number":72,"context_line":"                self.daemon.logger.error("}],"source_content_type":"text/x-python","patch_set":14,"id":"da86d52c_ee2ea58b","line":69,"in_reply_to":"da86d52c_84c7d311","updated":"2015-02-10 02:32:49.000000000","message":"if we\u0027re a sync job, candiates \u003d\u003d send_list and the returned list is empty - maybe on sync we want to return all of candidates?","commit_id":"d52d661628fc92b14e8d8322d5959687384f8037"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"c04346c924e801984967dc1d17e13e718c18d674","unresolved":false,"context_lines":[{"line_number":92,"context_line":"            # would only get called if the above except Exception handler"},{"line_number":93,"context_line":"            # failed (bad node or job data)."},{"line_number":94,"context_line":"            self.daemon.logger.exception(\u0027EXCEPTION in replication.Sender\u0027)"},{"line_number":95,"context_line":"        return False, list(set(self.candidates) - set(self.send_list))"},{"line_number":96,"context_line":""},{"line_number":97,"context_line":"    def connect(self):"},{"line_number":98,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":14,"id":"da86d52c_a91bd07e","line":95,"updated":"2015-02-09 23:40:47.000000000","message":"does the second return value matter if we failed?","commit_id":"d52d661628fc92b14e8d8322d5959687384f8037"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"c04346c924e801984967dc1d17e13e718c18d674","unresolved":false,"context_lines":[{"line_number":181,"context_line":"            self.policy_idx, self.suffixes)"},{"line_number":182,"context_line":"        if self.obj_list:"},{"line_number":183,"context_line":"            hash_gen \u003d filter(lambda (path, object_hash, timestamp):"},{"line_number":184,"context_line":"                              object_hash in self.obj_list, hash_gen)"},{"line_number":185,"context_line":"        for path, object_hash, timestamp in hash_gen:"},{"line_number":186,"context_line":"            self.candidates.append(object_hash)"},{"line_number":187,"context_line":"            with exceptions.MessageTimeout("}],"source_content_type":"text/x-python","patch_set":14,"id":"da86d52c_a9e030cd","line":184,"updated":"2015-02-09 23:40:47.000000000","message":"I\u0027m pretty sure filter will consume the hash_gen here and make a flurry of disk activity.\n\nProbably better to just leave the existing loop and add:\n\n    if obj_list and object_hash not in obj_list:\n        continue\n\nBTW, what is the cardinality of obj_list compared to the hash list (hash list can be all of the objects in all of the suffixes in the partition, maybe 1M or so)","commit_id":"d52d661628fc92b14e8d8322d5959687384f8037"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"083ab02578d1e500aaa8f44da92f949182aa58d5","unresolved":false,"context_lines":[{"line_number":181,"context_line":"            self.policy_idx, self.suffixes)"},{"line_number":182,"context_line":"        if self.obj_list:"},{"line_number":183,"context_line":"            hash_gen \u003d filter(lambda (path, object_hash, timestamp):"},{"line_number":184,"context_line":"                              object_hash in self.obj_list, hash_gen)"},{"line_number":185,"context_line":"        for path, object_hash, timestamp in hash_gen:"},{"line_number":186,"context_line":"            self.candidates.append(object_hash)"},{"line_number":187,"context_line":"            with exceptions.MessageTimeout("}],"source_content_type":"text/x-python","patch_set":14,"id":"da86d52c_6e7f1567","line":184,"in_reply_to":"da86d52c_a9e030cd","updated":"2015-02-10 02:32:49.000000000","message":"oh itertools.ifilter is you super down on that","commit_id":"d52d661628fc92b14e8d8322d5959687384f8037"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"c04346c924e801984967dc1d17e13e718c18d674","unresolved":false,"context_lines":[{"line_number":183,"context_line":"            hash_gen \u003d filter(lambda (path, object_hash, timestamp):"},{"line_number":184,"context_line":"                              object_hash in self.obj_list, hash_gen)"},{"line_number":185,"context_line":"        for path, object_hash, timestamp in hash_gen:"},{"line_number":186,"context_line":"            self.candidates.append(object_hash)"},{"line_number":187,"context_line":"            with exceptions.MessageTimeout("},{"line_number":188,"context_line":"                    self.daemon.node_timeout,"},{"line_number":189,"context_line":"                    \u0027missing_check send line\u0027):"}],"source_content_type":"text/x-python","patch_set":14,"id":"da86d52c_49aa541a","line":186,"updated":"2015-02-09 23:40:47.000000000","message":"so candidates is a list of every object (hash) that we have locally and could possibly ship off to remote nodes?","commit_id":"d52d661628fc92b14e8d8322d5959687384f8037"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"083ab02578d1e500aaa8f44da92f949182aa58d5","unresolved":false,"context_lines":[{"line_number":183,"context_line":"            hash_gen \u003d filter(lambda (path, object_hash, timestamp):"},{"line_number":184,"context_line":"                              object_hash in self.obj_list, hash_gen)"},{"line_number":185,"context_line":"        for path, object_hash, timestamp in hash_gen:"},{"line_number":186,"context_line":"            self.candidates.append(object_hash)"},{"line_number":187,"context_line":"            with exceptions.MessageTimeout("},{"line_number":188,"context_line":"                    self.daemon.node_timeout,"},{"line_number":189,"context_line":"                    \u0027missing_check send line\u0027):"}],"source_content_type":"text/x-python","patch_set":14,"id":"da86d52c_8ec36989","line":186,"in_reply_to":"da86d52c_49aa541a","updated":"2015-02-10 02:32:49.000000000","message":"self.available_list is better name that candidates?\n\nThe only reason I was confused is because up one level in the replicator, \"candidates\" means \"candidates to delete\"","commit_id":"d52d661628fc92b14e8d8322d5959687384f8037"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"c04346c924e801984967dc1d17e13e718c18d674","unresolved":false,"context_lines":[{"line_number":219,"context_line":"                break"},{"line_number":220,"context_line":"            if line:"},{"line_number":221,"context_line":"                self.send_list.append(line)"},{"line_number":222,"context_line":"        # fail when all hashes required by receiver"},{"line_number":223,"context_line":"        if self.send_list \u003d\u003d self.obj_list:"},{"line_number":224,"context_line":"            self.failures +\u003d 1"},{"line_number":225,"context_line":""}],"source_content_type":"text/x-python","patch_set":14,"id":"da86d52c_69539839","line":222,"updated":"2015-02-09 23:40:47.000000000","message":"this comment says what the code does, not why it does it.\n\nWhen and how will this job be retried?  I take it that if obj_list \u003d\u003d [] we don\u0027t fail - and we actually *will* ship over the whole partition.","commit_id":"d52d661628fc92b14e8d8322d5959687384f8037"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"083ab02578d1e500aaa8f44da92f949182aa58d5","unresolved":false,"context_lines":[{"line_number":219,"context_line":"                break"},{"line_number":220,"context_line":"            if line:"},{"line_number":221,"context_line":"                self.send_list.append(line)"},{"line_number":222,"context_line":"        # fail when all hashes required by receiver"},{"line_number":223,"context_line":"        if self.send_list \u003d\u003d self.obj_list:"},{"line_number":224,"context_line":"            self.failures +\u003d 1"},{"line_number":225,"context_line":""}],"source_content_type":"text/x-python","patch_set":14,"id":"da86d52c_aef3cde5","line":222,"in_reply_to":"da86d52c_69539839","updated":"2015-02-10 02:32:49.000000000","message":"If we\u0027re a sync job obj_list is always empty - if we\u0027re a sync job and in sync with remote node then send_list is empty - this should not be a failure.  I have a doubt.","commit_id":"d52d661628fc92b14e8d8322d5959687384f8037"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"4f3efad9a3428201df84f2982813b96d73a70e58","unresolved":false,"context_lines":[{"line_number":100,"context_line":"            # would only get called if the above except Exception handler"},{"line_number":101,"context_line":"            # failed (bad node or job data)."},{"line_number":102,"context_line":"            self.daemon.logger.exception(\u0027EXCEPTION in replication.Sender\u0027)"},{"line_number":103,"context_line":"        return False, list(set(self.candidates) - set(self.send_list))"},{"line_number":104,"context_line":""},{"line_number":105,"context_line":"    def connect(self):"},{"line_number":106,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":15,"id":"da86d52c_b3fc1a41","line":103,"updated":"2015-02-10 01:37:56.000000000","message":"wasted computation since nobody ever looks at the candidates if success\u003d\u003dFalse","commit_id":"54ba45d862203dc622b753e6e29ffdd9bbf9aaf9"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"d914fdcc6251e966b72fe0b236002264d42cfc7d","unresolved":false,"context_lines":[{"line_number":38,"context_line":"        self.response \u003d None"},{"line_number":39,"context_line":"        self.response_buffer \u003d \u0027\u0027"},{"line_number":40,"context_line":"        self.response_chunk_left \u003d 0"},{"line_number":41,"context_line":"        self.available_list \u003d []"},{"line_number":42,"context_line":"        # When remote_check_objs is given in job, ssync_sender trys only to"},{"line_number":43,"context_line":"        # make sure those objects exist or not in remote."},{"line_number":44,"context_line":"        self.remote_check_objs \u003d remote_check_objs"}],"source_content_type":"text/x-python","patch_set":16,"id":"da86d52c_78b41635","line":41,"updated":"2015-02-10 19:16:43.000000000","message":"make this always a set.","commit_id":"228d5407943bec9c5d3a204693fa421456b2d0d5"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"d914fdcc6251e966b72fe0b236002264d42cfc7d","unresolved":false,"context_lines":[{"line_number":68,"context_line":"                        else set(self.available_list)"},{"line_number":69,"context_line":"                else:"},{"line_number":70,"context_line":"                    can_delete_obj \u003d set() if self.failures else \\"},{"line_number":71,"context_line":"                        set(self.available_list) - set(self.send_list)"},{"line_number":72,"context_line":"                self.disconnect()"},{"line_number":73,"context_line":"                return self.failures \u003d\u003d 0, list(can_delete_obj)"},{"line_number":74,"context_line":"            except (exceptions.MessageTimeout,"}],"source_content_type":"text/x-python","patch_set":16,"id":"da86d52c_ee23dca4","line":71,"updated":"2015-02-10 19:16:43.000000000","message":"gah, maybe like this:\n\nhttps://gist.github.com/clayg/c14dc9aaad2f2decf258","commit_id":"228d5407943bec9c5d3a204693fa421456b2d0d5"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"fdd68e0c6cdd524248061f70b79db4161189be04","unresolved":false,"context_lines":[{"line_number":68,"context_line":"                        else set(self.available_list)"},{"line_number":69,"context_line":"                else:"},{"line_number":70,"context_line":"                    can_delete_obj \u003d set() if self.failures else \\"},{"line_number":71,"context_line":"                        set(self.available_list) - set(self.send_list)"},{"line_number":72,"context_line":"                self.disconnect()"},{"line_number":73,"context_line":"                return self.failures \u003d\u003d 0, list(can_delete_obj)"},{"line_number":74,"context_line":"            except (exceptions.MessageTimeout,"}],"source_content_type":"text/x-python","patch_set":16,"id":"da86d52c_9c837955","line":71,"in_reply_to":"da86d52c_ee23dca4","updated":"2015-02-10 20:34:26.000000000","message":"I like that gist more too. Just don\u0027t forget to fix up rsync case with empty sets.","commit_id":"228d5407943bec9c5d3a204693fa421456b2d0d5"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"d914fdcc6251e966b72fe0b236002264d42cfc7d","unresolved":false,"context_lines":[{"line_number":70,"context_line":"                    can_delete_obj \u003d set() if self.failures else \\"},{"line_number":71,"context_line":"                        set(self.available_list) - set(self.send_list)"},{"line_number":72,"context_line":"                self.disconnect()"},{"line_number":73,"context_line":"                return self.failures \u003d\u003d 0, list(can_delete_obj)"},{"line_number":74,"context_line":"            except (exceptions.MessageTimeout,"},{"line_number":75,"context_line":"                    exceptions.ReplicationException) as err:"},{"line_number":76,"context_line":"                self.daemon.logger.error("}],"source_content_type":"text/x-python","patch_set":16,"id":"da86d52c_18af9ae4","line":73,"updated":"2015-02-10 19:16:43.000000000","message":"leave this as a set.","commit_id":"228d5407943bec9c5d3a204693fa421456b2d0d5"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"d914fdcc6251e966b72fe0b236002264d42cfc7d","unresolved":false,"context_lines":[{"line_number":225,"context_line":"                self.send_list.append(line)"},{"line_number":226,"context_line":"        # fail when all hashes required by receiver"},{"line_number":227,"context_line":"        if self.remote_check_objs and self.send_list \u003d\u003d self.remote_check_objs:"},{"line_number":228,"context_line":"            self.failures +\u003d 1"},{"line_number":229,"context_line":""},{"line_number":230,"context_line":"    def updates(self):"},{"line_number":231,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":16,"id":"da86d52c_9331ad0d","line":228,"updated":"2015-02-10 19:16:43.000000000","message":"i really like to understand who depends on this behavior, you may just have to explain it to me if it\u0027s too many words to put in the comment here.","commit_id":"228d5407943bec9c5d3a204693fa421456b2d0d5"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"e66cae8ee592b38acc2de7a128e4ce53206a7c80","unresolved":false,"context_lines":[{"line_number":38,"context_line":"        self.response \u003d None"},{"line_number":39,"context_line":"        self.response_buffer \u003d \u0027\u0027"},{"line_number":40,"context_line":"        self.response_chunk_left \u003d 0"},{"line_number":41,"context_line":"        self.available_list \u003d set()"},{"line_number":42,"context_line":"        # When remote_check_objs is given in job, ssync_sender trys only to"},{"line_number":43,"context_line":"        # make sure those objects exist or not in remote."},{"line_number":44,"context_line":"        self.remote_check_objs \u003d remote_check_objs"}],"source_content_type":"text/x-python","patch_set":17,"id":"da86d52c_29005b33","line":41,"updated":"2015-02-11 19:52:21.000000000","message":"for some reason I\u0027m only realizing now this is a *terrible* name for this attribute - it\u0027s not a list, it\u0027s a set.","commit_id":"f038febfeea837e04ef6ed745eddd12809970ce3"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"e66cae8ee592b38acc2de7a128e4ce53206a7c80","unresolved":false,"context_lines":[{"line_number":72,"context_line":"                self.missing_check()"},{"line_number":73,"context_line":"                if not self.remote_check_objs:"},{"line_number":74,"context_line":"                    self.updates()"},{"line_number":75,"context_line":"                    can_delete_obj \u003d set(self.available_list)"},{"line_number":76,"context_line":"                else:"},{"line_number":77,"context_line":"                    # when we are initialized with remote_check_objs we don\u0027t"},{"line_number":78,"context_line":"                    # *send* any requested updates; instead we only collect"}],"source_content_type":"text/x-python","patch_set":17,"id":"da86d52c_49902fed","line":75,"updated":"2015-02-11 19:52:21.000000000","message":"if the attributes name was not available_*list* I think it would have been more obvious this is not helpful.","commit_id":"f038febfeea837e04ef6ed745eddd12809970ce3"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"e66cae8ee592b38acc2de7a128e4ce53206a7c80","unresolved":false,"context_lines":[{"line_number":239,"context_line":"        # This is a just check to be safe if we don\u0027t need to delete any"},{"line_number":240,"context_line":"        # objects."},{"line_number":241,"context_line":"        if self.remote_check_objs and self.send_list \u003d\u003d self.remote_check_objs:"},{"line_number":242,"context_line":"            self.failures +\u003d 1"},{"line_number":243,"context_line":""},{"line_number":244,"context_line":"    def updates(self):"},{"line_number":245,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":17,"id":"da86d52c_ec7e5169","line":242,"updated":"2015-02-11 19:52:21.000000000","message":"this still doesn\u0027t make sense to me - it\u0027s confusing, distracting and not necessary.\n\nI\u0027ve come to realize that available is always a subset of remote_check (because of the ifitler), and send_list is always a subset of available (because missing_check only responds to the available)... so:\n\n    remote_check_objs \u003e\u003d available \u003e\u003d send_list\n\nIn the very specific case where send_list is exactly the same items in the same order as remote_check_objs - we know it also has the same elements as available because of the transitive property.\n\nSo when we return:\n\n    can_delete_obj \u003d self.available_list.difference(self.send_list)\n\nIt *has* to be an empty list... so why is it helpful to incrementing failures.\n\nIMHO counter intuitive for the sync job that successfully validates the remote end needs everything we want to offer that this is a failure?  It seems rather it\u0027s successfully validated that no objects can be safely deleted, which is exactly what it does without this check.","commit_id":"f038febfeea837e04ef6ed745eddd12809970ce3"}],"test/unit/obj/test_replicator.py":[{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"8dd37c6dc58120a15dda91673da26b0614a50fce","unresolved":false,"context_lines":[{"line_number":610,"context_line":"            self.call_nums \u003d 0"},{"line_number":611,"context_line":""},{"line_number":612,"context_line":"            def _fake_ssync(node, job, suffixes):"},{"line_number":613,"context_line":"                self.call_nums +\u003d 1"},{"line_number":614,"context_line":"                if self.call_nums \u003c 3:"},{"line_number":615,"context_line":"                    # In the first time of replication every sync returns"},{"line_number":616,"context_line":"                    # True, [] and skip the primaries in the same region"}],"source_content_type":"text/x-python","patch_set":8,"id":"baa201ad_abc0062b","line":613,"updated":"2014-10-02 15:47:47.000000000","message":"This line should be moved into line 622.","commit_id":"53d0cf832ab9b861dc302b19b59d6af5c9bc850e"}]}