)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":6968,"name":"Christian Schwede","email":"cschwede@nvidia.com","username":"cschwede"},"change_message_id":"38a4050c89149775484bfac6ac982ae44b1be2a0","unresolved":false,"context_lines":[{"line_number":10,"context_line":"or hash invalidation not succeed for new suffix dir."},{"line_number":11,"context_line":""},{"line_number":12,"context_line":"Change-Id: Ib5c9dd17e40150450ec57a728ae8652fbc730af6"},{"line_number":13,"context_line":"Related-Bug: #1634967"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":2,"id":"5a74a57a_00a3deb2","line":13,"updated":"2016-11-28 11:13:44.000000000","message":"Missing:\n\n Closes-Bug: 1644807\n\nAlso, please mention that this makes the whole process deterministic","commit_id":"11ef7a96fe3777fb0af26b54d87f4d7294748828"}],"swift/obj/diskfile.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"753e793c3173071e554191b94eb30dd5d853a34d","unresolved":false,"context_lines":[{"line_number":1065,"context_line":"                if len(suff) \u003d\u003d 3:"},{"line_number":1066,"context_line":"                    hashes.setdefault(suff, None)"},{"line_number":1067,"context_line":"            modified \u003d True"},{"line_number":1068,"context_line":"            self.logger.debug(\u0027Run listdir on %s\u0027, partition_path)"},{"line_number":1069,"context_line":"        hashes.update((suffix, None) for suffix in recalculate)"},{"line_number":1070,"context_line":"        for suffix, hash_ in hashes.items():"},{"line_number":1071,"context_line":"            if not hash_:"}],"source_content_type":"text/x-python","patch_set":10,"id":"3a71b18c_91b60b3c","line":1068,"updated":"2016-12-01 10:45:56.000000000","message":"wfm\n\nDec  1 10:41:35 saio object-6030: Run listdir on /srv/node3/sdb3/objects/645","commit_id":"3c730abb8bd00d410466d5e8a3dbe14bc8c5e2e6"}],"swift/obj/replicator.py":[{"author":{"_account_id":6968,"name":"Christian Schwede","email":"cschwede@nvidia.com","username":"cschwede"},"change_message_id":"38a4050c89149775484bfac6ac982ae44b1be2a0","unresolved":false,"context_lines":[{"line_number":75,"context_line":"        self.next_check \u003d time.time() + self.ring_check_interval"},{"line_number":76,"context_line":"        self.reclaim_age \u003d int(conf.get(\u0027reclaim_age\u0027, 86400 * 7))"},{"line_number":77,"context_line":"        self.do_list_interval \u003d int("},{"line_number":78,"context_line":"            conf.get(\u0027do_list_interval\u0027, 10))"},{"line_number":79,"context_line":"        self.replication_phase \u003d random.randint("},{"line_number":80,"context_line":"            0,"},{"line_number":81,"context_line":"            self.do_list_interval - 1)"}],"source_content_type":"text/x-python","patch_set":2,"id":"5a74a57a_f796aeee","line":78,"updated":"2016-11-28 11:13:44.000000000","message":"We need backports for this, so I think this should be part of another patch, and not included in this bugfix","commit_id":"11ef7a96fe3777fb0af26b54d87f4d7294748828"},{"author":{"_account_id":23449,"name":"Pavel Kvasnička","email":"pavel.kvasnicka@firma.seznam.cz","username":"PavelK"},"change_message_id":"62cd75eb70865f9a5363145a86244c54b4186834","unresolved":false,"context_lines":[{"line_number":75,"context_line":"        self.next_check \u003d time.time() + self.ring_check_interval"},{"line_number":76,"context_line":"        self.reclaim_age \u003d int(conf.get(\u0027reclaim_age\u0027, 86400 * 7))"},{"line_number":77,"context_line":"        self.do_list_interval \u003d int("},{"line_number":78,"context_line":"            conf.get(\u0027do_list_interval\u0027, 10))"},{"line_number":79,"context_line":"        self.replication_phase \u003d random.randint("},{"line_number":80,"context_line":"            0,"},{"line_number":81,"context_line":"            self.do_list_interval - 1)"}],"source_content_type":"text/x-python","patch_set":2,"id":"5a74a57a_f19cf6c1","line":78,"in_reply_to":"5a74a57a_f796aeee","updated":"2016-11-28 12:03:26.000000000","message":"Sure, so default value is \"10\" and no configuration change is required","commit_id":"11ef7a96fe3777fb0af26b54d87f4d7294748828"},{"author":{"_account_id":6968,"name":"Christian Schwede","email":"cschwede@nvidia.com","username":"cschwede"},"change_message_id":"38a4050c89149775484bfac6ac982ae44b1be2a0","unresolved":false,"context_lines":[{"line_number":78,"context_line":"            conf.get(\u0027do_list_interval\u0027, 10))"},{"line_number":79,"context_line":"        self.replication_phase \u003d random.randint("},{"line_number":80,"context_line":"            0,"},{"line_number":81,"context_line":"            self.do_list_interval - 1)"},{"line_number":82,"context_line":"        self.partition_times \u003d []"},{"line_number":83,"context_line":"        self.interval \u003d int(conf.get(\u0027interval\u0027) or"},{"line_number":84,"context_line":"                            conf.get(\u0027run_pause\u0027) or 30)"}],"source_content_type":"text/x-python","patch_set":2,"id":"5a74a57a_77045e49","line":81,"updated":"2016-11-28 11:13:44.000000000","message":"I don\u0027t think this needs to be a random number - the partition itself is already randomly selected, so we can just start with 0","commit_id":"11ef7a96fe3777fb0af26b54d87f4d7294748828"},{"author":{"_account_id":23449,"name":"Pavel Kvasnička","email":"pavel.kvasnicka@firma.seznam.cz","username":"PavelK"},"change_message_id":"62cd75eb70865f9a5363145a86244c54b4186834","unresolved":false,"context_lines":[{"line_number":78,"context_line":"            conf.get(\u0027do_list_interval\u0027, 10))"},{"line_number":79,"context_line":"        self.replication_phase \u003d random.randint("},{"line_number":80,"context_line":"            0,"},{"line_number":81,"context_line":"            self.do_list_interval - 1)"},{"line_number":82,"context_line":"        self.partition_times \u003d []"},{"line_number":83,"context_line":"        self.interval \u003d int(conf.get(\u0027interval\u0027) or"},{"line_number":84,"context_line":"                            conf.get(\u0027run_pause\u0027) or 30)"}],"source_content_type":"text/x-python","patch_set":2,"id":"5a74a57a_51f68af9","line":81,"in_reply_to":"5a74a57a_77045e49","updated":"2016-11-28 12:03:26.000000000","message":"I think random number is better because someone can use \"run_once\" by CRON.\n\nint(job[\u0027partition\u0027]) does\u0027t deppend on order.","commit_id":"11ef7a96fe3777fb0af26b54d87f4d7294748828"},{"author":{"_account_id":6968,"name":"Christian Schwede","email":"cschwede@nvidia.com","username":"cschwede"},"change_message_id":"38a4050c89149775484bfac6ac982ae44b1be2a0","unresolved":false,"context_lines":[{"line_number":401,"context_line":"        \"\"\""},{"line_number":402,"context_line":"        self.replication_count +\u003d 1"},{"line_number":403,"context_line":"        do_list \u003d ("},{"line_number":404,"context_line":"            (int(job[\u0027partition\u0027]) + self.replication_phase) %"},{"line_number":405,"context_line":"            self.do_list_interval \u003d\u003d 0"},{"line_number":406,"context_line":"        )"},{"line_number":407,"context_line":"        self.logger.increment(\u0027partition.update.count.%s\u0027 % (job[\u0027device\u0027],))"}],"source_content_type":"text/x-python","patch_set":2,"id":"5a74a57a_fabbf3e2","line":404,"updated":"2016-11-28 11:13:44.000000000","message":"This basically means each full replication pass invalidates the hashes.pkl depending on the last digit of the partition number. Looks like a nice solution to me!\n\nHowever, I like this to be a simple method, to make it testable","commit_id":"11ef7a96fe3777fb0af26b54d87f4d7294748828"},{"author":{"_account_id":23449,"name":"Pavel Kvasnička","email":"pavel.kvasnicka@firma.seznam.cz","username":"PavelK"},"change_message_id":"62cd75eb70865f9a5363145a86244c54b4186834","unresolved":false,"context_lines":[{"line_number":401,"context_line":"        \"\"\""},{"line_number":402,"context_line":"        self.replication_count +\u003d 1"},{"line_number":403,"context_line":"        do_list \u003d ("},{"line_number":404,"context_line":"            (int(job[\u0027partition\u0027]) + self.replication_phase) %"},{"line_number":405,"context_line":"            self.do_list_interval \u003d\u003d 0"},{"line_number":406,"context_line":"        )"},{"line_number":407,"context_line":"        self.logger.increment(\u0027partition.update.count.%s\u0027 % (job[\u0027device\u0027],))"}],"source_content_type":"text/x-python","patch_set":2,"id":"5a74a57a_d150dad6","line":404,"in_reply_to":"5a74a57a_fabbf3e2","updated":"2016-11-28 12:03:26.000000000","message":"Sure, can be separated into simple function with tests.","commit_id":"11ef7a96fe3777fb0af26b54d87f4d7294748828"},{"author":{"_account_id":6968,"name":"Christian Schwede","email":"cschwede@nvidia.com","username":"cschwede"},"change_message_id":"b39be57acff930eb1b3fe002d10b6e4c79903e5d","unresolved":false,"context_lines":[{"line_number":413,"context_line":"        try:"},{"line_number":414,"context_line":"            hashed, local_hash \u003d tpool_reraise("},{"line_number":415,"context_line":"                self._diskfile_mgr._get_hashes, job[\u0027path\u0027],"},{"line_number":416,"context_line":"                do_list\u003ddo_list,"},{"line_number":417,"context_line":"                reclaim_age\u003dself.reclaim_age)"},{"line_number":418,"context_line":"            self.suffix_hash +\u003d hashed"},{"line_number":419,"context_line":"            self.logger.update_stats(\u0027suffix.hashes\u0027, hashed)"}],"source_content_type":"text/x-python","patch_set":2,"id":"5a74a57a_77d61ff2","line":416,"updated":"2016-11-28 09:35:21.000000000","message":"This option does not exist, it\u0027s do_listdir","commit_id":"11ef7a96fe3777fb0af26b54d87f4d7294748828"},{"author":{"_account_id":23449,"name":"Pavel Kvasnička","email":"pavel.kvasnicka@firma.seznam.cz","username":"PavelK"},"change_message_id":"62cd75eb70865f9a5363145a86244c54b4186834","unresolved":false,"context_lines":[{"line_number":413,"context_line":"        try:"},{"line_number":414,"context_line":"            hashed, local_hash \u003d tpool_reraise("},{"line_number":415,"context_line":"                self._diskfile_mgr._get_hashes, job[\u0027path\u0027],"},{"line_number":416,"context_line":"                do_list\u003ddo_list,"},{"line_number":417,"context_line":"                reclaim_age\u003dself.reclaim_age)"},{"line_number":418,"context_line":"            self.suffix_hash +\u003d hashed"},{"line_number":419,"context_line":"            self.logger.update_stats(\u0027suffix.hashes\u0027, hashed)"}],"source_content_type":"text/x-python","patch_set":2,"id":"5a74a57a_9194a243","line":416,"in_reply_to":"5a74a57a_77d61ff2","updated":"2016-11-28 12:03:26.000000000","message":"Sure. I had some code prepared for https://review.openstack.org/#/c/402043/ and I wanted to commit this to show my idea, how to simply resolve determinism.","commit_id":"11ef7a96fe3777fb0af26b54d87f4d7294748828"},{"author":{"_account_id":23449,"name":"Pavel Kvasnička","email":"pavel.kvasnicka@firma.seznam.cz","username":"PavelK"},"change_message_id":"f93f00370c3364640fadef45ec3c02e9dd905eb9","unresolved":false,"context_lines":[{"line_number":391,"context_line":""},{"line_number":392,"context_line":"    def _do_listdir(self, partition):"},{"line_number":393,"context_line":"        self.logger.debug(\u0027Run listdir on partition %d\u0027, partition)"},{"line_number":394,"context_line":"        return (((partition + self.replication_phase) % 10) \u003d\u003d 0)"},{"line_number":395,"context_line":""},{"line_number":396,"context_line":"    def update(self, job):"},{"line_number":397,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":3,"id":"5a74a57a_ecb7b9d5","line":394,"updated":"2016-11-28 12:21:49.000000000","message":"Works fine with `run_forever` but lists only 10% of partitions when you use `run_once`. That\u0027s the reason why I used random number for self.replication_phase initialization.","commit_id":"7b4df4a654cf5cf7893a1f3d01dc90cda2153fa3"},{"author":{"_account_id":23449,"name":"Pavel Kvasnička","email":"pavel.kvasnicka@firma.seznam.cz","username":"PavelK"},"change_message_id":"ddb90faaf7001938184383890ee002a7a327a9e9","unresolved":false,"context_lines":[{"line_number":74,"context_line":"        self.ring_check_interval \u003d int(conf.get(\u0027ring_check_interval\u0027, 15))"},{"line_number":75,"context_line":"        self.next_check \u003d time.time() + self.ring_check_interval"},{"line_number":76,"context_line":"        self.reclaim_age \u003d int(conf.get(\u0027reclaim_age\u0027, 86400 * 7))"},{"line_number":77,"context_line":"        self.replication_phase \u003d random.randint(0, 10)"},{"line_number":78,"context_line":"        self.partition_times \u003d []"},{"line_number":79,"context_line":"        self.interval \u003d int(conf.get(\u0027interval\u0027) or"},{"line_number":80,"context_line":"                            conf.get(\u0027run_pause\u0027) or 30)"}],"source_content_type":"text/x-python","patch_set":7,"id":"5a74a57a_0a97703a","line":77,"range":{"start_line":77,"start_character":51,"end_line":77,"end_character":53},"updated":"2016-11-28 13:59:24.000000000","message":"9","commit_id":"f8bd91ba20887b434ea91a8074fd803a52ee8443"},{"author":{"_account_id":6968,"name":"Christian Schwede","email":"cschwede@nvidia.com","username":"cschwede"},"change_message_id":"10636a275c5b46b663de6a77098dd8a86015762e","unresolved":false,"context_lines":[{"line_number":74,"context_line":"        self.ring_check_interval \u003d int(conf.get(\u0027ring_check_interval\u0027, 15))"},{"line_number":75,"context_line":"        self.next_check \u003d time.time() + self.ring_check_interval"},{"line_number":76,"context_line":"        self.reclaim_age \u003d int(conf.get(\u0027reclaim_age\u0027, 86400 * 7))"},{"line_number":77,"context_line":"        self.replication_phase \u003d random.randint(0, 10)"},{"line_number":78,"context_line":"        self.partition_times \u003d []"},{"line_number":79,"context_line":"        self.interval \u003d int(conf.get(\u0027interval\u0027) or"},{"line_number":80,"context_line":"                            conf.get(\u0027run_pause\u0027) or 30)"}],"source_content_type":"text/x-python","patch_set":7,"id":"5a74a57a_a7aa2e83","line":77,"range":{"start_line":77,"start_character":51,"end_line":77,"end_character":53},"in_reply_to":"5a74a57a_0a97703a","updated":"2016-11-28 16:24:35.000000000","message":"Done","commit_id":"f8bd91ba20887b434ea91a8074fd803a52ee8443"},{"author":{"_account_id":23449,"name":"Pavel Kvasnička","email":"pavel.kvasnicka@firma.seznam.cz","username":"PavelK"},"change_message_id":"ddb90faaf7001938184383890ee002a7a327a9e9","unresolved":false,"context_lines":[{"line_number":390,"context_line":"        return success_paths, error_paths"},{"line_number":391,"context_line":""},{"line_number":392,"context_line":"    def _do_listdir(self, partition):"},{"line_number":393,"context_line":"        self.logger.debug(\u0027Run listdir on partition %d\u0027, partition)"},{"line_number":394,"context_line":"        return (((partition + self.replication_phase) % 10) \u003d\u003d 0)"},{"line_number":395,"context_line":""},{"line_number":396,"context_line":"    def update(self, job):"}],"source_content_type":"text/x-python","patch_set":7,"id":"5a74a57a_ca2c9804","line":393,"range":{"start_line":393,"start_character":8,"end_line":393,"end_character":67},"updated":"2016-11-28 13:59:24.000000000","message":"isn\u0027t it better to log only when returning True?","commit_id":"f8bd91ba20887b434ea91a8074fd803a52ee8443"},{"author":{"_account_id":6968,"name":"Christian Schwede","email":"cschwede@nvidia.com","username":"cschwede"},"change_message_id":"10636a275c5b46b663de6a77098dd8a86015762e","unresolved":false,"context_lines":[{"line_number":390,"context_line":"        return success_paths, error_paths"},{"line_number":391,"context_line":""},{"line_number":392,"context_line":"    def _do_listdir(self, partition):"},{"line_number":393,"context_line":"        self.logger.debug(\u0027Run listdir on partition %d\u0027, partition)"},{"line_number":394,"context_line":"        return (((partition + self.replication_phase) % 10) \u003d\u003d 0)"},{"line_number":395,"context_line":""},{"line_number":396,"context_line":"    def update(self, job):"}],"source_content_type":"text/x-python","patch_set":7,"id":"5a74a57a_a7b30e09","line":393,"range":{"start_line":393,"start_character":8,"end_line":393,"end_character":67},"in_reply_to":"5a74a57a_ca2c9804","updated":"2016-11-28 16:24:35.000000000","message":"Done","commit_id":"f8bd91ba20887b434ea91a8074fd803a52ee8443"},{"author":{"_account_id":330,"name":"John Dickinson","email":"me@not.mn","username":"notmyname"},"change_message_id":"32fcfb7e9d988e4a5590a9898979ca04cef6452a","unresolved":false,"context_lines":[{"line_number":389,"context_line":"                        suffix_dir)"},{"line_number":390,"context_line":"        return success_paths, error_paths"},{"line_number":391,"context_line":""},{"line_number":392,"context_line":"    def _do_listdir(self, partition):"},{"line_number":393,"context_line":"        if (((partition + self.replication_phase) % 10) \u003d\u003d 0):"},{"line_number":394,"context_line":"            self.logger.debug(\u0027Run listdir on partition %d\u0027, partition)"},{"line_number":395,"context_line":"            return True"}],"source_content_type":"text/x-python","patch_set":8,"id":"5a74a57a_d53346f4","line":392,"updated":"2016-11-28 23:41:35.000000000","message":"Can we also explicitly return False at the end? I\u0027d rather see that than rely on duck-typing truthiness.","commit_id":"9bd8ec39308bb6c1e8548e62285488eb11d9283a"},{"author":{"_account_id":6968,"name":"Christian Schwede","email":"cschwede@nvidia.com","username":"cschwede"},"change_message_id":"5c718e4f19d885304d13289da5a1b7f8dccb2e97","unresolved":false,"context_lines":[{"line_number":389,"context_line":"                        suffix_dir)"},{"line_number":390,"context_line":"        return success_paths, error_paths"},{"line_number":391,"context_line":""},{"line_number":392,"context_line":"    def _do_listdir(self, partition):"},{"line_number":393,"context_line":"        if (((partition + self.replication_phase) % 10) \u003d\u003d 0):"},{"line_number":394,"context_line":"            self.logger.debug(\u0027Run listdir on partition %d\u0027, partition)"},{"line_number":395,"context_line":"            return True"}],"source_content_type":"text/x-python","patch_set":8,"id":"5a74a57a_fdf3b670","line":392,"in_reply_to":"5a74a57a_d53346f4","updated":"2016-11-29 09:18:48.000000000","message":"Done","commit_id":"9bd8ec39308bb6c1e8548e62285488eb11d9283a"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"91e9b9e5a8b2d1bb7175ff38f3b29d3209858652","unresolved":false,"context_lines":[{"line_number":392,"context_line":"    def _do_listdir(self, partition):"},{"line_number":393,"context_line":"        if (((partition + self.replication_phase) % 10) \u003d\u003d 0):"},{"line_number":394,"context_line":"            self.logger.debug(\u0027Run listdir on partition %d\u0027, partition)"},{"line_number":395,"context_line":"            return True"},{"line_number":396,"context_line":""},{"line_number":397,"context_line":"    def update(self, job):"},{"line_number":398,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":8,"id":"5a74a57a_6fa468b9","line":395,"updated":"2016-11-28 17:30:29.000000000","message":"this seems to target a listdir in 10% of parts each pass (N.B. rings make no effort to balance placement of partition integers, some nodes might listdir 20% of parts in a pass others 0%)","commit_id":"9bd8ec39308bb6c1e8548e62285488eb11d9283a"},{"author":{"_account_id":6968,"name":"Christian Schwede","email":"cschwede@nvidia.com","username":"cschwede"},"change_message_id":"5c718e4f19d885304d13289da5a1b7f8dccb2e97","unresolved":false,"context_lines":[{"line_number":392,"context_line":"    def _do_listdir(self, partition):"},{"line_number":393,"context_line":"        if (((partition + self.replication_phase) % 10) \u003d\u003d 0):"},{"line_number":394,"context_line":"            self.logger.debug(\u0027Run listdir on partition %d\u0027, partition)"},{"line_number":395,"context_line":"            return True"},{"line_number":396,"context_line":""},{"line_number":397,"context_line":"    def update(self, job):"},{"line_number":398,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":8,"id":"5a74a57a_ddf0ba69","line":395,"in_reply_to":"5a74a57a_6fa468b9","updated":"2016-11-29 09:18:48.000000000","message":"That\u0027s true. However, with a large number of partitions per replicator (let\u0027s say 2500 partitions for a medium node) there is a good chance that they are normal distributed, and in that case we should see ~ 10% of hashes invalidated per run, and all of them after the 10th run.","commit_id":"9bd8ec39308bb6c1e8548e62285488eb11d9283a"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7053fd4ca2435ea77a7f6bc5459301ff7d011b0f","unresolved":false,"context_lines":[{"line_number":74,"context_line":"        self.ring_check_interval \u003d int(conf.get(\u0027ring_check_interval\u0027, 15))"},{"line_number":75,"context_line":"        self.next_check \u003d time.time() + self.ring_check_interval"},{"line_number":76,"context_line":"        self.reclaim_age \u003d int(conf.get(\u0027reclaim_age\u0027, 86400 * 7))"},{"line_number":77,"context_line":"        self.replication_phase \u003d random.randint(0, 9)"},{"line_number":78,"context_line":"        self.partition_times \u003d []"},{"line_number":79,"context_line":"        self.interval \u003d int(conf.get(\u0027interval\u0027) or"},{"line_number":80,"context_line":"                            conf.get(\u0027run_pause\u0027) or 30)"}],"source_content_type":"text/x-python","patch_set":9,"id":"5a74a57a_0fb819e5","line":77,"updated":"2016-11-29 18:10:56.000000000","message":"I don\u0027t think it\u0027s important at all that this initialization be undeterministic.\n\nEven when different replicas of a partition on different nodes all get listdir\u0027d in the same pass of a replicator it\u0027s not really going to be at exactly the same time - and the fact that this is just random grants that even if they did it\u0027s not bad - in fact it seems somewhat *more* likely to reduce do-nothing rsyncs if the listdirs were co-ordinated.\n\nIt\u0027s also coupled with the magic number 10 in the modulo below.","commit_id":"48da844fa3264b581941bdb93a2d0fa17c9e9396"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7053fd4ca2435ea77a7f6bc5459301ff7d011b0f","unresolved":false,"context_lines":[{"line_number":390,"context_line":"        return success_paths, error_paths"},{"line_number":391,"context_line":""},{"line_number":392,"context_line":"    def _do_listdir(self, partition):"},{"line_number":393,"context_line":"        if (((partition + self.replication_phase) % 10) \u003d\u003d 0):"},{"line_number":394,"context_line":"            self.logger.debug(\u0027Run listdir on partition %d\u0027, partition)"},{"line_number":395,"context_line":"            return True"},{"line_number":396,"context_line":"        return False"}],"source_content_type":"text/x-python","patch_set":9,"id":"5a74a57a_af2f25a6","line":393,"updated":"2016-11-29 18:10:56.000000000","message":"I think I\u0027ve managed to convince myself all the terms and order of operations are equivalent in either method.\n\n\"3 replicas worth of listdirs on 10% of all parts each pass across N nodes\"\n\n3 * (2 ** P * .1) / N\n\nas\n\n\"listdirs on 10% of each part-replica each pass across N nodes\"\n\n.1 * (3 * 2 ** P) / N\n\nThe original has a slightly better chance that each node does an equivalent number of listdirs, but as long as you have enough parts on each node it\u0027s more likely to be roughly even than not.","commit_id":"48da844fa3264b581941bdb93a2d0fa17c9e9396"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7053fd4ca2435ea77a7f6bc5459301ff7d011b0f","unresolved":false,"context_lines":[{"line_number":391,"context_line":""},{"line_number":392,"context_line":"    def _do_listdir(self, partition):"},{"line_number":393,"context_line":"        if (((partition + self.replication_phase) % 10) \u003d\u003d 0):"},{"line_number":394,"context_line":"            self.logger.debug(\u0027Run listdir on partition %d\u0027, partition)"},{"line_number":395,"context_line":"            return True"},{"line_number":396,"context_line":"        return False"},{"line_number":397,"context_line":""}],"source_content_type":"text/x-python","patch_set":9,"id":"5a74a57a_af3565a9","line":394,"updated":"2016-11-29 18:10:56.000000000","message":"this debug log call should go in diskfile closer to the listdir and include the reference to the device as well as partition (maybe just the path)","commit_id":"48da844fa3264b581941bdb93a2d0fa17c9e9396"},{"author":{"_account_id":6968,"name":"Christian Schwede","email":"cschwede@nvidia.com","username":"cschwede"},"change_message_id":"9cf56c067c6a20859b6445cdbadf917d805a4b9a","unresolved":false,"context_lines":[{"line_number":391,"context_line":""},{"line_number":392,"context_line":"    def _do_listdir(self, partition):"},{"line_number":393,"context_line":"        if (((partition + self.replication_phase) % 10) \u003d\u003d 0):"},{"line_number":394,"context_line":"            self.logger.debug(\u0027Run listdir on partition %d\u0027, partition)"},{"line_number":395,"context_line":"            return True"},{"line_number":396,"context_line":"        return False"},{"line_number":397,"context_line":""}],"source_content_type":"text/x-python","patch_set":9,"id":"5a74a57a_5cd47546","line":394,"in_reply_to":"5a74a57a_af3565a9","updated":"2016-11-30 08:44:19.000000000","message":"Done","commit_id":"48da844fa3264b581941bdb93a2d0fa17c9e9396"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7053fd4ca2435ea77a7f6bc5459301ff7d011b0f","unresolved":false,"context_lines":[{"line_number":411,"context_line":"        try:"},{"line_number":412,"context_line":"            hashed, local_hash \u003d tpool_reraise("},{"line_number":413,"context_line":"                self._diskfile_mgr._get_hashes, job[\u0027path\u0027],"},{"line_number":414,"context_line":"                do_listdir\u003dself._do_listdir(int(job[\u0027partition\u0027])),"},{"line_number":415,"context_line":"                reclaim_age\u003dself.reclaim_age)"},{"line_number":416,"context_line":"            self.suffix_hash +\u003d hashed"},{"line_number":417,"context_line":"            self.logger.update_stats(\u0027suffix.hashes\u0027, hashed)"}],"source_content_type":"text/x-python","patch_set":9,"id":"5a74a57a_8fa9897e","line":414,"updated":"2016-11-29 18:10:56.000000000","message":"no one tests that update calls do_listdir, or that _get_hashes do_listdir option is different depending on the phase \u0026 part #","commit_id":"48da844fa3264b581941bdb93a2d0fa17c9e9396"},{"author":{"_account_id":6968,"name":"Christian Schwede","email":"cschwede@nvidia.com","username":"cschwede"},"change_message_id":"9cf56c067c6a20859b6445cdbadf917d805a4b9a","unresolved":false,"context_lines":[{"line_number":411,"context_line":"        try:"},{"line_number":412,"context_line":"            hashed, local_hash \u003d tpool_reraise("},{"line_number":413,"context_line":"                self._diskfile_mgr._get_hashes, job[\u0027path\u0027],"},{"line_number":414,"context_line":"                do_listdir\u003dself._do_listdir(int(job[\u0027partition\u0027])),"},{"line_number":415,"context_line":"                reclaim_age\u003dself.reclaim_age)"},{"line_number":416,"context_line":"            self.suffix_hash +\u003d hashed"},{"line_number":417,"context_line":"            self.logger.update_stats(\u0027suffix.hashes\u0027, hashed)"}],"source_content_type":"text/x-python","patch_set":9,"id":"5a74a57a_9cf7edf4","line":414,"in_reply_to":"5a74a57a_8fa9897e","updated":"2016-11-30 08:44:19.000000000","message":"Done","commit_id":"48da844fa3264b581941bdb93a2d0fa17c9e9396"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"bde069ea6b1efe0741a18b8182031364212fb7e9","unresolved":false,"context_lines":[{"line_number":699,"context_line":"        self.suffix_hash \u003d 0"},{"line_number":700,"context_line":"        self.replication_count \u003d 0"},{"line_number":701,"context_line":"        self.last_replication_count \u003d -1"},{"line_number":702,"context_line":"        self.replication_phase +\u003d 1"},{"line_number":703,"context_line":"        self.partition_times \u003d []"},{"line_number":704,"context_line":"        self.my_replication_ips \u003d self._get_my_replication_ips()"},{"line_number":705,"context_line":"        self.all_devs_info \u003d set()"}],"source_content_type":"text/x-python","patch_set":9,"id":"5a74a57a_3f7a7dbd","line":702,"updated":"2016-11-29 11:34:52.000000000","message":"NIT: Not that the replicator would ever be running long enough between system updates and Swift upgrades, but Python will keep incrementing this number until you run out of RAM if you let it. So this is kinda the slowest memory leak ever :)\n\nWhich is why it\u0027s only a NIT.  But we could correct this with:\n\n  self.replication_phase \u003d (self.replication_phase + 1) % 10","commit_id":"48da844fa3264b581941bdb93a2d0fa17c9e9396"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7053fd4ca2435ea77a7f6bc5459301ff7d011b0f","unresolved":false,"context_lines":[{"line_number":699,"context_line":"        self.suffix_hash \u003d 0"},{"line_number":700,"context_line":"        self.replication_count \u003d 0"},{"line_number":701,"context_line":"        self.last_replication_count \u003d -1"},{"line_number":702,"context_line":"        self.replication_phase +\u003d 1"},{"line_number":703,"context_line":"        self.partition_times \u003d []"},{"line_number":704,"context_line":"        self.my_replication_ips \u003d self._get_my_replication_ips()"},{"line_number":705,"context_line":"        self.all_devs_info \u003d set()"}],"source_content_type":"text/x-python","patch_set":9,"id":"5a74a57a_eff11d03","line":702,"in_reply_to":"5a74a57a_080e65e4","updated":"2016-11-29 18:10:56.000000000","message":"lots of other places in this source file we refer to each invocation and completion of the replicate method as a \"pass\" or \"cycle\" - and that\u0027s really what this number is tracking.\n\nI think replication_pass_count or replication_cycle_count would be better.\n\nAlso this counter isn\u0027t tested.","commit_id":"48da844fa3264b581941bdb93a2d0fa17c9e9396"},{"author":{"_account_id":6968,"name":"Christian Schwede","email":"cschwede@nvidia.com","username":"cschwede"},"change_message_id":"08b1deac8cc95e87ac724f315c558c1ba60a673f","unresolved":false,"context_lines":[{"line_number":699,"context_line":"        self.suffix_hash \u003d 0"},{"line_number":700,"context_line":"        self.replication_count \u003d 0"},{"line_number":701,"context_line":"        self.last_replication_count \u003d -1"},{"line_number":702,"context_line":"        self.replication_phase +\u003d 1"},{"line_number":703,"context_line":"        self.partition_times \u003d []"},{"line_number":704,"context_line":"        self.my_replication_ips \u003d self._get_my_replication_ips()"},{"line_number":705,"context_line":"        self.all_devs_info \u003d set()"}],"source_content_type":"text/x-python","patch_set":9,"id":"5a74a57a_080e65e4","line":702,"in_reply_to":"5a74a57a_3f7a7dbd","updated":"2016-11-29 12:14:21.000000000","message":"Heh, I was already wondering who stumbles upon this first!\n\nI don\u0027t think you will ever run out of memory due to this. Using a 32 bit integer with the default interval of 30 seconds should be sufficient for 4085 years.\n\nI\u0027d be worried to see Swift itself still running in 4000 years ;)","commit_id":"48da844fa3264b581941bdb93a2d0fa17c9e9396"},{"author":{"_account_id":6968,"name":"Christian Schwede","email":"cschwede@nvidia.com","username":"cschwede"},"change_message_id":"9cf56c067c6a20859b6445cdbadf917d805a4b9a","unresolved":false,"context_lines":[{"line_number":699,"context_line":"        self.suffix_hash \u003d 0"},{"line_number":700,"context_line":"        self.replication_count \u003d 0"},{"line_number":701,"context_line":"        self.last_replication_count \u003d -1"},{"line_number":702,"context_line":"        self.replication_phase +\u003d 1"},{"line_number":703,"context_line":"        self.partition_times \u003d []"},{"line_number":704,"context_line":"        self.my_replication_ips \u003d self._get_my_replication_ips()"},{"line_number":705,"context_line":"        self.all_devs_info \u003d set()"}],"source_content_type":"text/x-python","patch_set":9,"id":"5a74a57a_7cf21108","line":702,"in_reply_to":"5a74a57a_eff11d03","updated":"2016-11-30 08:44:19.000000000","message":"Done","commit_id":"48da844fa3264b581941bdb93a2d0fa17c9e9396"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"753e793c3173071e554191b94eb30dd5d853a34d","unresolved":false,"context_lines":[{"line_number":390,"context_line":"        return success_paths, error_paths"},{"line_number":391,"context_line":""},{"line_number":392,"context_line":"    def _do_listdir(self, partition, replication_cycle_count):"},{"line_number":393,"context_line":"        return (((partition + replication_cycle_count) % 10) \u003d\u003d 0)"},{"line_number":394,"context_line":""},{"line_number":395,"context_line":"    def update(self, job):"},{"line_number":396,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":10,"id":"3a71b18c_be875611","line":393,"updated":"2016-12-01 10:45:56.000000000","message":"this function doesn\u0027t use self","commit_id":"3c730abb8bd00d410466d5e8a3dbe14bc8c5e2e6"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"65e5df0d7ebb51bc9db20531e7e8a6f6f786f15d","unresolved":false,"context_lines":[{"line_number":390,"context_line":"        return success_paths, error_paths"},{"line_number":391,"context_line":""},{"line_number":392,"context_line":"    def _do_listdir(self, partition, replication_cycle_count):"},{"line_number":393,"context_line":"        return (((partition + replication_cycle_count) % 10) \u003d\u003d 0)"},{"line_number":394,"context_line":""},{"line_number":395,"context_line":"    def update(self, job):"},{"line_number":396,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":10,"id":"3a71b18c_bc490cd3","line":393,"in_reply_to":"3a71b18c_be875611","updated":"2016-12-01 18:16:01.000000000","message":"the same effect would be achieved with \n\n  return partition % 10 \u003d\u003d replication_cycle_count\n\nand looks simpler to me","commit_id":"3c730abb8bd00d410466d5e8a3dbe14bc8c5e2e6"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"753e793c3173071e554191b94eb30dd5d853a34d","unresolved":false,"context_lines":[{"line_number":410,"context_line":"                self._diskfile_mgr._get_hashes, job[\u0027path\u0027],"},{"line_number":411,"context_line":"                do_listdir\u003dself._do_listdir("},{"line_number":412,"context_line":"                    int(job[\u0027partition\u0027]),"},{"line_number":413,"context_line":"                    self.replication_cycle_count),"},{"line_number":414,"context_line":"                reclaim_age\u003dself.reclaim_age)"},{"line_number":415,"context_line":"            self.suffix_hash +\u003d hashed"},{"line_number":416,"context_line":"            self.logger.update_stats(\u0027suffix.hashes\u0027, hashed)"}],"source_content_type":"text/x-python","patch_set":10,"id":"3a71b18c_decb321d","line":413,"updated":"2016-12-01 10:45:56.000000000","message":"the way this is checked you add `or True` and all tests would still pass\n\n\tdiff --git a/swift/obj/replicator.py b/swift/obj/replicator.py\n\tindex 9b90620..0985173 100644\n\t--- a/swift/obj/replicator.py\n\t+++ b/swift/obj/replicator.py\n\t@@ -410,7 +410,7 @@ class ObjectReplicator(Daemon):\n\t\t\t self._diskfile_mgr._get_hashes, job[\u0027path\u0027],\n\t\t\t do_listdir\u003dself._do_listdir(\n\t\t\t     int(job[\u0027partition\u0027]),\n\t-                    self.replication_cycle_count),\n\t+                    self.replication_cycle_count) and False,\n\t\t\t reclaim_age\u003dself.reclaim_age)\n\t\t     self.suffix_hash +\u003d hashed\n\t\t     self.logger.update_stats(\u0027suffix.hashes\u0027, hashed)","commit_id":"3c730abb8bd00d410466d5e8a3dbe14bc8c5e2e6"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"753e793c3173071e554191b94eb30dd5d853a34d","unresolved":false,"context_lines":[{"line_number":698,"context_line":"        self.suffix_hash \u003d 0"},{"line_number":699,"context_line":"        self.replication_count \u003d 0"},{"line_number":700,"context_line":"        self.last_replication_count \u003d -1"},{"line_number":701,"context_line":"        self.replication_cycle_count \u003d (self.replication_cycle_count + 1) % 10"},{"line_number":702,"context_line":"        self.partition_times \u003d []"},{"line_number":703,"context_line":"        self.my_replication_ips \u003d self._get_my_replication_ips()"},{"line_number":704,"context_line":"        self.all_devs_info \u003d set()"}],"source_content_type":"text/x-python","patch_set":10,"id":"3a71b18c_feaaae84","line":701,"updated":"2016-12-01 10:45:56.000000000","message":"well it\u0027s not really a *count* anymore :\\\n\nand it\u0027s still not tested\n\n\tdiff --git a/swift/obj/replicator.py b/swift/obj/replicator.py\n\tindex 9b90620..a98846f 100644\n\t--- a/swift/obj/replicator.py\n\t+++ b/swift/obj/replicator.py\n\t@@ -698,7 +698,7 @@ class ObjectReplicator(Daemon):\n\t\t self.suffix_hash \u003d 0\n\t\t self.replication_count \u003d 0\n\t\t self.last_replication_count \u003d -1\n\t-        self.replication_cycle_count \u003d (self.replication_cycle_count + 1) % 10\n\t+        self.replication_cycle_count +\u003d 42\n\t\t self.partition_times \u003d []\n\t\t self.my_replication_ips \u003d self._get_my_replication_ips()\n\t\t self.all_devs_info \u003d set()","commit_id":"3c730abb8bd00d410466d5e8a3dbe14bc8c5e2e6"}],"test/unit/obj/test_replicator.py":[{"author":{"_account_id":23449,"name":"Pavel Kvasnička","email":"pavel.kvasnicka@firma.seznam.cz","username":"PavelK"},"change_message_id":"3a143e34de80fb25ef02c662a6defec0712ab618","unresolved":false,"context_lines":[{"line_number":1915,"context_line":"                        invalidated +\u003d 1"},{"line_number":1916,"context_line":"                self.replicator.replication_phase +\u003d 1"},{"line_number":1917,"context_line":"                # Every 10th partition is seen after each phase"},{"line_number":1918,"context_line":"                self.assertEqual(expected[phase], invalidated)"},{"line_number":1919,"context_line":""},{"line_number":1920,"context_line":"            # After 10 cycles every partition is seen exactly once"},{"line_number":1921,"context_line":"            self.assertEqual(sorted(range(partitions)), sorted(seen))"}],"source_content_type":"text/x-python","patch_set":4,"id":"5a74a57a_cc011d08","line":1918,"updated":"2016-11-28 12:28:38.000000000","message":"self.replicator.replication_phase is random now, so \"expected \u003d\u003d invalidated or expected - 1 \u003d\u003d invalidated\" is True","commit_id":"bd12416e7866203838580a2514b44bb3034c2542"},{"author":{"_account_id":23449,"name":"Pavel Kvasnička","email":"pavel.kvasnicka@firma.seznam.cz","username":"PavelK"},"change_message_id":"a0cf1c580ab28e9d796454fa00f046ec893a41c7","unresolved":false,"context_lines":[{"line_number":1915,"context_line":"                        invalidated +\u003d 1"},{"line_number":1916,"context_line":"                self.replicator.replication_phase +\u003d 1"},{"line_number":1917,"context_line":"                # Every 10th partition is seen after each phase"},{"line_number":1918,"context_line":"                self.assertEqual(expected[phase], invalidated)"},{"line_number":1919,"context_line":""},{"line_number":1920,"context_line":"            # After 10 cycles every partition is seen exactly once"},{"line_number":1921,"context_line":"            self.assertEqual(sorted(range(partitions)), sorted(seen))"}],"source_content_type":"text/x-python","patch_set":4,"id":"5a74a57a_e0b7bdd5","line":1918,"in_reply_to":"5a74a57a_cc011d08","updated":"2016-11-28 12:38:56.000000000","message":"## for example:\n\n                self.assertTrue(expected[0] \u003e\u003d invalidated \u003e\u003d expected[0])\n\n            # After 10 cycles every partition is seen exactly once\n            self.assertEqual(sum(partitions)), sum(seen))","commit_id":"bd12416e7866203838580a2514b44bb3034c2542"},{"author":{"_account_id":6968,"name":"Christian Schwede","email":"cschwede@nvidia.com","username":"cschwede"},"change_message_id":"508678b8ee8f72dd17b48b8ced2856718d901e0a","unresolved":false,"context_lines":[{"line_number":1915,"context_line":"                        invalidated +\u003d 1"},{"line_number":1916,"context_line":"                self.replicator.replication_phase +\u003d 1"},{"line_number":1917,"context_line":"                # Every 10th partition is seen after each phase"},{"line_number":1918,"context_line":"                self.assertEqual(expected[phase], invalidated)"},{"line_number":1919,"context_line":""},{"line_number":1920,"context_line":"            # After 10 cycles every partition is seen exactly once"},{"line_number":1921,"context_line":"            self.assertEqual(sorted(range(partitions)), sorted(seen))"}],"source_content_type":"text/x-python","patch_set":4,"id":"5a74a57a_4a432842","line":1918,"in_reply_to":"5a74a57a_cc011d08","updated":"2016-11-28 13:48:15.000000000","message":"Bummer, forgot that one.\n\nI set this to 0 in the tests, to get the same results in every run","commit_id":"bd12416e7866203838580a2514b44bb3034c2542"},{"author":{"_account_id":6968,"name":"Christian Schwede","email":"cschwede@nvidia.com","username":"cschwede"},"change_message_id":"508678b8ee8f72dd17b48b8ced2856718d901e0a","unresolved":false,"context_lines":[{"line_number":1915,"context_line":"                        invalidated +\u003d 1"},{"line_number":1916,"context_line":"                self.replicator.replication_phase +\u003d 1"},{"line_number":1917,"context_line":"                # Every 10th partition is seen after each phase"},{"line_number":1918,"context_line":"                self.assertEqual(expected[phase], invalidated)"},{"line_number":1919,"context_line":""},{"line_number":1920,"context_line":"            # After 10 cycles every partition is seen exactly once"},{"line_number":1921,"context_line":"            self.assertEqual(sorted(range(partitions)), sorted(seen))"}],"source_content_type":"text/x-python","patch_set":4,"id":"5a74a57a_0a1e905e","line":1918,"in_reply_to":"5a74a57a_e0b7bdd5","updated":"2016-11-28 13:48:15.000000000","message":"This is not explicit - for example one pass might add 5 partitions, and the next 15 instead of 10 in both cases; but the sum is the same","commit_id":"bd12416e7866203838580a2514b44bb3034c2542"},{"author":{"_account_id":6968,"name":"Christian Schwede","email":"cschwede@nvidia.com","username":"cschwede"},"change_message_id":"5a35885c2f7ef7d029e2518aa652c6275f647896","unresolved":false,"context_lines":[{"line_number":1904,"context_line":"            9: [1, 0, 1, 1, 1, 1, 1, 1, 1, 1],"},{"line_number":1905,"context_line":"            29: [3, 2, 3, 3, 3, 3, 3, 3, 3, 3],"},{"line_number":1906,"context_line":"            111: [12, 11, 11, 11, 11, 11, 11, 11, 11, 11]}"},{"line_number":1907,"context_line":""},{"line_number":1908,"context_line":"        for partitions, expected in test_data.items():"},{"line_number":1909,"context_line":"            seen \u003d []"},{"line_number":1910,"context_line":"            for phase in range(10):"}],"source_content_type":"text/x-python","patch_set":6,"id":"5a74a57a_4adfa868","line":1907,"updated":"2016-11-28 13:52:28.000000000","message":"I would prefer to make this non-random and explicitly test the above exprected outcome like before\n\n self.replicator.replication_phase \u003d 0","commit_id":"599896e66fb3cfefb249ea447ee14304dc5c9b67"},{"author":{"_account_id":6968,"name":"Christian Schwede","email":"cschwede@nvidia.com","username":"cschwede"},"change_message_id":"5a35885c2f7ef7d029e2518aa652c6275f647896","unresolved":false,"context_lines":[{"line_number":1918,"context_line":"                self.assertTrue(expected[0] \u003e\u003d invalidated \u003e\u003d expected[0] - 1)"},{"line_number":1919,"context_line":""},{"line_number":1920,"context_line":"            # After 10 cycles every partition is seen exactly once"},{"line_number":1921,"context_line":"            self.assertEqual(sum(partitions)), sum(seen))"},{"line_number":1922,"context_line":""},{"line_number":1923,"context_line":""},{"line_number":1924,"context_line":"if __name__ \u003d\u003d \u0027__main__\u0027:"}],"source_content_type":"text/x-python","patch_set":6,"id":"5a74a57a_8abc6007","line":1921,"updated":"2016-11-28 13:52:28.000000000","message":"This is not explicit - for example one pass might add 5 partitions, and the next 15 instead of 10 in both cases; but the sum is the same","commit_id":"599896e66fb3cfefb249ea447ee14304dc5c9b67"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7053fd4ca2435ea77a7f6bc5459301ff7d011b0f","unresolved":false,"context_lines":[{"line_number":1905,"context_line":"            29: [3, 2, 3, 3, 3, 3, 3, 3, 3, 3],"},{"line_number":1906,"context_line":"            111: [12, 11, 11, 11, 11, 11, 11, 11, 11, 11]}"},{"line_number":1907,"context_line":""},{"line_number":1908,"context_line":"        self.replicator.replication_phase \u003d 0"},{"line_number":1909,"context_line":""},{"line_number":1910,"context_line":"        for partitions, expected in test_data.items():"},{"line_number":1911,"context_line":"            seen \u003d []"}],"source_content_type":"text/x-python","patch_set":9,"id":"5a74a57a_6fc7cd64","line":1908,"updated":"2016-11-29 18:10:56.000000000","message":"it\u0027s annoying that this initialization isn\u0027t deterministic","commit_id":"48da844fa3264b581941bdb93a2d0fa17c9e9396"},{"author":{"_account_id":6968,"name":"Christian Schwede","email":"cschwede@nvidia.com","username":"cschwede"},"change_message_id":"9cf56c067c6a20859b6445cdbadf917d805a4b9a","unresolved":false,"context_lines":[{"line_number":1905,"context_line":"            29: [3, 2, 3, 3, 3, 3, 3, 3, 3, 3],"},{"line_number":1906,"context_line":"            111: [12, 11, 11, 11, 11, 11, 11, 11, 11, 11]}"},{"line_number":1907,"context_line":""},{"line_number":1908,"context_line":"        self.replicator.replication_phase \u003d 0"},{"line_number":1909,"context_line":""},{"line_number":1910,"context_line":"        for partitions, expected in test_data.items():"},{"line_number":1911,"context_line":"            seen \u003d []"}],"source_content_type":"text/x-python","patch_set":9,"id":"5a74a57a_9ceb6db0","line":1908,"in_reply_to":"5a74a57a_6fc7cd64","updated":"2016-11-30 08:44:19.000000000","message":"Done","commit_id":"48da844fa3264b581941bdb93a2d0fa17c9e9396"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7053fd4ca2435ea77a7f6bc5459301ff7d011b0f","unresolved":false,"context_lines":[{"line_number":1912,"context_line":"            for phase in range(10):"},{"line_number":1913,"context_line":"                invalidated \u003d 0"},{"line_number":1914,"context_line":"                for partition in range(partitions):"},{"line_number":1915,"context_line":"                    if self.replicator._do_listdir(partition):"},{"line_number":1916,"context_line":"                        seen.append(partition)"},{"line_number":1917,"context_line":"                        invalidated +\u003d 1"},{"line_number":1918,"context_line":"                self.replicator.replication_phase +\u003d 1"}],"source_content_type":"text/x-python","patch_set":9,"id":"5a74a57a_0f9d7959","line":1915,"updated":"2016-11-29 18:10:56.000000000","message":"if we want to test this function as a unit - maybe it shouldn\u0027t have the dependency on the internal state of the replicator and should just take the phase/count/pass/cycle arg as a parameter so it can be a real function.","commit_id":"48da844fa3264b581941bdb93a2d0fa17c9e9396"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"7053fd4ca2435ea77a7f6bc5459301ff7d011b0f","unresolved":false,"context_lines":[{"line_number":1915,"context_line":"                    if self.replicator._do_listdir(partition):"},{"line_number":1916,"context_line":"                        seen.append(partition)"},{"line_number":1917,"context_line":"                        invalidated +\u003d 1"},{"line_number":1918,"context_line":"                self.replicator.replication_phase +\u003d 1"},{"line_number":1919,"context_line":"                # Every 10th partition is seen after each phase"},{"line_number":1920,"context_line":"                self.assertEqual(expected[phase], invalidated)"},{"line_number":1921,"context_line":""}],"source_content_type":"text/x-python","patch_set":9,"id":"5a74a57a_0faf3995","line":1918,"updated":"2016-11-29 18:10:56.000000000","message":"it seems weird that this test has to manipulate this internal state","commit_id":"48da844fa3264b581941bdb93a2d0fa17c9e9396"},{"author":{"_account_id":6968,"name":"Christian Schwede","email":"cschwede@nvidia.com","username":"cschwede"},"change_message_id":"9cf56c067c6a20859b6445cdbadf917d805a4b9a","unresolved":false,"context_lines":[{"line_number":1915,"context_line":"                    if self.replicator._do_listdir(partition):"},{"line_number":1916,"context_line":"                        seen.append(partition)"},{"line_number":1917,"context_line":"                        invalidated +\u003d 1"},{"line_number":1918,"context_line":"                self.replicator.replication_phase +\u003d 1"},{"line_number":1919,"context_line":"                # Every 10th partition is seen after each phase"},{"line_number":1920,"context_line":"                self.assertEqual(expected[phase], invalidated)"},{"line_number":1921,"context_line":""}],"source_content_type":"text/x-python","patch_set":9,"id":"5a74a57a_bc1f4908","line":1918,"in_reply_to":"5a74a57a_0faf3995","updated":"2016-11-30 08:44:19.000000000","message":"Done","commit_id":"48da844fa3264b581941bdb93a2d0fa17c9e9396"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"65e5df0d7ebb51bc9db20531e7e8a6f6f786f15d","unresolved":false,"context_lines":[{"line_number":1722,"context_line":"        error \u003d \u0027%(replication_ip)s/%(device)s responded as unmounted\u0027"},{"line_number":1723,"context_line":"        expect \u003d \u0027Error syncing partition: \u0027"},{"line_number":1724,"context_line":"        expected_listdir_calls \u003d ["},{"line_number":1725,"context_line":"            mock.call((os.path.basename(job[\u0027path\u0027]),"},{"line_number":1726,"context_line":"                       self.replicator.replication_cycle_count))"},{"line_number":1727,"context_line":"            for job in jobs]"},{"line_number":1728,"context_line":"        for job in jobs:"}],"source_content_type":"text/x-python","patch_set":10,"id":"3a71b18c_1b1e83ba","line":1725,"updated":"2016-12-01 18:16:01.000000000","message":"should be int(job[\u0027partition\u0027]) and as Clay says there\u0027s extra parentheses that cause the (fixed) test to fail","commit_id":"3c730abb8bd00d410466d5e8a3dbe14bc8c5e2e6"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"753e793c3173071e554191b94eb30dd5d853a34d","unresolved":false,"context_lines":[{"line_number":1724,"context_line":"        expected_listdir_calls \u003d ["},{"line_number":1725,"context_line":"            mock.call((os.path.basename(job[\u0027path\u0027]),"},{"line_number":1726,"context_line":"                       self.replicator.replication_cycle_count))"},{"line_number":1727,"context_line":"            for job in jobs]"},{"line_number":1728,"context_line":"        for job in jobs:"},{"line_number":1729,"context_line":"            set_default(self)"},{"line_number":1730,"context_line":"            ring \u003d job[\u0027policy\u0027].object_ring"}],"source_content_type":"text/x-python","patch_set":10,"id":"3a71b18c_be7836fa","line":1727,"updated":"2016-12-01 10:45:56.000000000","message":"oic, every job is called in the same cycle (whatever it was randomly initialized too)","commit_id":"3c730abb8bd00d410466d5e8a3dbe14bc8c5e2e6"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"9b66cb3eb7c0c885624dbde3a5b5bbbcfb565681","unresolved":false,"context_lines":[{"line_number":1724,"context_line":"        expected_listdir_calls \u003d ["},{"line_number":1725,"context_line":"            mock.call((os.path.basename(job[\u0027path\u0027]),"},{"line_number":1726,"context_line":"                       self.replicator.replication_cycle_count))"},{"line_number":1727,"context_line":"            for job in jobs]"},{"line_number":1728,"context_line":"        for job in jobs:"},{"line_number":1729,"context_line":"            set_default(self)"},{"line_number":1730,"context_line":"            ring \u003d job[\u0027policy\u0027].object_ring"}],"source_content_type":"text/x-python","patch_set":10,"id":"3a71b18c_b41135b2","line":1727,"in_reply_to":"3a71b18c_be7836fa","updated":"2016-12-01 11:53:49.000000000","message":"I think these parens are spurious, maybe even the first arg should be an int","commit_id":"3c730abb8bd00d410466d5e8a3dbe14bc8c5e2e6"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"753e793c3173071e554191b94eb30dd5d853a34d","unresolved":false,"context_lines":[{"line_number":1754,"context_line":"            mock_http.assert_has_calls(reqs, any_order\u003dTrue)"},{"line_number":1755,"context_line":"            mock_http.reset_mock()"},{"line_number":1756,"context_line":"            self.logger.clear()"},{"line_number":1757,"context_line":"        self.replicator._do_listdir.has_calls(expected_listdir_calls)"},{"line_number":1758,"context_line":"        # Check uncorrect http_connect with status 400 !\u003d HTTP_OK"},{"line_number":1759,"context_line":"        resp.status \u003d 400"},{"line_number":1760,"context_line":"        error \u003d \u0027Invalid response %(resp)s from %(ip)s\u0027"}],"source_content_type":"text/x-python","patch_set":10,"id":"3a71b18c_1e884ae7","line":1757,"updated":"2016-12-01 10:45:56.000000000","message":"nice!","commit_id":"3c730abb8bd00d410466d5e8a3dbe14bc8c5e2e6"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"9b66cb3eb7c0c885624dbde3a5b5bbbcfb565681","unresolved":false,"context_lines":[{"line_number":1754,"context_line":"            mock_http.assert_has_calls(reqs, any_order\u003dTrue)"},{"line_number":1755,"context_line":"            mock_http.reset_mock()"},{"line_number":1756,"context_line":"            self.logger.clear()"},{"line_number":1757,"context_line":"        self.replicator._do_listdir.has_calls(expected_listdir_calls)"},{"line_number":1758,"context_line":"        # Check uncorrect http_connect with status 400 !\u003d HTTP_OK"},{"line_number":1759,"context_line":"        resp.status \u003d 400"},{"line_number":1760,"context_line":"        error \u003d \u0027Invalid response %(resp)s from %(ip)s\u0027"}],"source_content_type":"text/x-python","patch_set":10,"id":"3a71b18c_d493114e","line":1757,"in_reply_to":"3a71b18c_1e884ae7","updated":"2016-12-01 11:53:49.000000000","message":"Umm... I think should be like \u0027assert_has_calls\u0027\n\n\tdiff --git a/test/unit/obj/test_replicator.py b/test/unit/obj/test_replicator.py\n\tindex 032c3ec..cd268c9 100644\n\t--- a/test/unit/obj/test_replicator.py\n\t+++ b/test/unit/obj/test_replicator.py\n\t@@ -1724,7 +1724,7 @@ class TestObjectReplicator(unittest.TestCase):\n\t\t expected_listdir_calls \u003d [\n\t\t     mock.call((os.path.basename(job[\u0027path\u0027]),\n\t\t\t\tself.replicator.replication_cycle_count))\n\t-            for job in jobs]\n\t+            for job in jobs] + [\u0027foo\u0027]\n\t\t for job in jobs:\n\t\t     set_default(self)\n\t\t     ring \u003d job[\u0027policy\u0027].object_ring","commit_id":"3c730abb8bd00d410466d5e8a3dbe14bc8c5e2e6"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"65e5df0d7ebb51bc9db20531e7e8a6f6f786f15d","unresolved":false,"context_lines":[{"line_number":1754,"context_line":"            mock_http.assert_has_calls(reqs, any_order\u003dTrue)"},{"line_number":1755,"context_line":"            mock_http.reset_mock()"},{"line_number":1756,"context_line":"            self.logger.clear()"},{"line_number":1757,"context_line":"        self.replicator._do_listdir.has_calls(expected_listdir_calls)"},{"line_number":1758,"context_line":"        # Check uncorrect http_connect with status 400 !\u003d HTTP_OK"},{"line_number":1759,"context_line":"        resp.status \u003d 400"},{"line_number":1760,"context_line":"        error \u003d \u0027Invalid response %(resp)s from %(ip)s\u0027"}],"source_content_type":"text/x-python","patch_set":10,"id":"3a71b18c_fb58e758","line":1757,"in_reply_to":"3a71b18c_d493114e","updated":"2016-12-01 18:16:01.000000000","message":"Yep, test will fail if fixed to call assert_has_calls \n\nThis test only verifies that replicator._do_listdir gets called with expected args, but not that its result is ever used or has the desired effect.","commit_id":"3c730abb8bd00d410466d5e8a3dbe14bc8c5e2e6"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"65e5df0d7ebb51bc9db20531e7e8a6f6f786f15d","unresolved":false,"context_lines":[{"line_number":1901,"context_line":"                            _m_os_path_exists.call_args_list[-2][0][0],"},{"line_number":1902,"context_line":"                            os.path.join(job[\u0027path\u0027]))"},{"line_number":1903,"context_line":""},{"line_number":1904,"context_line":"    def test_do_listdir(self):"},{"line_number":1905,"context_line":"        # Test if do_listdir is enabled for every 10th partition to rehash"},{"line_number":1906,"context_line":"        # First number is the number of partitions in the job, list entries"},{"line_number":1907,"context_line":"        # are the expected partition numbers per run"}],"source_content_type":"text/x-python","patch_set":10,"id":"3a71b18c_9bb9b3ec","line":1904,"updated":"2016-12-01 18:16:01.000000000","message":"ok this test verifies the isolated behaviour of _do_listdir","commit_id":"3c730abb8bd00d410466d5e8a3dbe14bc8c5e2e6"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"65e5df0d7ebb51bc9db20531e7e8a6f6f786f15d","unresolved":false,"context_lines":[{"line_number":1916,"context_line":"                invalidated \u003d 0"},{"line_number":1917,"context_line":"                for partition in range(partitions):"},{"line_number":1918,"context_line":"                    if self.replicator._do_listdir(partition, phase):"},{"line_number":1919,"context_line":"                        seen.append(partition)"},{"line_number":1920,"context_line":"                        invalidated +\u003d 1"},{"line_number":1921,"context_line":"                # Every 10th partition is seen after each phase"},{"line_number":1922,"context_line":"                self.assertEqual(expected[phase], invalidated)"}],"source_content_type":"text/x-python","patch_set":10,"id":"3a71b18c_184cc9c6","line":1919,"updated":"2016-12-01 18:16:01.000000000","message":"nit: you could use a set to accumulate partitions then compare with set(range(partitions)) below","commit_id":"3c730abb8bd00d410466d5e8a3dbe14bc8c5e2e6"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"753e793c3173071e554191b94eb30dd5d853a34d","unresolved":false,"context_lines":[{"line_number":1922,"context_line":"                self.assertEqual(expected[phase], invalidated)"},{"line_number":1923,"context_line":""},{"line_number":1924,"context_line":"            # After 10 cycles every partition is seen exactly once"},{"line_number":1925,"context_line":"            self.assertEqual(sorted(range(partitions)), sorted(seen))"},{"line_number":1926,"context_line":""},{"line_number":1927,"context_line":""},{"line_number":1928,"context_line":"if __name__ \u003d\u003d \u0027__main__\u0027:"}],"source_content_type":"text/x-python","patch_set":10,"id":"3a71b18c_9ec1ba31","line":1925,"updated":"2016-12-01 10:45:56.000000000","message":"this is a pretty good clean test of a pure function","commit_id":"3c730abb8bd00d410466d5e8a3dbe14bc8c5e2e6"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"65e5df0d7ebb51bc9db20531e7e8a6f6f786f15d","unresolved":false,"context_lines":[{"line_number":1922,"context_line":"                self.assertEqual(expected[phase], invalidated)"},{"line_number":1923,"context_line":""},{"line_number":1924,"context_line":"            # After 10 cycles every partition is seen exactly once"},{"line_number":1925,"context_line":"            self.assertEqual(sorted(range(partitions)), sorted(seen))"},{"line_number":1926,"context_line":""},{"line_number":1927,"context_line":""},{"line_number":1928,"context_line":"if __name__ \u003d\u003d \u0027__main__\u0027:"}],"source_content_type":"text/x-python","patch_set":10,"id":"3a71b18c_98b539cf","line":1925,"in_reply_to":"3a71b18c_9ec1ba31","updated":"2016-12-01 18:16:01.000000000","message":"the range(partitions) does not need to be sorted","commit_id":"3c730abb8bd00d410466d5e8a3dbe14bc8c5e2e6"}]}