)]}'
{"swift/common/db.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"f94b29ee1190860a7aedd64ba054c6ef809a3ab8","unresolved":true,"context_lines":[{"line_number":93,"context_line":"            return []"},{"line_number":94,"context_line":"        raise"},{"line_number":95,"context_line":"    if not files:"},{"line_number":96,"context_line":"        return []"},{"line_number":97,"context_line":"    results \u003d []"},{"line_number":98,"context_line":"    for f in files:"},{"line_number":99,"context_line":"        _root, ext \u003d os.path.splitext(f)"}],"source_content_type":"text/x-python","patch_set":8,"id":"8af9a0d6_3b0288af","line":96,"updated":"2022-03-24 23:30:31.000000000","message":"nit: We could drop this clause -- it\u0027ll get handled by the for loop anyway,","commit_id":"38a6616894f99b99e97e6c793c74848dfec482c4"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"6da015605349bf6e143dc3a6f03f425758c34412","unresolved":false,"context_lines":[{"line_number":93,"context_line":"            return []"},{"line_number":94,"context_line":"        raise"},{"line_number":95,"context_line":"    if not files:"},{"line_number":96,"context_line":"        return []"},{"line_number":97,"context_line":"    results \u003d []"},{"line_number":98,"context_line":"    for f in files:"},{"line_number":99,"context_line":"        _root, ext \u003d os.path.splitext(f)"}],"source_content_type":"text/x-python","patch_set":8,"id":"0aa11cc7_44bb5ac0","line":96,"in_reply_to":"8af9a0d6_3b0288af","updated":"2022-03-25 05:37:41.000000000","message":"Done","commit_id":"38a6616894f99b99e97e6c793c74848dfec482c4"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"f94b29ee1190860a7aedd64ba054c6ef809a3ab8","unresolved":true,"context_lines":[{"line_number":353,"context_line":"        self.conn \u003d None"},{"line_number":354,"context_line":"        self._db_file \u003d db_file"},{"line_number":355,"context_line":"        if not pending_shards:"},{"line_number":356,"context_line":"            pending_shards \u003d 1"},{"line_number":357,"context_line":"        self.pending_files \u003d ["},{"line_number":358,"context_line":"            get_zero_indexed_base_string(self._db_file, i) + \u0027.pending\u0027"},{"line_number":359,"context_line":"            for i in range(pending_shards)]"}],"source_content_type":"text/x-python","patch_set":8,"id":"b6e92f6e_63ea5429","line":356,"updated":"2022-03-24 23:30:31.000000000","message":"When would we pass in a False-y pending_shards value? Or are we just being defensive?","commit_id":"38a6616894f99b99e97e6c793c74848dfec482c4"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"6da015605349bf6e143dc3a6f03f425758c34412","unresolved":true,"context_lines":[{"line_number":353,"context_line":"        self.conn \u003d None"},{"line_number":354,"context_line":"        self._db_file \u003d db_file"},{"line_number":355,"context_line":"        if not pending_shards:"},{"line_number":356,"context_line":"            pending_shards \u003d 1"},{"line_number":357,"context_line":"        self.pending_files \u003d ["},{"line_number":358,"context_line":"            get_zero_indexed_base_string(self._db_file, i) + \u0027.pending\u0027"},{"line_number":359,"context_line":"            for i in range(pending_shards)]"}],"source_content_type":"text/x-python","patch_set":8,"id":"e0e828f4_aee5cbdb","line":356,"in_reply_to":"b6e92f6e_63ea5429","updated":"2022-03-25 05:37:41.000000000","message":"Just being defensive. But happy to remove it if we \"trust\" ourselves :P","commit_id":"38a6616894f99b99e97e6c793c74848dfec482c4"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"f94b29ee1190860a7aedd64ba054c6ef809a3ab8","unresolved":true,"context_lines":[{"line_number":820,"context_line":"            if err.errno !\u003d errno.ENOENT:"},{"line_number":821,"context_line":"                raise"},{"line_number":822,"context_line":"        if pending_size \u003e PENDING_CAP:"},{"line_number":823,"context_line":"            with lock_parent_directory("},{"line_number":824,"context_line":"                    self.pending_files[0],"},{"line_number":825,"context_line":"                    self.pending_timeout, blocking\u003dFalse) as locked:"},{"line_number":826,"context_line":"                if locked:"}],"source_content_type":"text/x-python","patch_set":8,"id":"3cfd8d3b_2b7feea8","line":823,"updated":"2022-03-24 23:30:31.000000000","message":"What happens if somebody takes this lock and dies/hangs so it\u0027s never released? Currently on master, we\u0027ll start seeing a ton of lock timeouts popping, which\u0027ll be a decent signal to ops to investigate -- restart container-server and if that doesn\u0027t fix it, reboot the node.\n\nAs it is, looks like we\u0027ll just balloon the many pending files...\n\nMaybe we could do something with skipping any pending files that are already at the cap, and throwing an error if that means we don\u0027t have anywhere left to put it?","commit_id":"38a6616894f99b99e97e6c793c74848dfec482c4"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"6da015605349bf6e143dc3a6f03f425758c34412","unresolved":true,"context_lines":[{"line_number":820,"context_line":"            if err.errno !\u003d errno.ENOENT:"},{"line_number":821,"context_line":"                raise"},{"line_number":822,"context_line":"        if pending_size \u003e PENDING_CAP:"},{"line_number":823,"context_line":"            with lock_parent_directory("},{"line_number":824,"context_line":"                    self.pending_files[0],"},{"line_number":825,"context_line":"                    self.pending_timeout, blocking\u003dFalse) as locked:"},{"line_number":826,"context_line":"                if locked:"}],"source_content_type":"text/x-python","patch_set":8,"id":"7a14054b_b706fc5a","line":823,"in_reply_to":"3cfd8d3b_2b7feea8","updated":"2022-03-25 05:37:41.000000000","message":"Good idea. I\u0027ll raise an error for now. Another option is to allow them to balloon but once they get bigger then the pending_cap start logging a warning for OPs.","commit_id":"38a6616894f99b99e97e6c793c74848dfec482c4"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"f94b29ee1190860a7aedd64ba054c6ef809a3ab8","unresolved":true,"context_lines":[{"line_number":871,"context_line":"        for pending_file, size in zip(pending_files, pending_sizes):"},{"line_number":872,"context_line":"            if not size:"},{"line_number":873,"context_line":"                continue"},{"line_number":874,"context_line":"            with lock_file(pending_file, unlink\u003dFalse) as fp:"},{"line_number":875,"context_line":"                for entry in fp.read().split(b\u0027:\u0027):"},{"line_number":876,"context_line":"                    if entry:"},{"line_number":877,"context_line":"                        try:"}],"source_content_type":"text/x-python","patch_set":8,"id":"c83c3be0_164d8324","line":874,"updated":"2022-03-24 23:30:31.000000000","message":"What happens when one of these times out?","commit_id":"38a6616894f99b99e97e6c793c74848dfec482c4"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"cdf2492b5a01520f21491218f4d1af3139f10d17","unresolved":true,"context_lines":[{"line_number":871,"context_line":"        for pending_file, size in zip(pending_files, pending_sizes):"},{"line_number":872,"context_line":"            if not size:"},{"line_number":873,"context_line":"                continue"},{"line_number":874,"context_line":"            with lock_file(pending_file, unlink\u003dFalse) as fp:"},{"line_number":875,"context_line":"                for entry in fp.read().split(b\u0027:\u0027):"},{"line_number":876,"context_line":"                    if entry:"},{"line_number":877,"context_line":"                        try:"}],"source_content_type":"text/x-python","patch_set":8,"id":"51690974_28d30e0d","line":874,"in_reply_to":"9efe5139_0495ee0e","updated":"2022-03-28 05:49:23.000000000","message":"So my thinking was, if we\n\n* process one pending file, merging its items with the one we just got from the client,\n* call merge_items() to get all that into the DB, but then\n* pop a lock timeout on the next pending file\n\nwe return an error to the client even though we *could* justifiably return success. The trouble, of course, is differentiating it (as a caller) from the case when the timeout pops on the first file (so we *haven\u0027t* written down the client\u0027s row).\n\nI\u0027m probably just over-thinking it, though -- we definitely want \"all succeed or raise a timeout\" behavior in the read path, for example, so it\u0027s probably best \u0026 simplest to impose that everywhere. I was just thinking about whether the new partial-failure case might warrant different handling in some cases.","commit_id":"38a6616894f99b99e97e6c793c74848dfec482c4"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"6da015605349bf6e143dc3a6f03f425758c34412","unresolved":true,"context_lines":[{"line_number":871,"context_line":"        for pending_file, size in zip(pending_files, pending_sizes):"},{"line_number":872,"context_line":"            if not size:"},{"line_number":873,"context_line":"                continue"},{"line_number":874,"context_line":"            with lock_file(pending_file, unlink\u003dFalse) as fp:"},{"line_number":875,"context_line":"                for entry in fp.read().split(b\u0027:\u0027):"},{"line_number":876,"context_line":"                    if entry:"},{"line_number":877,"context_line":"                        try:"}],"source_content_type":"text/x-python","patch_set":8,"id":"9efe5139_0495ee0e","line":874,"in_reply_to":"c83c3be0_164d8324","updated":"2022-03-25 05:37:41.000000000","message":"hmm, I hope we get it next time?","commit_id":"38a6616894f99b99e97e6c793c74848dfec482c4"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"f94b29ee1190860a7aedd64ba054c6ef809a3ab8","unresolved":true,"context_lines":[{"line_number":886,"context_line":"                                _(\u0027Invalid pending entry %(file)s: %(entry)s\u0027),"},{"line_number":887,"context_line":"                                {\u0027file\u0027: pending_file, \u0027entry\u0027: entry})"},{"line_number":888,"context_line":"                            # TODO(matt) Instead of bombing out maybe continue"},{"line_number":889,"context_line":"                            #  with the next pending file?"},{"line_number":890,"context_line":"                if item_list:"},{"line_number":891,"context_line":"                    self.merge_items(item_list)"},{"line_number":892,"context_line":"                try:"}],"source_content_type":"text/x-python","patch_set":8,"id":"1cd3ab76_be5dc8b1","line":889,"updated":"2022-03-24 23:30:31.000000000","message":"We don\u0027t bomb out, though -- we just skip this entry.\n\n*Does* make me think that logger.exception() probably isn\u0027t appropriate, though -- there isn\u0027t much of anything the stack trace will tell us that we couldn\u0027t pick up from the distinctive error message. Might could use a reference to the actual error, though.","commit_id":"38a6616894f99b99e97e6c793c74848dfec482c4"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"6da015605349bf6e143dc3a6f03f425758c34412","unresolved":false,"context_lines":[{"line_number":886,"context_line":"                                _(\u0027Invalid pending entry %(file)s: %(entry)s\u0027),"},{"line_number":887,"context_line":"                                {\u0027file\u0027: pending_file, \u0027entry\u0027: entry})"},{"line_number":888,"context_line":"                            # TODO(matt) Instead of bombing out maybe continue"},{"line_number":889,"context_line":"                            #  with the next pending file?"},{"line_number":890,"context_line":"                if item_list:"},{"line_number":891,"context_line":"                    self.merge_items(item_list)"},{"line_number":892,"context_line":"                try:"}],"source_content_type":"text/x-python","patch_set":8,"id":"99021399_88e3df9e","line":889,"in_reply_to":"1cd3ab76_be5dc8b1","updated":"2022-03-25 05:37:41.000000000","message":"Ack","commit_id":"38a6616894f99b99e97e6c793c74848dfec482c4"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"f94b29ee1190860a7aedd64ba054c6ef809a3ab8","unresolved":true,"context_lines":[{"line_number":888,"context_line":"                            # TODO(matt) Instead of bombing out maybe continue"},{"line_number":889,"context_line":"                            #  with the next pending file?"},{"line_number":890,"context_line":"                if item_list:"},{"line_number":891,"context_line":"                    self.merge_items(item_list)"},{"line_number":892,"context_line":"                try:"},{"line_number":893,"context_line":"                    os.ftruncate(fp.fileno(), 0)"},{"line_number":894,"context_line":"                except OSError as err:"}],"source_content_type":"text/x-python","patch_set":8,"id":"3df5aed6_ce32d702","line":891,"updated":"2022-03-24 23:30:31.000000000","message":"We should clear item_list once we call merge_items to avoid quadratic behaviors.","commit_id":"38a6616894f99b99e97e6c793c74848dfec482c4"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"6da015605349bf6e143dc3a6f03f425758c34412","unresolved":false,"context_lines":[{"line_number":888,"context_line":"                            # TODO(matt) Instead of bombing out maybe continue"},{"line_number":889,"context_line":"                            #  with the next pending file?"},{"line_number":890,"context_line":"                if item_list:"},{"line_number":891,"context_line":"                    self.merge_items(item_list)"},{"line_number":892,"context_line":"                try:"},{"line_number":893,"context_line":"                    os.ftruncate(fp.fileno(), 0)"},{"line_number":894,"context_line":"                except OSError as err:"}],"source_content_type":"text/x-python","patch_set":8,"id":"35497aa2_20d77d28","line":891,"in_reply_to":"3df5aed6_ce32d702","updated":"2022-03-25 05:37:41.000000000","message":"Done","commit_id":"38a6616894f99b99e97e6c793c74848dfec482c4"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"f94b29ee1190860a7aedd64ba054c6ef809a3ab8","unresolved":true,"context_lines":[{"line_number":903,"context_line":"        if self._skip_commit_puts():"},{"line_number":904,"context_line":"            return"},{"line_number":905,"context_line":"        try:"},{"line_number":906,"context_line":"            with lock_parent_directory(self.pending_files[0],"},{"line_number":907,"context_line":"                                       self.pending_timeout):"},{"line_number":908,"context_line":"                self._commit_puts()"},{"line_number":909,"context_line":"        except (LockTimeout, sqlite3.OperationalError):"}],"source_content_type":"text/x-python","patch_set":8,"id":"b7d2875d_9ddea282","line":906,"updated":"2022-03-24 23:30:31.000000000","message":"I wonder if these would be more obvious if we used self.db_file / self.db_dir ...","commit_id":"38a6616894f99b99e97e6c793c74848dfec482c4"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"528cbd4141596bcfcc58e3059da3641990047e3c","unresolved":false,"context_lines":[{"line_number":903,"context_line":"        if self._skip_commit_puts():"},{"line_number":904,"context_line":"            return"},{"line_number":905,"context_line":"        try:"},{"line_number":906,"context_line":"            with lock_parent_directory(self.pending_files[0],"},{"line_number":907,"context_line":"                                       self.pending_timeout):"},{"line_number":908,"context_line":"                self._commit_puts()"},{"line_number":909,"context_line":"        except (LockTimeout, sqlite3.OperationalError):"}],"source_content_type":"text/x-python","patch_set":8,"id":"20e1b718_e3351e06","line":906,"in_reply_to":"10bea723_74b76349","updated":"2022-03-31 06:15:37.000000000","message":"Done","commit_id":"38a6616894f99b99e97e6c793c74848dfec482c4"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"f78a6ec1e55f9c9d815fa1034b8ff9849c851f4b","unresolved":true,"context_lines":[{"line_number":903,"context_line":"        if self._skip_commit_puts():"},{"line_number":904,"context_line":"            return"},{"line_number":905,"context_line":"        try:"},{"line_number":906,"context_line":"            with lock_parent_directory(self.pending_files[0],"},{"line_number":907,"context_line":"                                       self.pending_timeout):"},{"line_number":908,"context_line":"                self._commit_puts()"},{"line_number":909,"context_line":"        except (LockTimeout, sqlite3.OperationalError):"}],"source_content_type":"text/x-python","patch_set":8,"id":"10bea723_74b76349","line":906,"in_reply_to":"b7d2875d_9ddea282","updated":"2022-03-29 23:57:04.000000000","message":"Good idea","commit_id":"38a6616894f99b99e97e6c793c74848dfec482c4"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"cdf2492b5a01520f21491218f4d1af3139f10d17","unresolved":true,"context_lines":[{"line_number":868,"context_line":"        self._preallocate()"},{"line_number":869,"context_line":"        # There could be more or less pending files then we expect, so grab"},{"line_number":870,"context_line":"        # the names from disk."},{"line_number":871,"context_line":"        pending_files \u003d get_pending_files(self.db_file)"},{"line_number":872,"context_line":"        pending_sizes \u003d list(map(os.path.getsize, pending_files))"},{"line_number":873,"context_line":"        if not any(pending_sizes):"},{"line_number":874,"context_line":"            if item_list:"}],"source_content_type":"text/x-python","patch_set":9,"id":"a272c064_98317d65","line":871,"updated":"2022-03-28 05:49:23.000000000","message":"I wonder if we might want to shuffle these before processing, similar to what we do when we\u0027re filling them.\n\nBasically, in case there\u0027s some weird thing where we\u0027re frequently unable to process all the files -- the first few files might get cleared, but then there\u0027s a lock timeout or something, and none of the rest get processed, so they just keep filling. Once we\u0027ve got something in place where we\u0027re unwilling to write to a shard that\u0027s larger than the pending cap, that\u0027ll shrink the set of available pending files such that even though you\u0027ve configured 10 files (say), you\u0027re only able to make use of 2 or something.","commit_id":"7221b3ddc1c60742972aa7db5ed26e2d57ce7eee"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"528cbd4141596bcfcc58e3059da3641990047e3c","unresolved":false,"context_lines":[{"line_number":868,"context_line":"        self._preallocate()"},{"line_number":869,"context_line":"        # There could be more or less pending files then we expect, so grab"},{"line_number":870,"context_line":"        # the names from disk."},{"line_number":871,"context_line":"        pending_files \u003d get_pending_files(self.db_file)"},{"line_number":872,"context_line":"        pending_sizes \u003d list(map(os.path.getsize, pending_files))"},{"line_number":873,"context_line":"        if not any(pending_sizes):"},{"line_number":874,"context_line":"            if item_list:"}],"source_content_type":"text/x-python","patch_set":9,"id":"83d33511_cb02f1e7","line":871,"in_reply_to":"04420456_9fb4b633","updated":"2022-03-31 06:15:37.000000000","message":"Done","commit_id":"7221b3ddc1c60742972aa7db5ed26e2d57ce7eee"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"f78a6ec1e55f9c9d815fa1034b8ff9849c851f4b","unresolved":true,"context_lines":[{"line_number":868,"context_line":"        self._preallocate()"},{"line_number":869,"context_line":"        # There could be more or less pending files then we expect, so grab"},{"line_number":870,"context_line":"        # the names from disk."},{"line_number":871,"context_line":"        pending_files \u003d get_pending_files(self.db_file)"},{"line_number":872,"context_line":"        pending_sizes \u003d list(map(os.path.getsize, pending_files))"},{"line_number":873,"context_line":"        if not any(pending_sizes):"},{"line_number":874,"context_line":"            if item_list:"}],"source_content_type":"text/x-python","patch_set":9,"id":"04420456_9fb4b633","line":871,"in_reply_to":"a272c064_98317d65","updated":"2022-03-29 23:57:04.000000000","message":"yeah, we shuffle on the other side, we should also do it here. Great idea!","commit_id":"7221b3ddc1c60742972aa7db5ed26e2d57ce7eee"}],"swift/common/utils.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"f94b29ee1190860a7aedd64ba054c6ef809a3ab8","unresolved":true,"context_lines":[{"line_number":3015,"context_line":"        fd \u003d os.open(filename, flags)"},{"line_number":3016,"context_line":"        file_obj \u003d os.fdopen(fd, mode)"},{"line_number":3017,"context_line":"        try:"},{"line_number":3018,"context_line":"            with swift.common.exceptions.LockTimeout(timeout, filename):"},{"line_number":3019,"context_line":"                try:"},{"line_number":3020,"context_line":"                    fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)"},{"line_number":3021,"context_line":"                    try:"}],"source_content_type":"text/x-python","patch_set":8,"id":"47a80745_d13d7f10","line":3018,"updated":"2022-03-24 23:30:31.000000000","message":"If everybody\u0027s locked... I\u0027m not sure we\u0027ll ever trip the timeout. We want this outside the for loop.","commit_id":"38a6616894f99b99e97e6c793c74848dfec482c4"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"6da015605349bf6e143dc3a6f03f425758c34412","unresolved":false,"context_lines":[{"line_number":3015,"context_line":"        fd \u003d os.open(filename, flags)"},{"line_number":3016,"context_line":"        file_obj \u003d os.fdopen(fd, mode)"},{"line_number":3017,"context_line":"        try:"},{"line_number":3018,"context_line":"            with swift.common.exceptions.LockTimeout(timeout, filename):"},{"line_number":3019,"context_line":"                try:"},{"line_number":3020,"context_line":"                    fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)"},{"line_number":3021,"context_line":"                    try:"}],"source_content_type":"text/x-python","patch_set":8,"id":"64184b97_e1bda4af","line":3018,"in_reply_to":"47a80745_d13d7f10","updated":"2022-03-25 05:37:41.000000000","message":"Done","commit_id":"38a6616894f99b99e97e6c793c74848dfec482c4"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"f94b29ee1190860a7aedd64ba054c6ef809a3ab8","unresolved":true,"context_lines":[{"line_number":3027,"context_line":"                            file_obj.close()"},{"line_number":3028,"context_line":"                            continue"},{"line_number":3029,"context_line":"                        raise"},{"line_number":3030,"context_line":"                    yield file_obj"},{"line_number":3031,"context_line":"                    if unlink:"},{"line_number":3032,"context_line":"                        os.unlink(filename)"},{"line_number":3033,"context_line":"                    break"}],"source_content_type":"text/x-python","patch_set":8,"id":"0fa0f12c_15f02c84","line":3030,"updated":"2022-03-24 23:30:31.000000000","message":"Definitely *don\u0027t* want the yield inside the LockTimeout -- the guy that *succeeds* in getting the lock can have a LockTimeout pop in the middle of its work!","commit_id":"38a6616894f99b99e97e6c793c74848dfec482c4"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"6da015605349bf6e143dc3a6f03f425758c34412","unresolved":false,"context_lines":[{"line_number":3027,"context_line":"                            file_obj.close()"},{"line_number":3028,"context_line":"                            continue"},{"line_number":3029,"context_line":"                        raise"},{"line_number":3030,"context_line":"                    yield file_obj"},{"line_number":3031,"context_line":"                    if unlink:"},{"line_number":3032,"context_line":"                        os.unlink(filename)"},{"line_number":3033,"context_line":"                    break"}],"source_content_type":"text/x-python","patch_set":8,"id":"f499b07d_34a392e4","line":3030,"in_reply_to":"0fa0f12c_15f02c84","updated":"2022-03-25 05:37:41.000000000","message":"Done","commit_id":"38a6616894f99b99e97e6c793c74848dfec482c4"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"f94b29ee1190860a7aedd64ba054c6ef809a3ab8","unresolved":true,"context_lines":[{"line_number":3034,"context_line":"                except IOError as err:"},{"line_number":3035,"context_line":"                    if err.errno !\u003d errno.EAGAIN:"},{"line_number":3036,"context_line":"                        raise"},{"line_number":3037,"context_line":"                sleep(0.01)"},{"line_number":3038,"context_line":"        finally:"},{"line_number":3039,"context_line":"            file_obj.close()"},{"line_number":3040,"context_line":""}],"source_content_type":"text/x-python","patch_set":8,"id":"98c1bf0a_17e37a6d","line":3037,"updated":"2022-03-24 23:30:31.000000000","message":"I\u0027m not sure I like this between every lock attempt -- maybe we could do something closer to\n\n\n have_lock \u003d False\n with LockTimeout(...):\n     while True:\n         for filename in filenames:\n             try:\n                 # get lock\n                 have_lock \u003d True\n                 break\n             except:\n                 ...\n         if have_lock:\n             break\n         sleep(0.01)\n yield ...","commit_id":"38a6616894f99b99e97e6c793c74848dfec482c4"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"6da015605349bf6e143dc3a6f03f425758c34412","unresolved":false,"context_lines":[{"line_number":3034,"context_line":"                except IOError as err:"},{"line_number":3035,"context_line":"                    if err.errno !\u003d errno.EAGAIN:"},{"line_number":3036,"context_line":"                        raise"},{"line_number":3037,"context_line":"                sleep(0.01)"},{"line_number":3038,"context_line":"        finally:"},{"line_number":3039,"context_line":"            file_obj.close()"},{"line_number":3040,"context_line":""}],"source_content_type":"text/x-python","patch_set":8,"id":"0a1e9b1c_6f29dbf1","line":3037,"in_reply_to":"98c1bf0a_17e37a6d","updated":"2022-03-25 05:37:41.000000000","message":"Done","commit_id":"38a6616894f99b99e97e6c793c74848dfec482c4"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"cdf2492b5a01520f21491218f4d1af3139f10d17","unresolved":true,"context_lines":[{"line_number":3000,"context_line":"    :param unlink: True if the file should be unlinked at the end"},{"line_number":3001,"context_line":"    :param shuffle_files: True if the list should be shuffled before cycled"},{"line_number":3002,"context_line":"        through"},{"line_number":3003,"context_line":"    :param max_file_size: If provided will skip files once they get to this"},{"line_number":3004,"context_line":"        size"},{"line_number":3005,"context_line":"    \"\"\""},{"line_number":3006,"context_line":"    if timeout is None:"}],"source_content_type":"text/x-python","patch_set":9,"id":"0f8b5bbf_faf93a9b","line":3003,"updated":"2022-03-28 05:49:23.000000000","message":"I was thinking, since we already call getsize() on everybody in the caller, we could just filter the filenames we pass in. Sure, there could be a write between then and when we actually manage to take the lock, but I don\u0027t think we actually need a *hard* cap -- just something to make sure these guys don\u0027t grow without bound.","commit_id":"7221b3ddc1c60742972aa7db5ed26e2d57ce7eee"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"f78a6ec1e55f9c9d815fa1034b8ff9849c851f4b","unresolved":true,"context_lines":[{"line_number":3000,"context_line":"    :param unlink: True if the file should be unlinked at the end"},{"line_number":3001,"context_line":"    :param shuffle_files: True if the list should be shuffled before cycled"},{"line_number":3002,"context_line":"        through"},{"line_number":3003,"context_line":"    :param max_file_size: If provided will skip files once they get to this"},{"line_number":3004,"context_line":"        size"},{"line_number":3005,"context_line":"    \"\"\""},{"line_number":3006,"context_line":"    if timeout is None:"}],"source_content_type":"text/x-python","patch_set":9,"id":"bf84d4fd_85bfac76","line":3003,"in_reply_to":"0f8b5bbf_faf93a9b","updated":"2022-03-29 23:57:04.000000000","message":"oh, I didn\u0027t even think about that. So that could make it more of a soft block. Once it gets into this function it\u0027s a viable file to return. Also love that we could use the existing getsize values and save some extra IO.","commit_id":"7221b3ddc1c60742972aa7db5ed26e2d57ce7eee"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"cdf2492b5a01520f21491218f4d1af3139f10d17","unresolved":true,"context_lines":[{"line_number":3038,"context_line":"                break"},{"line_number":3039,"context_line":"            except IOError as err:"},{"line_number":3040,"context_line":"                if err.errno !\u003d errno.EAGAIN:"},{"line_number":3041,"context_line":"                    raise"},{"line_number":3042,"context_line":"            sleep(0.01)"},{"line_number":3043,"context_line":"            file_obj.close()"},{"line_number":3044,"context_line":""}],"source_content_type":"text/x-python","patch_set":9,"id":"781695e5_c141c49d","line":3041,"updated":"2022-03-28 05:49:23.000000000","message":"I think we leave a file descriptor open here. Also, if there\u0027s any *other* sort of error raised...","commit_id":"7221b3ddc1c60742972aa7db5ed26e2d57ce7eee"}]}