)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":6198,"name":"Peter Portante","email":"peter.a.portante@gmail.com","username":"peter-a-portante"},"change_message_id":"1f4065ef3417fbd894f8ae483d5836b96e1628c4","unresolved":false,"context_lines":[{"line_number":24,"context_line":"eventlet.tpool.execute. You won\u0027t end up blocking a whole object"},{"line_number":25,"context_line":"server while waiting for a huge fsync."},{"line_number":26,"context_line":""},{"line_number":27,"context_line":"If you decide not to use threadpools, the only extra overhead should"},{"line_number":28,"context_line":"be one extra Python function call for each blocking syscall. This is"},{"line_number":29,"context_line":"accomplished by setting threads_per_disk \u003d 0 in the config."},{"line_number":30,"context_line":""}],"source_content_type":"text/x-gerrit-commit-message","patch_set":7,"id":"AAAAOn%2F%2FUSw%3D","line":27,"updated":"2013-05-14 21:36:05.000000000","message":"The more I think about this more I am inclined to suggest we just go with always have thread pools per disk. Is there any substantial downside to not using thread pools?","commit_id":"9c05808ccc02303eeabf8cdc0f6757a1581dd541"},{"author":{"_account_id":6198,"name":"Peter Portante","email":"peter.a.portante@gmail.com","username":"peter-a-portante"},"change_message_id":"9405bb255252832c911841a4da82241b25f11266","unresolved":false,"context_lines":[{"line_number":4,"context_line":"Commit:     Samuel Merritt \u003csam@swiftstack.com\u003e"},{"line_number":5,"context_line":"CommitDate: 2013-05-20 10:09:21 -0700"},{"line_number":6,"context_line":""},{"line_number":7,"context_line":"Use threadpools in the object server for performance."},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"Without a (per-disk) threadpool, requests to a slow disk would affect"},{"line_number":10,"context_line":"all clients by blocking the entire eventlet reactor on"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":8,"id":"AAAAPH%2F%2F5qo%3D","line":7,"updated":"2013-05-20 18:52:30.000000000","message":"Is it worth referring to thread pools as two nouns instead of one merged word?","commit_id":"7e24345f49c666397043cf58be56ab7679a16445"},{"author":{"_account_id":6198,"name":"Peter Portante","email":"peter.a.portante@gmail.com","username":"peter-a-portante"},"change_message_id":"9405bb255252832c911841a4da82241b25f11266","unresolved":false,"context_lines":[{"line_number":12,"context_line":"object server, you frequently have at least one slow disk due to"},{"line_number":13,"context_line":"auditing and replication activity sucking up all the available IO. By"},{"line_number":14,"context_line":"kicking those blocking calls out to a separate OS thread, we let the"},{"line_number":15,"context_line":"eventlet reactor make progress in other greenthreads, and by having a"},{"line_number":16,"context_line":"per-disk pool, we ensure that one slow disk can\u0027t suck up all the"},{"line_number":17,"context_line":"resources of an entire object server."},{"line_number":18,"context_line":""}],"source_content_type":"text/x-gerrit-commit-message","patch_set":8,"id":"AAAAPH%2F%2F5qk%3D","line":15,"updated":"2013-05-20 18:52:30.000000000","message":"should greenthreads be \"greenlets\"?","commit_id":"7e24345f49c666397043cf58be56ab7679a16445"},{"author":{"_account_id":2696,"name":"Darrell Bishop","email":"dbishop@nvidia.com","username":"darrellb"},"change_message_id":"34dde6f3e57efe696c9e6294814f0a283793550f","unresolved":false,"context_lines":[{"line_number":12,"context_line":"object server, you frequently have at least one slow disk due to"},{"line_number":13,"context_line":"auditing and replication activity sucking up all the available IO. By"},{"line_number":14,"context_line":"kicking those blocking calls out to a separate OS thread, we let the"},{"line_number":15,"context_line":"eventlet reactor make progress in other greenthreads, and by having a"},{"line_number":16,"context_line":"per-disk pool, we ensure that one slow disk can\u0027t suck up all the"},{"line_number":17,"context_line":"resources of an entire object server."},{"line_number":18,"context_line":""}],"source_content_type":"text/x-gerrit-commit-message","patch_set":8,"id":"AAAAPH%2F%2F5ZA%3D","line":15,"in_reply_to":"AAAAPH%2F%2F5qk%3D","updated":"2013-05-20 19:25:49.000000000","message":"Not sure, but my understanding is that eventlet implements \"greenthreads\" (which can be used like system threads) using greenlet which provides \"greenlets\" which are at a lower level than \"greenthreads\" provided by eventlet.\n\nThe greenlet docs have this to say about itself:\n\n  A “greenlet”, on the other hand [vs. Stackless Python\u0027s micro-\n  threads], is a still more primitive notion of micro-thread with no\n  implicit scheduling...You can build custom scheduled micro-\n  threads on top of greenlet;...","commit_id":"7e24345f49c666397043cf58be56ab7679a16445"},{"author":{"_account_id":6198,"name":"Peter Portante","email":"peter.a.portante@gmail.com","username":"peter-a-portante"},"change_message_id":"9405bb255252832c911841a4da82241b25f11266","unresolved":false,"context_lines":[{"line_number":22,"context_line":"per-disk threadpools, (i.e. threads_per_disk \u003d 0, which is the"},{"line_number":23,"context_line":"default), those call sites will still ultimately end up using"},{"line_number":24,"context_line":"eventlet.tpool.execute. You won\u0027t end up blocking a whole object"},{"line_number":25,"context_line":"server while waiting for a huge fsync."},{"line_number":26,"context_line":""},{"line_number":27,"context_line":"If you decide not to use threadpools, the only extra overhead should"},{"line_number":28,"context_line":"be one extra Python function call for each blocking syscall. This is"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":8,"id":"AAAAPH%2F%2F5qE%3D","line":25,"updated":"2013-05-20 18:52:30.000000000","message":"Should we also talk about the fact that we grouped a set of potentially blocking operations into one method that is run in an OS thread that saves on tpool context switch overhead?","commit_id":"7e24345f49c666397043cf58be56ab7679a16445"}],"doc/source/deployment_guide.rst":[{"author":{"_account_id":4108,"name":"David Hadas","email":"davidh@il.ibm.com","username":"david-hadas"},"change_message_id":"a2699684abef0c85d586652076117b0d33d23c84","unresolved":false,"context_lines":[{"line_number":282,"context_line":"                                   kernel\u0027s buffer cache"},{"line_number":283,"context_line":"threads_per_disk    0              Size of the per-disk thread pool used for"},{"line_number":284,"context_line":"                                   performing disk I/O. The default of 0 means"},{"line_number":285,"context_line":"                                   to not use a per-disk thread pool."},{"line_number":286,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":287,"context_line":""},{"line_number":288,"context_line":"[object-replicator]"}],"source_content_type":"text/x-rst","patch_set":1,"id":"AAAAOn%2F%2F0Uw%3D","line":285,"updated":"2013-04-22 13:07:22.000000000","message":"Lets also state a recommended value... 3?","commit_id":"46f1b3e20375c164d536b1805032eee5910fd26d"},{"author":{"_account_id":2622,"name":"Samuel Merritt","email":"spam+launchpad@andcheese.org","username":"torgomatic"},"change_message_id":"94d51cc93783fa7fa67c80c2dbc8454c4ea16fd1","unresolved":false,"context_lines":[{"line_number":282,"context_line":"                                   kernel\u0027s buffer cache"},{"line_number":283,"context_line":"threads_per_disk    0              Size of the per-disk thread pool used for"},{"line_number":284,"context_line":"                                   performing disk I/O. The default of 0 means"},{"line_number":285,"context_line":"                                   to not use a per-disk thread pool."},{"line_number":286,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d  \u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":287,"context_line":""},{"line_number":288,"context_line":"[object-replicator]"}],"source_content_type":"text/x-rst","patch_set":1,"id":"AAAAOn%2F%2Fu%2BM%3D","line":285,"in_reply_to":"AAAAOn%2F%2F0Uw%3D","updated":"2013-04-25 18:22:18.000000000","message":"Done","commit_id":"46f1b3e20375c164d536b1805032eee5910fd26d"}],"etc/object-server.conf-sample":[{"author":{"_account_id":4108,"name":"David Hadas","email":"davidh@il.ibm.com","username":"david-hadas"},"change_message_id":"a2699684abef0c85d586652076117b0d33d23c84","unresolved":false,"context_lines":[{"line_number":61,"context_line":"# This list is in addition to X-Object-Meta-* headers and cannot include"},{"line_number":62,"context_line":"# Content-Type, etag, Content-Length, or deleted"},{"line_number":63,"context_line":"# allowed_headers \u003d Content-Disposition, Content-Encoding, X-Delete-At, X-Object-Manifest, X-Static-Large-Object"},{"line_number":64,"context_line":"# auto_create_account_prefix \u003d ."},{"line_number":65,"context_line":"# A value of 0 means \"don\u0027t use thread pools\""},{"line_number":66,"context_line":"# threads_per_disk \u003d 0"},{"line_number":67,"context_line":""}],"source_content_type":"application/octet-stream","patch_set":1,"id":"AAAAOn%2F%2F0U0%3D","line":64,"updated":"2013-04-22 13:07:22.000000000","message":"Lets also state a recommended value... 3?","commit_id":"46f1b3e20375c164d536b1805032eee5910fd26d"},{"author":{"_account_id":2622,"name":"Samuel Merritt","email":"spam+launchpad@andcheese.org","username":"torgomatic"},"change_message_id":"94d51cc93783fa7fa67c80c2dbc8454c4ea16fd1","unresolved":false,"context_lines":[{"line_number":61,"context_line":"# This list is in addition to X-Object-Meta-* headers and cannot include"},{"line_number":62,"context_line":"# Content-Type, etag, Content-Length, or deleted"},{"line_number":63,"context_line":"# allowed_headers \u003d Content-Disposition, Content-Encoding, X-Delete-At, X-Object-Manifest, X-Static-Large-Object"},{"line_number":64,"context_line":"# auto_create_account_prefix \u003d ."},{"line_number":65,"context_line":"# A value of 0 means \"don\u0027t use thread pools\""},{"line_number":66,"context_line":"# threads_per_disk \u003d 0"},{"line_number":67,"context_line":""}],"source_content_type":"application/octet-stream","patch_set":1,"id":"AAAAOn%2F%2Fu%2BQ%3D","line":64,"in_reply_to":"AAAAOn%2F%2F0U0%3D","updated":"2013-04-25 18:22:18.000000000","message":"I\u0027d prefer to leave the recommendation in the admin guide. The sample config is long enough as it is.","commit_id":"46f1b3e20375c164d536b1805032eee5910fd26d"},{"author":{"_account_id":6198,"name":"Peter Portante","email":"peter.a.portante@gmail.com","username":"peter-a-portante"},"change_message_id":"89b3a461f2f9143f3dca781228d03b3d855b2751","unresolved":false,"context_lines":[{"line_number":63,"context_line":"# allowed_headers \u003d Content-Disposition, Content-Encoding, X-Delete-At, X-Object-Manifest, X-Static-Large-Object"},{"line_number":64,"context_line":"# auto_create_account_prefix \u003d ."},{"line_number":65,"context_line":"# max_clients \u003d 1024"},{"line_number":66,"context_line":"# A value of 0 means \"don\u0027t use thread pools\""},{"line_number":67,"context_line":"# threads_per_disk \u003d 0"},{"line_number":68,"context_line":""},{"line_number":69,"context_line":"[filter:healthcheck]"}],"source_content_type":"application/octet-stream","patch_set":6,"id":"AAAAOn%2F%2FaG8%3D","line":66,"updated":"2013-05-10 15:49:55.000000000","message":"Perhaps also mention the suggested starting value of 4 here as well?","commit_id":"22778de2e8101fbc213f063c65a1197245be0f5a"},{"author":{"_account_id":2622,"name":"Samuel Merritt","email":"spam+launchpad@andcheese.org","username":"torgomatic"},"change_message_id":"538df82f1e010add91851bdd591e66bf72e6b195","unresolved":false,"context_lines":[{"line_number":63,"context_line":"# allowed_headers \u003d Content-Disposition, Content-Encoding, X-Delete-At, X-Object-Manifest, X-Static-Large-Object"},{"line_number":64,"context_line":"# auto_create_account_prefix \u003d ."},{"line_number":65,"context_line":"# max_clients \u003d 1024"},{"line_number":66,"context_line":"# A value of 0 means \"don\u0027t use thread pools\""},{"line_number":67,"context_line":"# threads_per_disk \u003d 0"},{"line_number":68,"context_line":""},{"line_number":69,"context_line":"[filter:healthcheck]"}],"source_content_type":"application/octet-stream","patch_set":6,"id":"AAAAOn%2F%2FZ0I%3D","line":66,"in_reply_to":"AAAAOn%2F%2FaG8%3D","updated":"2013-05-10 18:34:02.000000000","message":"Sure. Will fix.","commit_id":"22778de2e8101fbc213f063c65a1197245be0f5a"}],"swift/common/utils.py":[{"author":{"_account_id":4108,"name":"David Hadas","email":"davidh@il.ibm.com","username":"david-hadas"},"change_message_id":"a2699684abef0c85d586652076117b0d33d23c84","unresolved":false,"context_lines":[{"line_number":1675,"context_line":"        self.bytes_received +\u003d len(line)"},{"line_number":1676,"context_line":"        return line"},{"line_number":1677,"context_line":""},{"line_number":1678,"context_line":""},{"line_number":1679,"context_line":"class DummyThreadPool(object):"},{"line_number":1680,"context_line":"    \"\"\""},{"line_number":1681,"context_line":"    Useful as a standin for a ThreadPool when you don\u0027t actually want threads."}],"source_content_type":"text/x-python","patch_set":1,"id":"AAAAOn%2F%2F0O0%3D","line":1678,"updated":"2013-04-22 13:07:22.000000000","message":"This may be a style issue - but I prefer it if we reuse the ThreadPool object as our DummyThreadPool while setting nthreads to zero instead of implementing yet another class and aligning the two.\n\nSo I suggest to remove the code of DummyThreadPool","commit_id":"46f1b3e20375c164d536b1805032eee5910fd26d"},{"author":{"_account_id":2622,"name":"Samuel Merritt","email":"spam+launchpad@andcheese.org","username":"torgomatic"},"change_message_id":"94d51cc93783fa7fa67c80c2dbc8454c4ea16fd1","unresolved":false,"context_lines":[{"line_number":1675,"context_line":"        self.bytes_received +\u003d len(line)"},{"line_number":1676,"context_line":"        return line"},{"line_number":1677,"context_line":""},{"line_number":1678,"context_line":""},{"line_number":1679,"context_line":"class DummyThreadPool(object):"},{"line_number":1680,"context_line":"    \"\"\""},{"line_number":1681,"context_line":"    Useful as a standin for a ThreadPool when you don\u0027t actually want threads."}],"source_content_type":"text/x-python","patch_set":1,"id":"AAAAOn%2F%2Fu6g%3D","line":1678,"in_reply_to":"AAAAOn%2F%2F0O0%3D","updated":"2013-04-25 18:22:18.000000000","message":"Done","commit_id":"46f1b3e20375c164d536b1805032eee5910fd26d"},{"author":{"_account_id":4108,"name":"David Hadas","email":"davidh@il.ibm.com","username":"david-hadas"},"change_message_id":"a2699684abef0c85d586652076117b0d33d23c84","unresolved":false,"context_lines":[{"line_number":1689,"context_line":"        multi-thread functionality."},{"line_number":1690,"context_line":"        \"\"\""},{"line_number":1691,"context_line":"        return func(*args, **kwargs)"},{"line_number":1692,"context_line":""},{"line_number":1693,"context_line":"    def force_run_in_thread(self, func, *args, **kwargs):"},{"line_number":1694,"context_line":"        \"\"\""},{"line_number":1695,"context_line":"        Really run something in an external thread."}],"source_content_type":"text/x-python","patch_set":1,"id":"AAAAOn%2F%2F0LQ%3D","line":1692,"updated":"2013-04-22 13:07:22.000000000","message":"Why do we need this interface? \nIf the user needs to force something to run in a seperate Thread, he can create a ThreadPool with nthread\u003d1 and use it.\n\nThis is awkward.","commit_id":"46f1b3e20375c164d536b1805032eee5910fd26d"},{"author":{"_account_id":4108,"name":"David Hadas","email":"davidh@il.ibm.com","username":"david-hadas"},"change_message_id":"a2699684abef0c85d586652076117b0d33d23c84","unresolved":false,"context_lines":[{"line_number":1716,"context_line":"    Call its methods from within greenlets to green-wait for results without"},{"line_number":1717,"context_line":"    blocking the eventlet reactor (hopefully)."},{"line_number":1718,"context_line":"    \"\"\""},{"line_number":1719,"context_line":"    def __init__(self, nthreads\u003d2):"},{"line_number":1720,"context_line":"        self._run_queue \u003d Queue()"},{"line_number":1721,"context_line":"        self._result_queue \u003d Queue()"},{"line_number":1722,"context_line":"        self._threads \u003d []"}],"source_content_type":"text/x-python","patch_set":1,"id":"AAAAOn%2F%2F0Ow%3D","line":1719,"updated":"2013-04-22 13:07:22.000000000","message":"Add here:\n\nif not nthreads:\n    return","commit_id":"46f1b3e20375c164d536b1805032eee5910fd26d"},{"author":{"_account_id":2860,"name":"litong01","email":"email4tong@yahoo.com","username":"litong01"},"change_message_id":"41fc46ca065312656b22ad15eb5f21d908c88500","unresolved":false,"context_lines":[{"line_number":1721,"context_line":"        self._result_queue \u003d Queue()"},{"line_number":1722,"context_line":"        self._threads \u003d []"},{"line_number":1723,"context_line":""},{"line_number":1724,"context_line":"        # We spawn a greenthread whose job it is to pull results from the"},{"line_number":1725,"context_line":"        # worker threads via a real Queue and send them to eventlet Events so"},{"line_number":1726,"context_line":"        # that the calling greenthreads can be awoken."},{"line_number":1727,"context_line":"        #"}],"source_content_type":"text/x-python","patch_set":1,"id":"AAAAOn%2F%2F1Vg%3D","line":1724,"updated":"2013-04-19 14:50:00.000000000","message":"The word \"it\" should be removed?","commit_id":"46f1b3e20375c164d536b1805032eee5910fd26d"},{"author":{"_account_id":2622,"name":"Samuel Merritt","email":"spam+launchpad@andcheese.org","username":"torgomatic"},"change_message_id":"94d51cc93783fa7fa67c80c2dbc8454c4ea16fd1","unresolved":false,"context_lines":[{"line_number":1721,"context_line":"        self._result_queue \u003d Queue()"},{"line_number":1722,"context_line":"        self._threads \u003d []"},{"line_number":1723,"context_line":""},{"line_number":1724,"context_line":"        # We spawn a greenthread whose job it is to pull results from the"},{"line_number":1725,"context_line":"        # worker threads via a real Queue and send them to eventlet Events so"},{"line_number":1726,"context_line":"        # that the calling greenthreads can be awoken."},{"line_number":1727,"context_line":"        #"}],"source_content_type":"text/x-python","patch_set":1,"id":"AAAAOn%2F%2Fu6k%3D","line":1724,"in_reply_to":"AAAAOn%2F%2F1Vg%3D","updated":"2013-04-25 18:22:18.000000000","message":"I think the sentence works either way.","commit_id":"46f1b3e20375c164d536b1805032eee5910fd26d"},{"author":{"_account_id":4108,"name":"David Hadas","email":"davidh@il.ibm.com","username":"david-hadas"},"change_message_id":"a2699684abef0c85d586652076117b0d33d23c84","unresolved":false,"context_lines":[{"line_number":1762,"context_line":"        :param work_queue: queue from which to pull work"},{"line_number":1763,"context_line":"        :param result_queue: queue into which to place results"},{"line_number":1764,"context_line":"        \"\"\""},{"line_number":1765,"context_line":"        while True:"},{"line_number":1766,"context_line":"            item \u003d work_queue.get()"},{"line_number":1767,"context_line":"            work_queue.task_done()"},{"line_number":1768,"context_line":"            event, func, args, kwargs \u003d item"}],"source_content_type":"text/x-python","patch_set":1,"id":"AAAAOn%2F%2F0OM%3D","line":1765,"updated":"2013-04-22 13:07:22.000000000","message":"Should be:\n\nitem \u003d work_queue.get(True)","commit_id":"46f1b3e20375c164d536b1805032eee5910fd26d"},{"author":{"_account_id":2622,"name":"Samuel Merritt","email":"spam+launchpad@andcheese.org","username":"torgomatic"},"change_message_id":"94d51cc93783fa7fa67c80c2dbc8454c4ea16fd1","unresolved":false,"context_lines":[{"line_number":1762,"context_line":"        :param work_queue: queue from which to pull work"},{"line_number":1763,"context_line":"        :param result_queue: queue into which to place results"},{"line_number":1764,"context_line":"        \"\"\""},{"line_number":1765,"context_line":"        while True:"},{"line_number":1766,"context_line":"            item \u003d work_queue.get()"},{"line_number":1767,"context_line":"            work_queue.task_done()"},{"line_number":1768,"context_line":"            event, func, args, kwargs \u003d item"}],"source_content_type":"text/x-python","patch_set":1,"id":"AAAAOn%2F%2Fu6o%3D","line":1765,"in_reply_to":"AAAAOn%2F%2F0OM%3D","updated":"2013-04-25 18:22:18.000000000","message":"The default for get() is block\u003dTrue, timeout\u003dNone.\n\nIf you want it in there for clarity purposes, I\u0027m okay with that.","commit_id":"46f1b3e20375c164d536b1805032eee5910fd26d"},{"author":{"_account_id":4108,"name":"David Hadas","email":"davidh@il.ibm.com","username":"david-hadas"},"change_message_id":"a2699684abef0c85d586652076117b0d33d23c84","unresolved":false,"context_lines":[{"line_number":1763,"context_line":"        :param result_queue: queue into which to place results"},{"line_number":1764,"context_line":"        \"\"\""},{"line_number":1765,"context_line":"        while True:"},{"line_number":1766,"context_line":"            item \u003d work_queue.get()"},{"line_number":1767,"context_line":"            work_queue.task_done()"},{"line_number":1768,"context_line":"            event, func, args, kwargs \u003d item"},{"line_number":1769,"context_line":"            try:"}],"source_content_type":"text/x-python","patch_set":1,"id":"AAAAOn%2F%2F0OI%3D","line":1766,"updated":"2013-04-22 13:07:22.000000000","message":"As far as I can understand we do not need this call if we never use join - but not sure. \n\nIf we do call task_done lets do it properly and call it after the task is done :)","commit_id":"46f1b3e20375c164d536b1805032eee5910fd26d"},{"author":{"_account_id":4108,"name":"David Hadas","email":"davidh@il.ibm.com","username":"david-hadas"},"change_message_id":"a2699684abef0c85d586652076117b0d33d23c84","unresolved":false,"context_lines":[{"line_number":1781,"context_line":""},{"line_number":1782,"context_line":"        Takes results from the worker OS threads and sends them to the waiting"},{"line_number":1783,"context_line":"        greenthreads."},{"line_number":1784,"context_line":"        \"\"\""},{"line_number":1785,"context_line":"        while True:"},{"line_number":1786,"context_line":"            try:"},{"line_number":1787,"context_line":"                self.rpipe.read(1)"}],"source_content_type":"text/x-python","patch_set":1,"id":"AAAAOn%2F%2F0N0%3D","line":1784,"updated":"2013-04-22 13:07:22.000000000","message":"I get what you have done, but it is rather complex... :) and I wonder if we cant think of a way to make it simpler.\n\nN being the number of worker threads per device,\ngiven that N is small, \ncant we have N slots in which work is submitted to the N worker threads, \nthan when a main worker greenthread try to submit job, if it finds all slots being full, it will yield i.e. sleep(0). \n\nIf a slot is empty, it will occupy the slot and signal the worker thread (Condition?) and wait till it is signaled that the work was done (via eventlet.semaphre() ?) Never tried any of that though...  :)","commit_id":"46f1b3e20375c164d536b1805032eee5910fd26d"},{"author":{"_account_id":4108,"name":"David Hadas","email":"davidh@il.ibm.com","username":"david-hadas"},"change_message_id":"a2699684abef0c85d586652076117b0d33d23c84","unresolved":false,"context_lines":[{"line_number":1810,"context_line":""},{"line_number":1811,"context_line":"        :returns: result of calling func"},{"line_number":1812,"context_line":""},{"line_number":1813,"context_line":"        \"\"\""},{"line_number":1814,"context_line":"        ev \u003d event.Event()"},{"line_number":1815,"context_line":""},{"line_number":1816,"context_line":"        self._run_queue.put((ev, func, args, kwargs), block\u003dFalse)"}],"source_content_type":"text/x-python","patch_set":1,"id":"AAAAOn%2F%2F0Oo%3D","line":1813,"updated":"2013-04-22 13:07:22.000000000","message":"Add here: \n\nif not nthreads:\n    return func(*args, **kwargs)","commit_id":"46f1b3e20375c164d536b1805032eee5910fd26d"},{"author":{"_account_id":2860,"name":"litong01","email":"email4tong@yahoo.com","username":"litong01"},"change_message_id":"41fc46ca065312656b22ad15eb5f21d908c88500","unresolved":false,"context_lines":[{"line_number":1813,"context_line":"        \"\"\""},{"line_number":1814,"context_line":"        ev \u003d event.Event()"},{"line_number":1815,"context_line":""},{"line_number":1816,"context_line":"        self._run_queue.put((ev, func, args, kwargs), block\u003dFalse)"},{"line_number":1817,"context_line":""},{"line_number":1818,"context_line":"        # blocks this greenlet (and only *this* greenlet) until the real"},{"line_number":1819,"context_line":"        # thread calls ev.send()."}],"source_content_type":"text/x-python","patch_set":1,"id":"AAAAOn%2F%2F1VI%3D","line":1816,"updated":"2013-04-19 14:50:00.000000000","message":"block\u003dFalse, that means there is a chance that this put can fail. I do not see\nhow it handle the case. Will the task be lost?","commit_id":"46f1b3e20375c164d536b1805032eee5910fd26d"},{"author":{"_account_id":2622,"name":"Samuel Merritt","email":"spam+launchpad@andcheese.org","username":"torgomatic"},"change_message_id":"92e655ac7585a7ed0db47d1a7870bd2f5b9627c8","unresolved":false,"context_lines":[{"line_number":1813,"context_line":"        \"\"\""},{"line_number":1814,"context_line":"        ev \u003d event.Event()"},{"line_number":1815,"context_line":""},{"line_number":1816,"context_line":"        self._run_queue.put((ev, func, args, kwargs), block\u003dFalse)"},{"line_number":1817,"context_line":""},{"line_number":1818,"context_line":"        # blocks this greenlet (and only *this* greenlet) until the real"},{"line_number":1819,"context_line":"        # thread calls ev.send()."}],"source_content_type":"text/x-python","patch_set":1,"id":"AAAAOn%2F%2F1G8%3D","line":1816,"in_reply_to":"AAAAOn%2F%2F1VI%3D","updated":"2013-04-19 22:25:21.000000000","message":"That\u0027s more a bit of defensive programming than anything else.\n\nThe way it\u0027s currently written, self._run_queue is a Queue.Queue() instantiated with no size limit, so the size is infinite, and so this call will never block.\n\nIf, in the future, someone puts a limit on the size of self._run_queue, then there would be a chance of blocking on the queue, which could (I think) deadlock the entire object server, as the main thread would block here, and so the result-consuming greenthread wouldn\u0027t run.\n\nHaving block\u003dFalse here is a bit of future-proofing against that; if you try to put into a queue that (a) has a size limit, and (b) is full, then the put() call raises Queue.Full. This would ultimately result in a 500 response returned to the proxy.","commit_id":"46f1b3e20375c164d536b1805032eee5910fd26d"},{"author":{"_account_id":4108,"name":"David Hadas","email":"davidh@il.ibm.com","username":"david-hadas"},"change_message_id":"a2699684abef0c85d586652076117b0d33d23c84","unresolved":false,"context_lines":[{"line_number":1820,"context_line":"        result \u003d ev.wait()"},{"line_number":1821,"context_line":"        return result"},{"line_number":1822,"context_line":""},{"line_number":1823,"context_line":"    def force_run_in_thread(self, func, *args, **kwargs):"},{"line_number":1824,"context_line":"        return self.run_in_thread(func, *args, **kwargs)"}],"source_content_type":"text/x-python","patch_set":1,"id":"AAAAOn%2F%2F0Ok%3D","line":1823,"updated":"2013-04-22 13:07:22.000000000","message":"Need to solve this one for the case of nthread \u003d\u003d 0","commit_id":"46f1b3e20375c164d536b1805032eee5910fd26d"},{"author":{"_account_id":6198,"name":"Peter Portante","email":"peter.a.portante@gmail.com","username":"peter-a-portante"},"change_message_id":"9405bb255252832c911841a4da82241b25f11266","unresolved":false,"context_lines":[{"line_number":1740,"context_line":"    BYTE \u003d \u0027a\u0027.encode(\u0027utf-8\u0027)"},{"line_number":1741,"context_line":""},{"line_number":1742,"context_line":"    \"\"\""},{"line_number":1743,"context_line":"    Perform blocking operations in background threads."},{"line_number":1744,"context_line":""},{"line_number":1745,"context_line":"    Call its methods from within greenlets to green-wait for results without"},{"line_number":1746,"context_line":"    blocking the eventlet reactor (hopefully)."}],"source_content_type":"text/x-python","patch_set":8,"id":"AAAAPH%2F%2F5p0%3D","line":1743,"updated":"2013-05-20 18:52:30.000000000","message":"Perhaps \"background OS threads\"?","commit_id":"7e24345f49c666397043cf58be56ab7679a16445"},{"author":{"_account_id":6198,"name":"Peter Portante","email":"peter.a.portante@gmail.com","username":"peter-a-portante"},"change_message_id":"9405bb255252832c911841a4da82241b25f11266","unresolved":false,"context_lines":[{"line_number":1754,"context_line":"        if nthreads \u003c\u003d 0:"},{"line_number":1755,"context_line":"            return"},{"line_number":1756,"context_line":""},{"line_number":1757,"context_line":"        # We spawn a greenthread whose job it is to pull results from the"},{"line_number":1758,"context_line":"        # worker threads via a real Queue and send them to eventlet Events so"},{"line_number":1759,"context_line":"        # that the calling greenthreads can be awoken."},{"line_number":1760,"context_line":"        #"}],"source_content_type":"text/x-python","patch_set":8,"id":"AAAAPH%2F%2F5pg%3D","line":1757,"updated":"2013-05-20 18:52:30.000000000","message":"Perhaps \"greenlet\" instead of \"greenthread\"?","commit_id":"7e24345f49c666397043cf58be56ab7679a16445"},{"author":{"_account_id":6198,"name":"Peter Portante","email":"peter.a.portante@gmail.com","username":"peter-a-portante"},"change_message_id":"9405bb255252832c911841a4da82241b25f11266","unresolved":false,"context_lines":[{"line_number":1755,"context_line":"            return"},{"line_number":1756,"context_line":""},{"line_number":1757,"context_line":"        # We spawn a greenthread whose job it is to pull results from the"},{"line_number":1758,"context_line":"        # worker threads via a real Queue and send them to eventlet Events so"},{"line_number":1759,"context_line":"        # that the calling greenthreads can be awoken."},{"line_number":1760,"context_line":"        #"},{"line_number":1761,"context_line":"        # Since each OS thread has its own collection of greenthreads, it"}],"source_content_type":"text/x-python","patch_set":8,"id":"AAAAPH%2F%2F5pU%3D","line":1758,"updated":"2013-05-20 18:52:30.000000000","message":"Maybe \"background worker OS threads via a real Queue ...\"","commit_id":"7e24345f49c666397043cf58be56ab7679a16445"},{"author":{"_account_id":6198,"name":"Peter Portante","email":"peter.a.portante@gmail.com","username":"peter-a-portante"},"change_message_id":"9405bb255252832c911841a4da82241b25f11266","unresolved":false,"context_lines":[{"line_number":1756,"context_line":""},{"line_number":1757,"context_line":"        # We spawn a greenthread whose job it is to pull results from the"},{"line_number":1758,"context_line":"        # worker threads via a real Queue and send them to eventlet Events so"},{"line_number":1759,"context_line":"        # that the calling greenthreads can be awoken."},{"line_number":1760,"context_line":"        #"},{"line_number":1761,"context_line":"        # Since each OS thread has its own collection of greenthreads, it"},{"line_number":1762,"context_line":"        # doesn\u0027t work to have the worker thread send stuff to the event, as"}],"source_content_type":"text/x-python","patch_set":8,"id":"AAAAPH%2F%2F5pQ%3D","line":1759,"updated":"2013-05-20 18:52:30.000000000","message":"greenthreads -\u003e greenlets?","commit_id":"7e24345f49c666397043cf58be56ab7679a16445"},{"author":{"_account_id":6198,"name":"Peter Portante","email":"peter.a.portante@gmail.com","username":"peter-a-portante"},"change_message_id":"9405bb255252832c911841a4da82241b25f11266","unresolved":false,"context_lines":[{"line_number":1758,"context_line":"        # worker threads via a real Queue and send them to eventlet Events so"},{"line_number":1759,"context_line":"        # that the calling greenthreads can be awoken."},{"line_number":1760,"context_line":"        #"},{"line_number":1761,"context_line":"        # Since each OS thread has its own collection of greenthreads, it"},{"line_number":1762,"context_line":"        # doesn\u0027t work to have the worker thread send stuff to the event, as"},{"line_number":1763,"context_line":"        # it then notifies its own thread-local eventlet hub to wake up, which"},{"line_number":1764,"context_line":"        # doesn\u0027t do anything to help out the actual calling greenthread over"}],"source_content_type":"text/x-python","patch_set":8,"id":"AAAAPH%2F%2F5pM%3D","line":1761,"updated":"2013-05-20 18:52:30.000000000","message":"greenthreads -\u003e greenlets?","commit_id":"7e24345f49c666397043cf58be56ab7679a16445"},{"author":{"_account_id":6198,"name":"Peter Portante","email":"peter.a.portante@gmail.com","username":"peter-a-portante"},"change_message_id":"9405bb255252832c911841a4da82241b25f11266","unresolved":false,"context_lines":[{"line_number":1761,"context_line":"        # Since each OS thread has its own collection of greenthreads, it"},{"line_number":1762,"context_line":"        # doesn\u0027t work to have the worker thread send stuff to the event, as"},{"line_number":1763,"context_line":"        # it then notifies its own thread-local eventlet hub to wake up, which"},{"line_number":1764,"context_line":"        # doesn\u0027t do anything to help out the actual calling greenthread over"},{"line_number":1765,"context_line":"        # in the main thread."},{"line_number":1766,"context_line":"        #"},{"line_number":1767,"context_line":"        # Thus, each worker sticks its results into a result queue and then"}],"source_content_type":"text/x-python","patch_set":8,"id":"AAAAPH%2F%2F5pI%3D","line":1764,"updated":"2013-05-20 18:52:30.000000000","message":"greenthread -\u003e greenlet?","commit_id":"7e24345f49c666397043cf58be56ab7679a16445"},{"author":{"_account_id":6198,"name":"Peter Portante","email":"peter.a.portante@gmail.com","username":"peter-a-portante"},"change_message_id":"9405bb255252832c911841a4da82241b25f11266","unresolved":false,"context_lines":[{"line_number":1766,"context_line":"        #"},{"line_number":1767,"context_line":"        # Thus, each worker sticks its results into a result queue and then"},{"line_number":1768,"context_line":"        # writes a byte to a pipe, signaling the result-consuming greenlet (in"},{"line_number":1769,"context_line":"        # the main thread) to wake up and consume results."},{"line_number":1770,"context_line":"        #"},{"line_number":1771,"context_line":"        # This is all stuff that eventlet.tpool does, but that code can\u0027t have"},{"line_number":1772,"context_line":"        # multiple instances instantiated. Since the object server uses one"}],"source_content_type":"text/x-python","patch_set":8,"id":"AAAAPH%2F%2F5jE%3D","line":1769,"updated":"2013-05-20 18:52:30.000000000","message":"Had you considered using eventfds per disk-pool so that you can get rid of the separate greenlets per object? I think you could get rid of the pipes and events and merge it down to just the eventfds.","commit_id":"7e24345f49c666397043cf58be56ab7679a16445"},{"author":{"_account_id":2622,"name":"Samuel Merritt","email":"spam+launchpad@andcheese.org","username":"torgomatic"},"change_message_id":"7bf3f867f8ee0bc5e255132b23cebc59c4d5bbb7","unresolved":false,"context_lines":[{"line_number":1766,"context_line":"        #"},{"line_number":1767,"context_line":"        # Thus, each worker sticks its results into a result queue and then"},{"line_number":1768,"context_line":"        # writes a byte to a pipe, signaling the result-consuming greenlet (in"},{"line_number":1769,"context_line":"        # the main thread) to wake up and consume results."},{"line_number":1770,"context_line":"        #"},{"line_number":1771,"context_line":"        # This is all stuff that eventlet.tpool does, but that code can\u0027t have"},{"line_number":1772,"context_line":"        # multiple instances instantiated. Since the object server uses one"}],"source_content_type":"text/x-python","patch_set":8,"id":"AAAAPH%2F%2F5UE%3D","line":1769,"in_reply_to":"AAAAPH%2F%2F5jE%3D","updated":"2013-05-20 19:50:23.000000000","message":"I thought about it, but I went with pipes for two reasons:\n\nFirst, they\u0027re cross-platform. eventfd() is a Linux-specific call. Now, that\u0027s not to say that Swift shouldn\u0027t use eventfds if available, but for the initial draft of this stuff, I went with the cross-platform thing. I think there are folks (Softlayer?) out there running Swift on FreeBSD, for example.\n\nSecond, there\u0027s already eventlet.greenio.GreenPipe for handing the trampolining and such around pipes, so that\u0027s code I didn\u0027t have to write. :)\n\nI\u0027m happy to entertain the notion of using eventfds as a future optimization, though.","commit_id":"7e24345f49c666397043cf58be56ab7679a16445"},{"author":{"_account_id":6198,"name":"Peter Portante","email":"peter.a.portante@gmail.com","username":"peter-a-portante"},"change_message_id":"9405bb255252832c911841a4da82241b25f11266","unresolved":false,"context_lines":[{"line_number":1850,"context_line":"        :raises: whatever func raises"},{"line_number":1851,"context_line":"        \"\"\""},{"line_number":1852,"context_line":"        if self.nthreads \u003c\u003d 0:"},{"line_number":1853,"context_line":"            return func(*args, **kwargs)"},{"line_number":1854,"context_line":""},{"line_number":1855,"context_line":"        ev \u003d event.Event()"},{"line_number":1856,"context_line":"        self._run_queue.put((ev, func, args, kwargs), block\u003dFalse)"}],"source_content_type":"text/x-python","patch_set":8,"id":"AAAAPH%2F%2F5hw%3D","line":1853,"updated":"2013-05-20 18:52:30.000000000","message":"Should this use tpool?","commit_id":"7e24345f49c666397043cf58be56ab7679a16445"},{"author":{"_account_id":2622,"name":"Samuel Merritt","email":"spam+launchpad@andcheese.org","username":"torgomatic"},"change_message_id":"7bf3f867f8ee0bc5e255132b23cebc59c4d5bbb7","unresolved":false,"context_lines":[{"line_number":1850,"context_line":"        :raises: whatever func raises"},{"line_number":1851,"context_line":"        \"\"\""},{"line_number":1852,"context_line":"        if self.nthreads \u003c\u003d 0:"},{"line_number":1853,"context_line":"            return func(*args, **kwargs)"},{"line_number":1854,"context_line":""},{"line_number":1855,"context_line":"        ev \u003d event.Event()"},{"line_number":1856,"context_line":"        self._run_queue.put((ev, func, args, kwargs), block\u003dFalse)"}],"source_content_type":"text/x-python","patch_set":8,"id":"AAAAPH%2F%2F5Tc%3D","line":1853,"in_reply_to":"AAAAPH%2F%2F5hw%3D","updated":"2013-05-20 19:50:23.000000000","message":"Nope.\n\nBasically, naming is hard.\n\nrun_in_thread is there to run something in a thread if there are threads to run in, or to just run the thing in the calling thread otherwise.\n\nforce_run_in_thread is there to run something in the per-disk threadpool if there are threads to run in, or to fall back to tpool otherwise.\n\nIt\u0027s weird, but it preserves the old behavior and performance characteristics when threads_per_disk is 0.","commit_id":"7e24345f49c666397043cf58be56ab7679a16445"},{"author":{"_account_id":6198,"name":"Peter Portante","email":"peter.a.portante@gmail.com","username":"peter-a-portante"},"change_message_id":"2caf7ebbe97ae17cb0446f7f7146b9599dca594e","unresolved":false,"context_lines":[{"line_number":1828,"context_line":"            except BaseException, err:"},{"line_number":1829,"context_line":"                result_queue.put((ev, False, err))"},{"line_number":1830,"context_line":"            finally:"},{"line_number":1831,"context_line":"                work_queue.task_done()"},{"line_number":1832,"context_line":"                os.write(self.wpipe, self.BYTE)"},{"line_number":1833,"context_line":""},{"line_number":1834,"context_line":"    def _consume_results(self, queue):"}],"source_content_type":"text/x-python","patch_set":9,"id":"AAAAPH%2F%2FuIY%3D","line":1831,"updated":"2013-05-24 14:44:45.000000000","message":"If we get an exception here, does not sending the BYTE below cause a problem?","commit_id":"d2edae3c74b6c4c496b551ceca391719f390d7b7"},{"author":{"_account_id":917,"name":"Chuck Thier","email":"cthier@gmail.com","username":"cthier"},"change_message_id":"f580128a474ff46953473bdc4326ac7d1db32239","unresolved":false,"context_lines":[{"line_number":1828,"context_line":"            except BaseException, err:"},{"line_number":1829,"context_line":"                result_queue.put((ev, False, err))"},{"line_number":1830,"context_line":"            finally:"},{"line_number":1831,"context_line":"                work_queue.task_done()"},{"line_number":1832,"context_line":"                os.write(self.wpipe, self.BYTE)"},{"line_number":1833,"context_line":""},{"line_number":1834,"context_line":"    def _consume_results(self, queue):"}],"source_content_type":"text/x-python","patch_set":9,"id":"AAAAPH%2F%2FtyM%3D","line":1831,"in_reply_to":"AAAAPH%2F%2FuIY%3D","updated":"2013-05-24 16:44:11.000000000","message":"The only way I could see that happening is if the queue gets out of sync, and task_done gets called more than required, which doesn\u0027t seem like it would happen.","commit_id":"d2edae3c74b6c4c496b551ceca391719f390d7b7"},{"author":{"_account_id":6198,"name":"Peter Portante","email":"peter.a.portante@gmail.com","username":"peter-a-portante"},"change_message_id":"2caf7ebbe97ae17cb0446f7f7146b9599dca594e","unresolved":false,"context_lines":[{"line_number":1892,"context_line":"        def inner():"},{"line_number":1893,"context_line":"            try:"},{"line_number":1894,"context_line":"                return (True, func(*args, **kwargs))"},{"line_number":1895,"context_line":"            except (Timeout, BaseException), err:"},{"line_number":1896,"context_line":"                return (False, err)"},{"line_number":1897,"context_line":""},{"line_number":1898,"context_line":"        success, result \u003d tpool.execute(inner)"}],"source_content_type":"text/x-python","patch_set":9,"id":"AAAAPH%2F%2FuII%3D","line":1895,"updated":"2013-05-24 14:44:45.000000000","message":"Should we start using the \"as\" syntax for exceptions?","commit_id":"d2edae3c74b6c4c496b551ceca391719f390d7b7"},{"author":{"_account_id":2622,"name":"Samuel Merritt","email":"spam+launchpad@andcheese.org","username":"torgomatic"},"change_message_id":"76123bdf5c812a537384e086fd722f5ea6c51d6b","unresolved":false,"context_lines":[{"line_number":1892,"context_line":"        def inner():"},{"line_number":1893,"context_line":"            try:"},{"line_number":1894,"context_line":"                return (True, func(*args, **kwargs))"},{"line_number":1895,"context_line":"            except (Timeout, BaseException), err:"},{"line_number":1896,"context_line":"                return (False, err)"},{"line_number":1897,"context_line":""},{"line_number":1898,"context_line":"        success, result \u003d tpool.execute(inner)"}],"source_content_type":"text/x-python","patch_set":9,"id":"AAAAPH%2F%2Ftuo%3D","line":1895,"in_reply_to":"AAAAPH%2F%2FtyI%3D","updated":"2013-05-24 17:11:24.000000000","message":"Yeah, it\u0027s a pretty simple mechanical transformation.\n\nHowever, since the \"as\" syntax works in 2.6 and 2.7, I\u0027ll change it in the next patchset (since I have to do one anyway). It\u0027s probably a good habit to start getting into.\n\n(There\u0027s already \"as\" syntax in Swift, like in recon middleware and some stuff in utils, so it\u0027s certainly known to work.)","commit_id":"d2edae3c74b6c4c496b551ceca391719f390d7b7"},{"author":{"_account_id":917,"name":"Chuck Thier","email":"cthier@gmail.com","username":"cthier"},"change_message_id":"f580128a474ff46953473bdc4326ac7d1db32239","unresolved":false,"context_lines":[{"line_number":1892,"context_line":"        def inner():"},{"line_number":1893,"context_line":"            try:"},{"line_number":1894,"context_line":"                return (True, func(*args, **kwargs))"},{"line_number":1895,"context_line":"            except (Timeout, BaseException), err:"},{"line_number":1896,"context_line":"                return (False, err)"},{"line_number":1897,"context_line":""},{"line_number":1898,"context_line":"        success, result \u003d tpool.execute(inner)"}],"source_content_type":"text/x-python","patch_set":9,"id":"AAAAPH%2F%2FtyI%3D","line":1895,"in_reply_to":"AAAAPH%2F%2FuII%3D","updated":"2013-05-24 16:44:11.000000000","message":"I wouldn\u0027t worry about this yet.  There are tools that make most of those conversions pretty easy once we get to python 3 compatibility (assuming that\u0027s the reason for the question)","commit_id":"d2edae3c74b6c4c496b551ceca391719f390d7b7"}],"swift/obj/server.py":[{"author":{"_account_id":4108,"name":"David Hadas","email":"davidh@il.ibm.com","username":"david-hadas"},"change_message_id":"a2699684abef0c85d586652076117b0d33d23c84","unresolved":false,"context_lines":[{"line_number":137,"context_line":"        self.quarantined_dir \u003d None"},{"line_number":138,"context_line":"        self.keep_cache \u003d False"},{"line_number":139,"context_line":"        self.suppress_file_closing \u003d False"},{"line_number":140,"context_line":"        self.threadpool \u003d threadpool or DummyThreadPool()"},{"line_number":141,"context_line":"        if not os.path.exists(self.datadir):"},{"line_number":142,"context_line":"            return"},{"line_number":143,"context_line":"        files \u003d sorted(os.listdir(self.datadir), reverse\u003dTrue)"}],"source_content_type":"text/x-python","patch_set":1,"id":"AAAAOn%2F%2F0Lk%3D","line":140,"updated":"2013-04-22 13:07:22.000000000","message":"Preferably use ThreadPool(nthreads\u003d0) insetad of DummyThreadPool() for the same results","commit_id":"46f1b3e20375c164d536b1805032eee5910fd26d"},{"author":{"_account_id":4108,"name":"David Hadas","email":"davidh@il.ibm.com","username":"david-hadas"},"change_message_id":"a2699684abef0c85d586652076117b0d33d23c84","unresolved":false,"context_lines":[{"line_number":332,"context_line":"            invalidate_hash(os.path.dirname(self.datadir))"},{"line_number":333,"context_line":"            renamer(self.tmppath,"},{"line_number":334,"context_line":"                    os.path.join(self.datadir, timestamp + extension))"},{"line_number":335,"context_line":"        self.threadpool.force_run_in_thread(finalize_put)"},{"line_number":336,"context_line":"        self.metadata \u003d metadata"},{"line_number":337,"context_line":""},{"line_number":338,"context_line":"    def put_metadata(self, metadata, tombstone\u003dFalse):"}],"source_content_type":"text/x-python","patch_set":1,"id":"AAAAOn%2F%2F0LM%3D","line":335,"updated":"2013-04-22 13:07:22.000000000","message":"How about eliminating the \u0027force_run_in_thread() interface and instead have one worker always created for such tasks - i.e. create another ThreadPool with one worker or have the existing ThreadPool to always have at least one thread to be used.","commit_id":"46f1b3e20375c164d536b1805032eee5910fd26d"},{"author":{"_account_id":4108,"name":"David Hadas","email":"davidh@il.ibm.com","username":"david-hadas"},"change_message_id":"a2699684abef0c85d586652076117b0d33d23c84","unresolved":false,"context_lines":[{"line_number":356,"context_line":"        timestamp \u003d normalize_timestamp(timestamp)"},{"line_number":357,"context_line":"        for fname in os.listdir(self.datadir):"},{"line_number":358,"context_line":"            if fname \u003c timestamp:"},{"line_number":359,"context_line":"                try:"},{"line_number":360,"context_line":"                    self.threadpool.run_in_thread("},{"line_number":361,"context_line":"                        os.unlink,"},{"line_number":362,"context_line":"                        os.path.join(self.datadir, fname))"}],"source_content_type":"text/x-python","patch_set":1,"id":"AAAAOn%2F%2F0LI%3D","line":359,"updated":"2013-04-22 13:07:22.000000000","message":"I wonder when we choose to force a separate thread and when do we choose not to?","commit_id":"46f1b3e20375c164d536b1805032eee5910fd26d"},{"author":{"_account_id":917,"name":"Chuck Thier","email":"cthier@gmail.com","username":"cthier"},"change_message_id":"1061afd23c4e57799d3804c4e090d6038df9883b","unresolved":false,"context_lines":[{"line_number":20,"context_line":"import errno"},{"line_number":21,"context_line":"import os"},{"line_number":22,"context_line":"import time"},{"line_number":23,"context_line":"import threading"},{"line_number":24,"context_line":"import traceback"},{"line_number":25,"context_line":"from collections import defaultdict"},{"line_number":26,"context_line":"from datetime import datetime"}],"source_content_type":"text/x-python","patch_set":2,"id":"AAAAOn%2F%2FnlU%3D","line":23,"updated":"2013-05-01 22:00:11.000000000","message":"This isn\u0027t used anywhere","commit_id":"c436874061b3428a96df19638f840c9d958577bf"},{"author":{"_account_id":2622,"name":"Samuel Merritt","email":"spam+launchpad@andcheese.org","username":"torgomatic"},"change_message_id":"3506a12151f423e98e700fb8d46aad74b2900965","unresolved":false,"context_lines":[{"line_number":20,"context_line":"import errno"},{"line_number":21,"context_line":"import os"},{"line_number":22,"context_line":"import time"},{"line_number":23,"context_line":"import threading"},{"line_number":24,"context_line":"import traceback"},{"line_number":25,"context_line":"from collections import defaultdict"},{"line_number":26,"context_line":"from datetime import datetime"}],"source_content_type":"text/x-python","patch_set":2,"id":"AAAAOn%2F%2FnIo%3D","line":23,"in_reply_to":"AAAAOn%2F%2FnlU%3D","updated":"2013-05-01 22:39:20.000000000","message":"Thanks. Will fix.","commit_id":"c436874061b3428a96df19638f840c9d958577bf"},{"author":{"_account_id":6198,"name":"Peter Portante","email":"peter.a.portante@gmail.com","username":"peter-a-portante"},"change_message_id":"89b3a461f2f9143f3dca781228d03b3d855b2751","unresolved":false,"context_lines":[{"line_number":453,"context_line":"        self.slow \u003d int(conf.get(\u0027slow\u0027, 0))"},{"line_number":454,"context_line":"        self.bytes_per_sync \u003d int(conf.get(\u0027mb_per_sync\u0027, 512)) * 1024 * 1024"},{"line_number":455,"context_line":"        self.threads_per_disk \u003d int(conf.get(\u0027threads_per_disk\u0027, \u00270\u0027))"},{"line_number":456,"context_line":"        self.threadpools \u003d defaultdict("},{"line_number":457,"context_line":"            lambda: ThreadPool(nthreads\u003dself.threads_per_disk))"},{"line_number":458,"context_line":"        default_allowed_headers \u003d \u0027\u0027\u0027"},{"line_number":459,"context_line":"            content-disposition,"}],"source_content_type":"text/x-python","patch_set":6,"id":"AAAAOn%2F%2FaGQ%3D","line":456,"updated":"2013-05-10 15:49:55.000000000","message":"Can we make this be a DiskFile implementation attribute?","commit_id":"22778de2e8101fbc213f063c65a1197245be0f5a"},{"author":{"_account_id":6198,"name":"Peter Portante","email":"peter.a.portante@gmail.com","username":"peter-a-portante"},"change_message_id":"1849fd7b2b456e1ee12d41b5d779dcf7e7514df9","unresolved":false,"context_lines":[{"line_number":453,"context_line":"        self.slow \u003d int(conf.get(\u0027slow\u0027, 0))"},{"line_number":454,"context_line":"        self.bytes_per_sync \u003d int(conf.get(\u0027mb_per_sync\u0027, 512)) * 1024 * 1024"},{"line_number":455,"context_line":"        self.threads_per_disk \u003d int(conf.get(\u0027threads_per_disk\u0027, \u00270\u0027))"},{"line_number":456,"context_line":"        self.threadpools \u003d defaultdict("},{"line_number":457,"context_line":"            lambda: ThreadPool(nthreads\u003dself.threads_per_disk))"},{"line_number":458,"context_line":"        default_allowed_headers \u003d \u0027\u0027\u0027"},{"line_number":459,"context_line":"            content-disposition,"}],"source_content_type":"text/x-python","patch_set":6,"id":"AAAAOn%2F%2FZx8%3D","line":456,"in_reply_to":"AAAAOn%2F%2FZ0U%3D","updated":"2013-05-10 18:46:35.000000000","message":"I don\u0027t mean to say that the threadpool class should only be scoped inside DiskFile, but that for the places where DiskFile is used the threadpool should be under that object.","commit_id":"22778de2e8101fbc213f063c65a1197245be0f5a"},{"author":{"_account_id":2622,"name":"Samuel Merritt","email":"spam+launchpad@andcheese.org","username":"torgomatic"},"change_message_id":"538df82f1e010add91851bdd591e66bf72e6b195","unresolved":false,"context_lines":[{"line_number":453,"context_line":"        self.slow \u003d int(conf.get(\u0027slow\u0027, 0))"},{"line_number":454,"context_line":"        self.bytes_per_sync \u003d int(conf.get(\u0027mb_per_sync\u0027, 512)) * 1024 * 1024"},{"line_number":455,"context_line":"        self.threads_per_disk \u003d int(conf.get(\u0027threads_per_disk\u0027, \u00270\u0027))"},{"line_number":456,"context_line":"        self.threadpools \u003d defaultdict("},{"line_number":457,"context_line":"            lambda: ThreadPool(nthreads\u003dself.threads_per_disk))"},{"line_number":458,"context_line":"        default_allowed_headers \u003d \u0027\u0027\u0027"},{"line_number":459,"context_line":"            content-disposition,"}],"source_content_type":"text/x-python","patch_set":6,"id":"AAAAOn%2F%2FZ0U%3D","line":456,"in_reply_to":"AAAAOn%2F%2FaGQ%3D","updated":"2013-05-10 18:34:02.000000000","message":"Not currently. If all the threadpool users eventually move into DiskFile, then it might be possible. Right now, though, there\u0027s calls in REPLICATE() and async_update() that use the threadpool outside of DiskFile, and I\u0027d rather have just one pool per disk.","commit_id":"22778de2e8101fbc213f063c65a1197245be0f5a"},{"author":{"_account_id":6198,"name":"Peter Portante","email":"peter.a.portante@gmail.com","username":"peter-a-portante"},"change_message_id":"89b3a461f2f9143f3dca781228d03b3d855b2751","unresolved":false,"context_lines":[{"line_number":705,"context_line":"                        return HTTPRequestTimeout(request\u003drequest)"},{"line_number":706,"context_line":"                    etag.update(chunk)"},{"line_number":707,"context_line":"                    while chunk:"},{"line_number":708,"context_line":"                        written \u003d os.write(fd, chunk)"},{"line_number":709,"context_line":"                        chunk \u003d chunk[written:]"},{"line_number":710,"context_line":"                    # For large files sync every 512MB (by default) written"},{"line_number":711,"context_line":"                    if upload_size - last_sync \u003e\u003d self.bytes_per_sync:"}],"source_content_type":"text/x-python","patch_set":6,"id":"AAAAOn%2F%2FaGc%3D","line":708,"updated":"2013-05-10 15:49:55.000000000","message":"Shouldn\u0027t this write also be in a thread pool thread?","commit_id":"22778de2e8101fbc213f063c65a1197245be0f5a"},{"author":{"_account_id":2622,"name":"Samuel Merritt","email":"spam+launchpad@andcheese.org","username":"torgomatic"},"change_message_id":"538df82f1e010add91851bdd591e66bf72e6b195","unresolved":false,"context_lines":[{"line_number":705,"context_line":"                        return HTTPRequestTimeout(request\u003drequest)"},{"line_number":706,"context_line":"                    etag.update(chunk)"},{"line_number":707,"context_line":"                    while chunk:"},{"line_number":708,"context_line":"                        written \u003d os.write(fd, chunk)"},{"line_number":709,"context_line":"                        chunk \u003d chunk[written:]"},{"line_number":710,"context_line":"                    # For large files sync every 512MB (by default) written"},{"line_number":711,"context_line":"                    if upload_size - last_sync \u003e\u003d self.bytes_per_sync:"}],"source_content_type":"text/x-python","patch_set":6,"id":"AAAAOn%2F%2FZ1I%3D","line":708,"in_reply_to":"AAAAOn%2F%2FaGc%3D","updated":"2013-05-10 18:34:02.000000000","message":"Probably.\n\nIn the normal case, writes are really fast because they just go to buffercache. I guess when there\u0027s memory pressure, though, this could be slow, so it probably ought to go in a dedicated thread.\n\nWill fix.","commit_id":"22778de2e8101fbc213f063c65a1197245be0f5a"},{"author":{"_account_id":6198,"name":"Peter Portante","email":"peter.a.portante@gmail.com","username":"peter-a-portante"},"change_message_id":"2caf7ebbe97ae17cb0446f7f7146b9599dca594e","unresolved":false,"context_lines":[{"line_number":399,"context_line":"        extension \u003d \u0027.ts\u0027 if tombstone else \u0027.meta\u0027"},{"line_number":400,"context_line":"        with self.writer() as writer:"},{"line_number":401,"context_line":"            writer.put(metadata, extension\u003dextension)"},{"line_number":402,"context_line":""},{"line_number":403,"context_line":"    def unlinkold(self, timestamp):"},{"line_number":404,"context_line":"        \"\"\""},{"line_number":405,"context_line":"        Remove any older versions of the object file.  Any file that has an"}],"source_content_type":"text/x-python","patch_set":9,"id":"AAAAPH%2F%2FuHg%3D","line":402,"updated":"2013-05-24 14:44:45.000000000","message":"Seems like the entire unlink loop should be run in the thread. Directory reads can block when they are not in cache.","commit_id":"d2edae3c74b6c4c496b551ceca391719f390d7b7"},{"author":{"_account_id":2622,"name":"Samuel Merritt","email":"spam+launchpad@andcheese.org","username":"torgomatic"},"change_message_id":"76123bdf5c812a537384e086fd722f5ea6c51d6b","unresolved":false,"context_lines":[{"line_number":399,"context_line":"        extension \u003d \u0027.ts\u0027 if tombstone else \u0027.meta\u0027"},{"line_number":400,"context_line":"        with self.writer() as writer:"},{"line_number":401,"context_line":"            writer.put(metadata, extension\u003dextension)"},{"line_number":402,"context_line":""},{"line_number":403,"context_line":"    def unlinkold(self, timestamp):"},{"line_number":404,"context_line":"        \"\"\""},{"line_number":405,"context_line":"        Remove any older versions of the object file.  Any file that has an"}],"source_content_type":"text/x-python","patch_set":9,"id":"AAAAPH%2F%2Ftuk%3D","line":402,"in_reply_to":"AAAAPH%2F%2FuHg%3D","updated":"2013-05-24 17:11:24.000000000","message":"This is a good idea. I\u0027ll do it in the next patchset.\n\nThanks!","commit_id":"d2edae3c74b6c4c496b551ceca391719f390d7b7"}],"test/unit/common/test_utils.py":[{"author":{"_account_id":6198,"name":"Peter Portante","email":"peter.a.portante@gmail.com","username":"peter-a-portante"},"change_message_id":"9405bb255252832c911841a4da82241b25f11266","unresolved":false,"context_lines":[{"line_number":1753,"context_line":"                utils.fsync(12345)"},{"line_number":1754,"context_line":"                self.assertEquals(called, [12345])"},{"line_number":1755,"context_line":""},{"line_number":1756,"context_line":"    def test_threadpool(self):"},{"line_number":1757,"context_line":"        tp \u003d utils.ThreadPool(1)"},{"line_number":1758,"context_line":""},{"line_number":1759,"context_line":"        def thread_id():"}],"source_content_type":"text/x-python","patch_set":8,"id":"AAAAPH%2F%2F5hg%3D","line":1756,"updated":"2013-05-20 18:52:30.000000000","message":"Can you create a different test case per function method of ThreadPool, and perhaps make this its own test class?","commit_id":"7e24345f49c666397043cf58be56ab7679a16445"},{"author":{"_account_id":2622,"name":"Samuel Merritt","email":"spam+launchpad@andcheese.org","username":"torgomatic"},"change_message_id":"7bf3f867f8ee0bc5e255132b23cebc59c4d5bbb7","unresolved":false,"context_lines":[{"line_number":1753,"context_line":"                utils.fsync(12345)"},{"line_number":1754,"context_line":"                self.assertEquals(called, [12345])"},{"line_number":1755,"context_line":""},{"line_number":1756,"context_line":"    def test_threadpool(self):"},{"line_number":1757,"context_line":"        tp \u003d utils.ThreadPool(1)"},{"line_number":1758,"context_line":""},{"line_number":1759,"context_line":"        def thread_id():"}],"source_content_type":"text/x-python","patch_set":8,"id":"AAAAPH%2F%2F5TQ%3D","line":1756,"in_reply_to":"AAAAPH%2F%2F5hg%3D","updated":"2013-05-20 19:50:23.000000000","message":"Sure, I could split this up more.","commit_id":"7e24345f49c666397043cf58be56ab7679a16445"}]}