)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":4108,"name":"David Hadas","email":"davidh@il.ibm.com","username":"david-hadas"},"change_message_id":"d76866840391d45a5c7c458b571027fc18f3f643","unresolved":false,"context_lines":[{"line_number":7,"context_line":"Add parallelism to object expirer daemon."},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"Two types of paralleism are added:"},{"line_number":10,"context_line":""},{"line_number":11,"context_line":"- threads to speed up what a single process does"},{"line_number":12,"context_line":"- a way to run multiple daemons to work on different parts of the work"},{"line_number":13,"context_line":""}],"source_content_type":"text/x-gerrit-commit-message","patch_set":4,"id":"AAAAOn%2F%2FYqI%3D","line":10,"updated":"2013-05-11 21:58:49.000000000","message":"I just dont get it  - why is such parallelism needed in the expirer?\nAnd even if parallelism is needed - do we really need to use seperate processes with individual config files? \n\nWhat  is wrong with threads?\nWhy do we need multiple processes?","commit_id":"c10681dc6247f7c38f3c723406ffb4f9cb8111f7"},{"author":{"_account_id":1216,"name":"gholt","email":"z-launchpad@brim.net","username":"gholt"},"change_message_id":"972c8974c1dfd3e8c6887b2059eec8fbaeea8629","unresolved":false,"context_lines":[{"line_number":6,"context_line":""},{"line_number":7,"context_line":"Add parallelism to object expirer daemon."},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"Two types of paralleism are added:"},{"line_number":10,"context_line":""},{"line_number":11,"context_line":"- threads to speed up what a single process does"},{"line_number":12,"context_line":"- a way to run multiple daemons to work on different parts of the work"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":7,"id":"AAAAPH%2F%2FoPM%3D","line":9,"updated":"2013-05-28 23:57:27.000000000","message":"minor: paralleism should be parallelism","commit_id":"8131622e58bc8172a00792acfab5f0ea5a2644d7"}],"bin/swift-object-expirer":[{"author":{"_account_id":1216,"name":"gholt","email":"z-launchpad@brim.net","username":"gholt"},"change_message_id":"972c8974c1dfd3e8c6887b2059eec8fbaeea8629","unresolved":false,"context_lines":[{"line_number":27,"context_line":"                      \"option to do all the work in one process\")"},{"line_number":28,"context_line":"    parser.add_option(\u0027--part\u0027, dest\u003d\u0027part\u0027,"},{"line_number":29,"context_line":"                      help\u003d\"Part for this process to work on, don\u0027t use \""},{"line_number":30,"context_line":"                      \"this option to do all the work in one process\")"},{"line_number":31,"context_line":"    conf_file, options \u003d parse_options(parser\u003dparser, once\u003dTrue)"},{"line_number":32,"context_line":"    run_daemon(ObjectExpirer, conf_file, **options)"}],"source_content_type":"application/octet-stream","patch_set":7,"id":"AAAAPH%2F%2FoOc%3D","line":30,"updated":"2013-05-28 23:57:27.000000000","message":"minor: Consider using --processes and --process instead since \"part\" is already an overloaded term representing a ring partition.","commit_id":"8131622e58bc8172a00792acfab5f0ea5a2644d7"}],"doc/source/overview_expiring_objects.rst":[{"author":{"_account_id":1216,"name":"gholt","email":"z-launchpad@brim.net","username":"gholt"},"change_message_id":"972c8974c1dfd3e8c6887b2059eec8fbaeea8629","unresolved":false,"context_lines":[{"line_number":11,"context_line":"As expiring objects are added to the system, the object servers will record the expirations in a hidden ``.expiring_objects`` account for the ``swift-object-expirer`` to handle later."},{"line_number":12,"context_line":""},{"line_number":13,"context_line":"Just one instance of the ``swift-object-expirer`` daemon needs to run for a cluster. This isn\u0027t exactly automatic failover high availability, but if this daemon doesn\u0027t run for a few hours it should not be any real issue. The expired-but-not-yet-deleted objects will still ``404 Not Found`` if someone tries to ``GET`` or ``HEAD`` them and they\u0027ll just be deleted a bit later when the daemon is restarted."},{"line_number":14,"context_line":""},{"line_number":15,"context_line":"The daemon uses the ``/etc/swift/object-expirer.conf`` by default, and here is a quick sample conf file::"},{"line_number":16,"context_line":""},{"line_number":17,"context_line":"    [DEFAULT]"}],"source_content_type":"text/x-rst","patch_set":7,"id":"AAAAPH%2F%2FoOM%3D","side":"PARENT","line":14,"updated":"2013-05-28 23:57:27.000000000","message":"minor: Maybe explain how the work is split.","commit_id":"d69fa437cd7d4b7c92a62d3005399ffa57611551"},{"author":{"_account_id":1216,"name":"gholt","email":"z-launchpad@brim.net","username":"gholt"},"change_message_id":"972c8974c1dfd3e8c6887b2059eec8fbaeea8629","unresolved":false,"context_lines":[{"line_number":25,"context_line":"    # log_name \u003d swift"},{"line_number":26,"context_line":"    # log_facility \u003d LOG_LOCAL0"},{"line_number":27,"context_line":"    # log_level \u003d INFO"},{"line_number":28,"context_line":""},{"line_number":29,"context_line":"    [object-expirer]"},{"line_number":30,"context_line":"    interval \u003d 300"},{"line_number":31,"context_line":""}],"source_content_type":"text/x-rst","patch_set":7,"id":"AAAAPH%2F%2FoOE%3D","line":28,"updated":"2013-05-28 23:57:27.000000000","message":"minor: Why all the new double spacing? Does this serve some purpose I\u0027m not getting?","commit_id":"8131622e58bc8172a00792acfab5f0ea5a2644d7"},{"author":{"_account_id":4108,"name":"David Hadas","email":"davidh@il.ibm.com","username":"david-hadas"},"change_message_id":"7084c929a1211990a2cd2062a6955a5d32a2f4f2","unresolved":false,"context_lines":[{"line_number":13,"context_line":"Usually, just one instance of the ``swift-object-expirer`` daemon needs to run for a cluster. This isn\u0027t exactly automatic failover high availability, but if this daemon doesn\u0027t run for a few hours it should not be any real issue. The expired-but-not-yet-deleted objects will still ``404 Not Found`` if someone tries to ``GET`` or ``HEAD`` them and they\u0027ll just be deleted a bit later when the daemon is restarted."},{"line_number":14,"context_line":""},{"line_number":15,"context_line":"By default, the ``swift-object-expirer`` daemon will run with a concurrency of 1024.  This is typically a sufficient amount of parallelism.  However, it is possible to run multiple daemons to do different parts of the work if that proves necessary (see the sample config file for details)."},{"line_number":16,"context_line":""},{"line_number":17,"context_line":"To run the ``swift-object-expirer`` as multiple processes, set ``processes`` to the number of processes (either in the config file or on the command line).  Then run one process for each part.  Use ``process`` to specify the part of the work to be done by a process using the command line or the config.  So, for example, if you\u0027d like to run three processes, set ``processes`` to 3 and run three processes with ``process`` set to 0, 1, and 2 for the three processes.  If multiple processes are used, it\u0027s necessary to run one for each part of the work or that part of the work will not be done."},{"line_number":18,"context_line":""},{"line_number":19,"context_line":"The daemon uses the ``/etc/swift/object-expirer.conf`` by default, and here is a quick sample conf file::"}],"source_content_type":"text/x-rst","patch_set":9,"id":"AAAAPX%2F%2F6Xk%3D","line":16,"updated":"2013-06-06 06:14:40.000000000","message":"This is a complex configuration that can be avoided and was created by the specific implementation. What appends if I messed the configuration files ?, how do I know that I messed the configuration?  \n\nAll this is simply not needed since we can achieve the same without using os level processes","commit_id":"bfd7aa535f611939173304646c4ac05e0656a1e1"}],"etc/object-expirer.conf-sample":[{"author":{"_account_id":4108,"name":"David Hadas","email":"davidh@il.ibm.com","username":"david-hadas"},"change_message_id":"d76866840391d45a5c7c458b571027fc18f3f643","unresolved":false,"context_lines":[{"line_number":26,"context_line":"# report_interval \u003d 300"},{"line_number":27,"context_line":"# worker_count is the number of threads to use to do the work, the number of"},{"line_number":28,"context_line":"# concurrent delete requests"},{"line_number":29,"context_line":"# worker_count \u003d 0"},{"line_number":30,"context_line":"# parts is now many parts to divide the work into, one part per process that"},{"line_number":31,"context_line":"#   will be doing the work"},{"line_number":32,"context_line":"# parts set 0 means that a single process will be doing all the work"}],"source_content_type":"application/octet-stream","patch_set":4,"id":"AAAAOn%2F%2FYqY%3D","line":29,"updated":"2013-05-11 21:58:49.000000000","message":"I find these config parameter completely 1. confusing 2. not needed 3. badly named.\n\nI suggest to find a way to not require these parameters","commit_id":"c10681dc6247f7c38f3c723406ffb4f9cb8111f7"},{"author":{"_account_id":2696,"name":"Darrell Bishop","email":"dbishop@nvidia.com","username":"darrellb"},"change_message_id":"9c4d444574e333a569c49db6a9651489217065fc","unresolved":false,"context_lines":[{"line_number":25,"context_line":"# auto_create_account_prefix \u003d ."},{"line_number":26,"context_line":"# report_interval \u003d 300"},{"line_number":27,"context_line":"# thread_count is the number of threads to use to do the work, the number of"},{"line_number":28,"context_line":"# concurrent delete requests, thread_coount must be set to at least 1"},{"line_number":29,"context_line":"# thread_count \u003d 1024"},{"line_number":30,"context_line":"# parts is now many parts to divide the work into, one part per process that"},{"line_number":31,"context_line":"#   will be doing the work"}],"source_content_type":"application/octet-stream","patch_set":6,"id":"AAAAPH%2F%2F6LA%3D","line":28,"updated":"2013-05-20 15:47:30.000000000","message":"thread_count misspelled; consider changing last comma to semicolon.","commit_id":"f823dd8d60d5c2b1cd69560d7d66bd94fb52becd"},{"author":{"_account_id":1216,"name":"gholt","email":"z-launchpad@brim.net","username":"gholt"},"change_message_id":"972c8974c1dfd3e8c6887b2059eec8fbaeea8629","unresolved":false,"context_lines":[{"line_number":26,"context_line":"# report_interval \u003d 300"},{"line_number":27,"context_line":"# thread_count is the number of threads to use to do the work, the number of"},{"line_number":28,"context_line":"# concurrent delete requests.  thread_count must be set to at least 1."},{"line_number":29,"context_line":"# thread_count \u003d 1024"},{"line_number":30,"context_line":"# parts is now many parts to divide the work into, one part per process that"},{"line_number":31,"context_line":"#   will be doing the work"},{"line_number":32,"context_line":"# parts set 0 means that a single process will be doing all the work"}],"source_content_type":"application/octet-stream","patch_set":7,"id":"AAAAPH%2F%2FoM4%3D","line":29,"updated":"2013-05-28 23:57:27.000000000","message":"minor: Consider naming this the more general term \"concurrency\" since \"thread\" implies actual threading, which this isn\u0027t doing.","commit_id":"8131622e58bc8172a00792acfab5f0ea5a2644d7"},{"author":{"_account_id":1216,"name":"gholt","email":"z-launchpad@brim.net","username":"gholt"},"change_message_id":"972c8974c1dfd3e8c6887b2059eec8fbaeea8629","unresolved":false,"context_lines":[{"line_number":36,"context_line":"# part is which of the parts a particular process will work on"},{"line_number":37,"context_line":"# part can also be specified on the command line and will overide the config"},{"line_number":38,"context_line":"#   value"},{"line_number":39,"context_line":"# part \u003d 0"},{"line_number":40,"context_line":""},{"line_number":41,"context_line":"[pipeline:main]"},{"line_number":42,"context_line":"pipeline \u003d catch_errors cache proxy-server"}],"source_content_type":"application/octet-stream","patch_set":7,"id":"AAAAPH%2F%2FoMc%3D","line":39,"updated":"2013-05-28 23:57:27.000000000","message":"minor: Consider adding a note that this is zero based. Three parts means setting this to 0, 1, or 2.","commit_id":"8131622e58bc8172a00792acfab5f0ea5a2644d7"},{"author":{"_account_id":4108,"name":"David Hadas","email":"davidh@il.ibm.com","username":"david-hadas"},"change_message_id":"7084c929a1211990a2cd2062a6955a5d32a2f4f2","unresolved":false,"context_lines":[{"line_number":35,"context_line":"# processes \u003d 0"},{"line_number":36,"context_line":"# process is which of the parts a particular process will work on"},{"line_number":37,"context_line":"# process can also be specified on the command line and will overide the config"},{"line_number":38,"context_line":"#   value"},{"line_number":39,"context_line":"# process is \"zero based\", if you want to use 3 processes, you should run"},{"line_number":40,"context_line":"#  processes with process set to 0, 1, and 2"},{"line_number":41,"context_line":"# process \u003d 0"}],"source_content_type":"application/octet-stream","patch_set":9,"id":"AAAAPX%2F%2F6ZU%3D","line":38,"updated":"2013-06-06 06:14:40.000000000","message":"No need to have this param for process id if we use worker threads (regular os threads)  instead of processes","commit_id":"bfd7aa535f611939173304646c4ac05e0656a1e1"}],"swift/obj/expirer.py":[{"author":{"_account_id":2696,"name":"Darrell Bishop","email":"dbishop@nvidia.com","username":"darrellb"},"change_message_id":"7249092394a66b0041ce08c8c6fce23b7d224a89","unresolved":false,"context_lines":[{"line_number":89,"context_line":"                       override the values from the config file if provided."},{"line_number":90,"context_line":"        \"\"\""},{"line_number":91,"context_line":"        parts, part \u003d self.get_parts(kwargs)"},{"line_number":92,"context_line":"        self.logger.info(\u0027parts: %d part: %d\u0027 % (parts, part))"},{"line_number":93,"context_line":"        if self.worker_count \u003e 0:"},{"line_number":94,"context_line":"            pool \u003d GreenPool(self.worker_count)"},{"line_number":95,"context_line":"        else:"}],"source_content_type":"text/x-python","patch_set":1,"id":"AAAAOn%2F%2FnG4%3D","line":92,"updated":"2013-05-01 22:28:17.000000000","message":"Looks like this new log output breaks some existing tests?","commit_id":"1e3b396b7f75821631225a45e8a70d95a596c289"},{"author":{"_account_id":4108,"name":"David Hadas","email":"davidh@il.ibm.com","username":"david-hadas"},"change_message_id":"d76866840391d45a5c7c458b571027fc18f3f643","unresolved":false,"context_lines":[{"line_number":89,"context_line":"                       override the values from the config file if provided."},{"line_number":90,"context_line":"        \"\"\""},{"line_number":91,"context_line":"        parts, part \u003d self.get_parts(kwargs)"},{"line_number":92,"context_line":"        if self.worker_count \u003e 0:"},{"line_number":93,"context_line":"            pool \u003d GreenPool(self.worker_count)"},{"line_number":94,"context_line":"        else:"},{"line_number":95,"context_line":"            pool \u003d None"}],"source_content_type":"text/x-python","patch_set":4,"id":"AAAAOn%2F%2FYrM%3D","line":92,"updated":"2013-05-11 21:58:49.000000000","message":"I assume that this should be if self.worker_count \u003e 1:","commit_id":"c10681dc6247f7c38f3c723406ffb4f9cb8111f7"},{"author":{"_account_id":4108,"name":"David Hadas","email":"davidh@il.ibm.com","username":"david-hadas"},"change_message_id":"d76866840391d45a5c7c458b571027fc18f3f643","unresolved":false,"context_lines":[{"line_number":91,"context_line":"        parts, part \u003d self.get_parts(kwargs)"},{"line_number":92,"context_line":"        if self.worker_count \u003e 0:"},{"line_number":93,"context_line":"            pool \u003d GreenPool(self.worker_count)"},{"line_number":94,"context_line":"        else:"},{"line_number":95,"context_line":"            pool \u003d None"},{"line_number":96,"context_line":"        containers_to_delete \u003d []"},{"line_number":97,"context_line":"        self.report_first_time \u003d self.report_last_time \u003d time()"}],"source_content_type":"text/x-python","patch_set":4,"id":"AAAAOn%2F%2FYrA%3D","line":94,"updated":"2013-05-11 21:58:49.000000000","message":"In fact, I would prefer it if you did not create two paths - one with the greenpool and one without it. This is just adding uneeded complexity. If we come to terms with the need to support more than one thread, we can support 1 using a pool of 1. \n\n(of course if we do that than my previous comment should be ignored)","commit_id":"c10681dc6247f7c38f3c723406ffb4f9cb8111f7"},{"author":{"_account_id":4108,"name":"David Hadas","email":"davidh@il.ibm.com","username":"david-hadas"},"change_message_id":"d76866840391d45a5c7c458b571027fc18f3f643","unresolved":false,"context_lines":[{"line_number":165,"context_line":"            if elapsed \u003c self.interval:"},{"line_number":166,"context_line":"                sleep(random() * (self.interval - elapsed))"},{"line_number":167,"context_line":""},{"line_number":168,"context_line":"    def get_parts(self, kwargs):"},{"line_number":169,"context_line":"        if kwargs.get(\u0027parts\u0027) is not None:"},{"line_number":170,"context_line":"            parts \u003d int(kwargs[\u0027parts\u0027])"},{"line_number":171,"context_line":"        else:"}],"source_content_type":"text/x-python","patch_set":4,"id":"AAAAOn%2F%2FYqo%3D","line":168,"updated":"2013-05-11 21:58:49.000000000","message":"Please use better naming than part and parts for code readability.","commit_id":"c10681dc6247f7c38f3c723406ffb4f9cb8111f7"},{"author":{"_account_id":1216,"name":"gholt","email":"z-launchpad@brim.net","username":"gholt"},"change_message_id":"972c8974c1dfd3e8c6887b2059eec8fbaeea8629","unresolved":false,"context_lines":[{"line_number":165,"context_line":"        \"\"\""},{"line_number":166,"context_line":"        Gets the parts, part from the kwargs if those values exist."},{"line_number":167,"context_line":""},{"line_number":168,"context_line":"        Otherise, return parts, part set in the config file."},{"line_number":169,"context_line":""},{"line_number":170,"context_line":"        :param kwargs: Keyword args passed into the run_forever(), run_once()"},{"line_number":171,"context_line":"                       methods.  They have values specified on the command"}],"source_content_type":"text/x-python","patch_set":7,"id":"AAAAPH%2F%2FoMM%3D","line":168,"updated":"2013-05-28 23:57:27.000000000","message":"minor: Otherise should be Otherwise.","commit_id":"8131622e58bc8172a00792acfab5f0ea5a2644d7"},{"author":{"_account_id":4108,"name":"David Hadas","email":"davidh@il.ibm.com","username":"david-hadas"},"change_message_id":"7084c929a1211990a2cd2062a6955a5d32a2f4f2","unresolved":false,"context_lines":[{"line_number":58,"context_line":"        self.concurrency \u003d int(conf.get(\u0027concurrency\u0027, 1024))"},{"line_number":59,"context_line":"        if self.concurrency \u003c 1:"},{"line_number":60,"context_line":"            raise ValueError(\"concurrency must be set to at least 1\")"},{"line_number":61,"context_line":"        self.processes \u003d int(self.conf.get(\u0027processes\u0027, 0))"},{"line_number":62,"context_line":"        self.process \u003d int(self.conf.get(\u0027process\u0027, 0))"},{"line_number":63,"context_line":""},{"line_number":64,"context_line":"    def report(self, final\u003dFalse):"}],"source_content_type":"text/x-python","patch_set":9,"id":"AAAAPX%2F%2F6ZQ%3D","line":61,"updated":"2013-06-06 06:14:40.000000000","message":"No need to have processid if we use worker threads (regular os threads)  instead of processes","commit_id":"bfd7aa535f611939173304646c4ac05e0656a1e1"},{"author":{"_account_id":4108,"name":"David Hadas","email":"davidh@il.ibm.com","username":"david-hadas"},"change_message_id":"7084c929a1211990a2cd2062a6955a5d32a2f4f2","unresolved":false,"context_lines":[{"line_number":191,"context_line":"                \u0027processes must be an intenger greater than or equal to 0\u0027)"},{"line_number":192,"context_line":""},{"line_number":193,"context_line":"        if processes and process \u003e\u003d processes:"},{"line_number":194,"context_line":"            raise ValueError("},{"line_number":195,"context_line":"                \u0027part must be less than or equal to parts\u0027)"},{"line_number":196,"context_line":""},{"line_number":197,"context_line":"        return processes, process"}],"source_content_type":"text/x-python","patch_set":9,"id":"AAAAPX%2F%2F6Zg%3D","line":194,"updated":"2013-06-06 06:14:40.000000000","message":"the remarks below still uses part and parts instead of process","commit_id":"bfd7aa535f611939173304646c4ac05e0656a1e1"},{"author":{"_account_id":917,"name":"Chuck Thier","email":"cthier@gmail.com","username":"cthier"},"change_message_id":"983d0f8fc8c42ec56ebaa7e4b1aa577c225888e9","unresolved":false,"context_lines":[{"line_number":55,"context_line":"        self.recon_cache_path \u003d conf.get(\u0027recon_cache_path\u0027,"},{"line_number":56,"context_line":"                                         \u0027/var/cache/swift\u0027)"},{"line_number":57,"context_line":"        self.rcache \u003d join(self.recon_cache_path, \u0027object.recon\u0027)"},{"line_number":58,"context_line":"        self.concurrency \u003d int(conf.get(\u0027concurrency\u0027, 1024))"},{"line_number":59,"context_line":"        if self.concurrency \u003c 1:"},{"line_number":60,"context_line":"            raise ValueError(\"concurrency must be set to at least 1\")"},{"line_number":61,"context_line":"        self.processes \u003d int(self.conf.get(\u0027processes\u0027, 0))"}],"source_content_type":"text/x-python","patch_set":10,"id":"AAAAPX%2F%2F104%3D","line":58,"updated":"2013-06-07 19:25:39.000000000","message":"We probably should set this to 1 by default so that it doesn\u0027t kill an SAIO if they don\u0027t have it set.","commit_id":"ef165a02e158ce494f3a7ec59f015498e91c38c4"},{"author":{"_account_id":917,"name":"Chuck Thier","email":"cthier@gmail.com","username":"cthier"},"change_message_id":"983d0f8fc8c42ec56ebaa7e4b1aa577c225888e9","unresolved":false,"context_lines":[{"line_number":57,"context_line":"        self.rcache \u003d join(self.recon_cache_path, \u0027object.recon\u0027)"},{"line_number":58,"context_line":"        self.concurrency \u003d int(conf.get(\u0027concurrency\u0027, 1024))"},{"line_number":59,"context_line":"        if self.concurrency \u003c 1:"},{"line_number":60,"context_line":"            raise ValueError(\"concurrency must be set to at least 1\")"},{"line_number":61,"context_line":"        self.processes \u003d int(self.conf.get(\u0027processes\u0027, 0))"},{"line_number":62,"context_line":"        self.process \u003d int(self.conf.get(\u0027process\u0027, 0))"},{"line_number":63,"context_line":""}],"source_content_type":"text/x-python","patch_set":10,"id":"AAAAPX%2F%2F11Y%3D","line":60,"updated":"2013-06-07 19:25:39.000000000","message":"you could just set concurrency to 1 in this case, but no biggie","commit_id":"ef165a02e158ce494f3a7ec59f015498e91c38c4"}]}