)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":10063,"name":"Steve McLellan","email":"steven.j.mclellan@gmail.com","username":"sjmc7"},"change_message_id":"d80708e6810f4ef418736688b70d15562907a325","unresolved":false,"context_lines":[{"line_number":4,"context_line":"Commit:     Rick Aulino \u003crick.aulino@hp.com\u003e"},{"line_number":5,"context_line":"CommitDate: 2016-07-06 21:03:35 -0600"},{"line_number":6,"context_line":""},{"line_number":7,"context_line":"To speed up Searchlight indexing and re-indexing we should use threads"},{"line_number":8,"context_line":"instead of inexing resource types sequentially."},{"line_number":9,"context_line":""},{"line_number":10,"context_line":"Change-Id: I2869758552ed6f7b4343f9b7123a35622edcab22"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":9,"id":"1aa78d24_47b5adce","line":7,"range":{"start_line":7,"start_character":12,"end_line":7,"end_character":23},"updated":"2016-07-08 15:59:27.000000000","message":"First line should be a ~50 character summary, longer description can follow in subsequent paragraphs. Long first lines mess up log listings.","commit_id":"c6682e207da102058dd7bf65e1c72a2532919193"}],"doc/source/indexingservice.rst":[{"author":{"_account_id":10063,"name":"Steve McLellan","email":"steven.j.mclellan@gmail.com","username":"sjmc7"},"change_message_id":"c386df55bd6537499a63331e3e2c8c2a85756b8c","unresolved":false,"context_lines":[{"line_number":199,"context_line":"thread name after the process ID to the logging message, the following"},{"line_number":200,"context_line":"settings can be added to the Searchlight configuration file::"},{"line_number":201,"context_line":""},{"line_number":202,"context_line":"    logging_context_format_string \u003d %(asctime)s.%(msecs)03d %(process)d %(thread)d %(threadName)s %(levelname)s %(name)s [%(request_id)s %(user_identity)s] %(instance)s%(message)s"},{"line_number":203,"context_line":""},{"line_number":204,"context_line":"    logging_default_format_string \u003d %(asctime)s.%(msecs)03d %(process)d %(thread)d %(threadName)s %(levelname)s %(name)s [-] %(instance)s%(message)s"},{"line_number":205,"context_line":""}],"source_content_type":"text/x-rst","patch_set":8,"id":"1aa78d24_93594e01","line":202,"range":{"start_line":202,"start_character":4,"end_line":202,"end_character":33},"updated":"2016-07-05 18:54:46.000000000","message":"Worth checking whether this format string\u0027s used for searchlight-manage. I believe it\u0027s only used in the API where there\u0027s a request context to use.","commit_id":"0fd53884d5150e7375f5ead2cfad871723ffa162"},{"author":{"_account_id":10063,"name":"Steve McLellan","email":"steven.j.mclellan@gmail.com","username":"sjmc7"},"change_message_id":"1dfb64f0518f263d1d14cf889a45059ad3de88c3","unresolved":false,"context_lines":[{"line_number":175,"context_line":"Multi-Thread Support"},{"line_number":176,"context_line":"--------------------"},{"line_number":177,"context_line":"The Newton Searchlight release introduced multiple thread support for"},{"line_number":178,"context_line":"indexing. Previously when the ``searchlight-manaage index sync``"},{"line_number":179,"context_line":"command was executed, all indexing occurred in a single thread. To boost"},{"line_number":180,"context_line":"the performance of the indexing functionality, each resource type"},{"line_number":181,"context_line":"will now index in its own thread. Multiple indexing threads will run"}],"source_content_type":"text/x-rst","patch_set":10,"id":"1aa78d24_2f578627","line":178,"range":{"start_line":178,"start_character":44,"end_line":178,"end_character":51},"updated":"2016-07-14 18:44:42.000000000","message":"typo - manage","commit_id":"e6e604dc62726733937053d2bdd926d441082d4d"},{"author":{"_account_id":14680,"name":"Rick Aulino","email":"rick.aulino@hpe.com","username":"ricka"},"change_message_id":"c8e03b3793f0cc4d983607e0982ebbcc47458636","unresolved":false,"context_lines":[{"line_number":175,"context_line":"Multi-Thread Support"},{"line_number":176,"context_line":"--------------------"},{"line_number":177,"context_line":"The Newton Searchlight release introduced multiple thread support for"},{"line_number":178,"context_line":"indexing. Previously when the ``searchlight-manaage index sync``"},{"line_number":179,"context_line":"command was executed, all indexing occurred in a single thread. To boost"},{"line_number":180,"context_line":"the performance of the indexing functionality, each resource type"},{"line_number":181,"context_line":"will now index in its own thread. Multiple indexing threads will run"}],"source_content_type":"text/x-rst","patch_set":10,"id":"1aa78d24_72532e6d","line":178,"range":{"start_line":178,"start_character":44,"end_line":178,"end_character":51},"in_reply_to":"1aa78d24_2f578627","updated":"2016-07-14 21:06:48.000000000","message":"untypo\u0027ed.","commit_id":"e6e604dc62726733937053d2bdd926d441082d4d"}],"releasenotes/notes/bp-index-performance-enhancement-c8dc21da98f45628.yaml":[{"author":{"_account_id":10063,"name":"Steve McLellan","email":"steven.j.mclellan@gmail.com","username":"sjmc7"},"change_message_id":"af40021192848ee162a0d3b2f77811c82980c62c","unresolved":false,"context_lines":[{"line_number":6,"context_line":"    a separate thread. Multiple threads will be run concurrently."},{"line_number":7,"context_line":"other:"},{"line_number":8,"context_line":"  - The maximum number of threads that will run concurrently can be controlled"},{"line_number":9,"context_line":"    by setting \"max_workers\" in the Searchlight configuration file: "},{"line_number":10,"context_line":""},{"line_number":11,"context_line":"    [manage]"},{"line_number":12,"context_line":"    max_workers \u003d 6"}],"source_content_type":"text/x-yaml","patch_set":5,"id":"3aaa91ec_cc60fd03","line":9,"range":{"start_line":9,"start_character":66,"end_line":9,"end_character":67},"updated":"2016-06-30 19:21:54.000000000","message":"The colon makes this invalid yaml. You can either quote the whole thing or do something like\n\nother:\n  - |\n     The blah blah:\n\n     [manage]\n     max_workers \u003d 6","commit_id":"758966e2cb4b618a642dff07e9026fb5caa906d7"},{"author":{"_account_id":14680,"name":"Rick Aulino","email":"rick.aulino@hpe.com","username":"ricka"},"change_message_id":"ddc899fd5ebcbd1be5cab695443ddbf6bef4705d","unresolved":false,"context_lines":[{"line_number":6,"context_line":"    a separate thread. Multiple threads will be run concurrently."},{"line_number":7,"context_line":"other:"},{"line_number":8,"context_line":"  - The maximum number of threads that will run concurrently can be controlled"},{"line_number":9,"context_line":"    by setting \"max_workers\" in the Searchlight configuration file: "},{"line_number":10,"context_line":""},{"line_number":11,"context_line":"    [manage]"},{"line_number":12,"context_line":"    max_workers \u003d 6"}],"source_content_type":"text/x-yaml","patch_set":5,"id":"3aaa91ec_87651ce2","line":9,"range":{"start_line":9,"start_character":66,"end_line":9,"end_character":67},"in_reply_to":"3aaa91ec_cc60fd03","updated":"2016-07-01 02:53:26.000000000","message":"Time for a subtotal colectomy.","commit_id":"758966e2cb4b618a642dff07e9026fb5caa906d7"}],"searchlight/cmd/manage.py":[{"author":{"_account_id":10063,"name":"Steve McLellan","email":"steven.j.mclellan@gmail.com","username":"sjmc7"},"change_message_id":"99eafa3a67a1f407098dd55181e7b6004bf773f1","unresolved":false,"context_lines":[{"line_number":70,"context_line":"            raise"},{"line_number":71,"context_line":""},{"line_number":72,"context_line":""},{"line_number":73,"context_line":"class EsReindexThread(threading.Thread):"},{"line_number":74,"context_line":"    \"\"\"Thread class to re-index using the ES reindex helper."},{"line_number":75,"context_line":"       This will allow all ES re-indexes to occur simultaneously."},{"line_number":76,"context_line":"    \"\"\""}],"source_content_type":"text/x-python","patch_set":1,"id":"3aaa91ec_a42bc893","line":73,"range":{"start_line":73,"start_character":6,"end_line":73,"end_character":21},"updated":"2016-06-27 16:34:57.000000000","message":"Still not convinced this will be worth the complexity. We\u0027re not sending any data to e-s; it\u0027s a bulk operation that will happen as fast as elasticsearch can do it and which will be I/O bound. Sending more than one at once seems unlikely to speed things up.","commit_id":"442af81a2f914e0d0161812114ba57d76c510118"},{"author":{"_account_id":10063,"name":"Steve McLellan","email":"steven.j.mclellan@gmail.com","username":"sjmc7"},"change_message_id":"af40021192848ee162a0d3b2f77811c82980c62c","unresolved":false,"context_lines":[{"line_number":49,"context_line":"    def __init__(self):"},{"line_number":50,"context_line":"        utils.register_plugin_opts()"},{"line_number":51,"context_line":""},{"line_number":52,"context_line":"    def pluginApiThread(self, plugin_obj, index_name):"},{"line_number":53,"context_line":"        \"\"\"Helper to re-index using the plugin API within a thread,"},{"line_number":54,"context_line":"           allowing all plugins to re-index simultaneously."},{"line_number":55,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":5,"id":"3aaa91ec_cfe90fe8","line":52,"range":{"start_line":52,"start_character":8,"end_line":52,"end_character":23},"updated":"2016-06-30 19:21:54.000000000","message":"I would name this \u0027index_plugin_data\u0027 or something; the fact it\u0027s going to be used as a threading target is not relevant to the function. Also use python function naming convention.","commit_id":"758966e2cb4b618a642dff07e9026fb5caa906d7"},{"author":{"_account_id":10063,"name":"Steve McLellan","email":"steven.j.mclellan@gmail.com","username":"sjmc7"},"change_message_id":"af40021192848ee162a0d3b2f77811c82980c62c","unresolved":false,"context_lines":[{"line_number":55,"context_line":"        \"\"\""},{"line_number":56,"context_line":"        try:"},{"line_number":57,"context_line":"            plugin_obj.initial_indexing(index_name\u003dindex_name)"},{"line_number":58,"context_line":"            es_utils.refresh_index(index_name)"},{"line_number":59,"context_line":"        except exceptions.EndpointNotFound:"},{"line_number":60,"context_line":"            # Display a warning, do not propogate."},{"line_number":61,"context_line":"            LOG.warning(_LW(\"Service is not available for plugin: \""}],"source_content_type":"text/x-python","patch_set":5,"id":"3aaa91ec_0fbdd701","line":58,"range":{"start_line":58,"start_character":21,"end_line":58,"end_character":34},"updated":"2016-06-30 19:21:54.000000000","message":"This is going to cause multiple refreshes as other plugins are indexing, which isn\u0027t necessarily a problem but something to be aware of.","commit_id":"758966e2cb4b618a642dff07e9026fb5caa906d7"},{"author":{"_account_id":10063,"name":"Steve McLellan","email":"steven.j.mclellan@gmail.com","username":"sjmc7"},"change_message_id":"af40021192848ee162a0d3b2f77811c82980c62c","unresolved":false,"context_lines":[{"line_number":65,"context_line":"                          \"%(ex)s: %(e)s\") % {\u0027ex\u0027: index_name, \u0027e\u0027: e})"},{"line_number":66,"context_line":"            raise"},{"line_number":67,"context_line":""},{"line_number":68,"context_line":"    def esReindexThread(self, es_reindex, resource_groups, index_names):"},{"line_number":69,"context_line":"        \"\"\"Helper to re-index using the ES reindex helper,"},{"line_number":70,"context_line":"           allowing all ES re-indexes to occur simultaneously."},{"line_number":71,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":5,"id":"3aaa91ec_efea0bd7","line":68,"range":{"start_line":68,"start_character":8,"end_line":68,"end_character":23},"updated":"2016-06-30 19:21:54.000000000","message":"Python naming convention please","commit_id":"758966e2cb4b618a642dff07e9026fb5caa906d7"},{"author":{"_account_id":10063,"name":"Steve McLellan","email":"steven.j.mclellan@gmail.com","username":"sjmc7"},"change_message_id":"af40021192848ee162a0d3b2f77811c82980c62c","unresolved":false,"context_lines":[{"line_number":77,"context_line":"                [a for a in resource_groups if a[0] \u003d\u003d group][0][1]"},{"line_number":78,"context_line":"            dst_index \u003d index_names[group]"},{"line_number":79,"context_line":"        try:"},{"line_number":80,"context_line":"            es_utils.reindex(src_index\u003dalias_search,"},{"line_number":81,"context_line":"                             dst_index\u003ddst_index,"},{"line_number":82,"context_line":"                             type_list\u003des_reindex)"},{"line_number":83,"context_line":"            es_utils.refresh_index(dst_index)"}],"source_content_type":"text/x-python","patch_set":5,"id":"3aaa91ec_4fd23f5d","line":80,"range":{"start_line":80,"start_character":20,"end_line":80,"end_character":21},"updated":"2016-06-30 19:21:54.000000000","message":"This looks like a bug - alias_search and dst_index will be assigned to multiple times (as many times as there are index_names) but only the last value will be used. It looks like the whole function should be inside the for:","commit_id":"758966e2cb4b618a642dff07e9026fb5caa906d7"},{"author":{"_account_id":10063,"name":"Steve McLellan","email":"steven.j.mclellan@gmail.com","username":"sjmc7"},"change_message_id":"af40021192848ee162a0d3b2f77811c82980c62c","unresolved":false,"context_lines":[{"line_number":273,"context_line":"                    es_reindex.append(doc_type)"},{"line_number":274,"context_line":"                    plugins_to_index.remove((resource_type, ext))"},{"line_number":275,"context_line":""},{"line_number":276,"context_line":"        max_workers \u003d cfg.CONF.manage.max_workers"},{"line_number":277,"context_line":"        ThreadPoolExec \u003d concurrent.futures.ThreadPoolExecutor"},{"line_number":278,"context_line":"        with ThreadPoolExec(max_workers\u003dmax_workers) as executor:"},{"line_number":279,"context_line":"            try:"}],"source_content_type":"text/x-python","patch_set":5,"id":"3aaa91ec_6fac5b9f","line":276,"range":{"start_line":276,"start_character":8,"end_line":276,"end_character":19},"updated":"2016-06-30 19:21:54.000000000","message":"Consider ripping this chunk out and putting it into its own function; the current function is getting ever-increasingly large","commit_id":"758966e2cb4b618a642dff07e9026fb5caa906d7"},{"author":{"_account_id":10063,"name":"Steve McLellan","email":"steven.j.mclellan@gmail.com","username":"sjmc7"},"change_message_id":"af40021192848ee162a0d3b2f77811c82980c62c","unresolved":false,"context_lines":[{"line_number":293,"context_line":"                                   es_reindex, resource_groups, index_names))"},{"line_number":294,"context_line":""},{"line_number":295,"context_line":"                # Sit back, relax and wait for the threads to complete."},{"line_number":296,"context_line":"                for future in concurrent.futures.as_completed(futures):"},{"line_number":297,"context_line":"                    try:"},{"line_number":298,"context_line":"                        future.result()"},{"line_number":299,"context_line":"                    except Exception as exc:"}],"source_content_type":"text/x-python","patch_set":5,"id":"3aaa91ec_9b10e54f","line":296,"range":{"start_line":296,"start_character":62,"end_line":296,"end_character":69},"updated":"2016-06-30 19:21:54.000000000","message":"If you add a timeout parameter you can catch the TimeoutErrors which will among other things allow keyboard interrupts to be handled more cleanly (catch KB interrupt, clean up or terminate threads)","commit_id":"758966e2cb4b618a642dff07e9026fb5caa906d7"},{"author":{"_account_id":10063,"name":"Steve McLellan","email":"steven.j.mclellan@gmail.com","username":"sjmc7"},"change_message_id":"af40021192848ee162a0d3b2f77811c82980c62c","unresolved":false,"context_lines":[{"line_number":299,"context_line":"                    except Exception as exc:"},{"line_number":300,"context_line":"                        # A plugin failed. Try to cancel the rest of the"},{"line_number":301,"context_line":"                        # remaining threads. Raise the exception to clean up."},{"line_number":302,"context_line":"                        for f in concurrent.futures.as_completed(futures):"},{"line_number":303,"context_line":"                            f.cancel()"},{"line_number":304,"context_line":"                        raise exc"},{"line_number":305,"context_line":"            except Exception as e:"}],"source_content_type":"text/x-python","patch_set":5,"id":"3aaa91ec_0080048a","line":302,"range":{"start_line":302,"start_character":33,"end_line":302,"end_character":43},"updated":"2016-06-30 19:21:54.000000000","message":"This looks a bit fishy - doesn\u0027t as_completed only yield when a job\u0027s finished? Why not just loop over futures and cancel them?","commit_id":"758966e2cb4b618a642dff07e9026fb5caa906d7"},{"author":{"_account_id":10063,"name":"Steve McLellan","email":"steven.j.mclellan@gmail.com","username":"sjmc7"},"change_message_id":"af40021192848ee162a0d3b2f77811c82980c62c","unresolved":false,"context_lines":[{"line_number":391,"context_line":"            result.append((i, getattr(obj, i)))"},{"line_number":392,"context_line":"    return result"},{"line_number":393,"context_line":""},{"line_number":394,"context_line":"manage_opts \u003d ["},{"line_number":395,"context_line":"    cfg.IntOpt(\u0027max_workers\u0027, default\u003d3,"},{"line_number":396,"context_line":"               help\u003d\"Maximum number of worker threads for indexing.\")"},{"line_number":397,"context_line":"]"}],"source_content_type":"text/x-python","patch_set":5,"id":"3aaa91ec_0c05e512","line":394,"range":{"start_line":394,"start_character":0,"end_line":394,"end_character":11},"updated":"2016-06-30 19:21:54.000000000","message":"It\u0027s traditional to put config options at the top of file for whatever reason.","commit_id":"758966e2cb4b618a642dff07e9026fb5caa906d7"},{"author":{"_account_id":10063,"name":"Steve McLellan","email":"steven.j.mclellan@gmail.com","username":"sjmc7"},"change_message_id":"af40021192848ee162a0d3b2f77811c82980c62c","unresolved":false,"context_lines":[{"line_number":392,"context_line":"    return result"},{"line_number":393,"context_line":""},{"line_number":394,"context_line":"manage_opts \u003d ["},{"line_number":395,"context_line":"    cfg.IntOpt(\u0027max_workers\u0027, default\u003d3,"},{"line_number":396,"context_line":"               help\u003d\"Maximum number of worker threads for indexing.\")"},{"line_number":397,"context_line":"]"},{"line_number":398,"context_line":""}],"source_content_type":"text/x-python","patch_set":5,"id":"3aaa91ec_15796493","line":395,"range":{"start_line":395,"start_character":16,"end_line":395,"end_character":27},"updated":"2016-06-30 19:21:54.000000000","message":"Perhaps just \u0027workers\u0027 so it\u0027s more consistent with other section settings.\n\nThis either needs to specify a minimum number of 0 or handle the case of 0 workers more gracefully (right now it hangs; be better if it ran in the main thread).","commit_id":"758966e2cb4b618a642dff07e9026fb5caa906d7"},{"author":{"_account_id":10063,"name":"Steve McLellan","email":"steven.j.mclellan@gmail.com","username":"sjmc7"},"change_message_id":"c386df55bd6537499a63331e3e2c8c2a85756b8c","unresolved":false,"context_lines":[{"line_number":57,"context_line":"class IndexCommands(object):"},{"line_number":58,"context_line":"    def __init__(self):"},{"line_number":59,"context_line":"        utils.register_plugin_opts()"},{"line_number":60,"context_line":"        self.thread_stop \u003d False"},{"line_number":61,"context_line":"        self.index_names \u003d {}"},{"line_number":62,"context_line":""},{"line_number":63,"context_line":"    def plugin_api(self, plugin_obj, index_name):"}],"source_content_type":"text/x-python","patch_set":8,"id":"1aa78d24_1379def6","line":60,"range":{"start_line":60,"start_character":13,"end_line":60,"end_character":24},"updated":"2016-07-05 18:54:46.000000000","message":"See later comment. If this state must be recorded it\u0027d be better to create a single object containing anything required for controlling the various threads (the stop flag, index names to clean up, the threads themselves) to make it clear they\u0027re all related and why they\u0027re member status of this class.","commit_id":"0fd53884d5150e7375f5ead2cfad871723ffa162"},{"author":{"_account_id":10063,"name":"Steve McLellan","email":"steven.j.mclellan@gmail.com","username":"sjmc7"},"change_message_id":"c386df55bd6537499a63331e3e2c8c2a85756b8c","unresolved":false,"context_lines":[{"line_number":64,"context_line":"        \"\"\"Helper to re-index using the plugin API within a thread,"},{"line_number":65,"context_line":"           allowing all plugins to re-index simultaneously."},{"line_number":66,"context_line":"        \"\"\""},{"line_number":67,"context_line":"        if self.thread_stop:"},{"line_number":68,"context_line":"            # The main thread wants all threads to stop."},{"line_number":69,"context_line":"            return"},{"line_number":70,"context_line":"        try:"}],"source_content_type":"text/x-python","patch_set":8,"id":"1aa78d24_ae6ea193","line":67,"range":{"start_line":67,"start_character":16,"end_line":67,"end_character":27},"updated":"2016-07-05 18:54:46.000000000","message":"This shouldn\u0027t be required; see docs for future.cancel.","commit_id":"0fd53884d5150e7375f5ead2cfad871723ffa162"},{"author":{"_account_id":10063,"name":"Steve McLellan","email":"steven.j.mclellan@gmail.com","username":"sjmc7"},"change_message_id":"c386df55bd6537499a63331e3e2c8c2a85756b8c","unresolved":false,"context_lines":[{"line_number":68,"context_line":"            # The main thread wants all threads to stop."},{"line_number":69,"context_line":"            return"},{"line_number":70,"context_line":"        try:"},{"line_number":71,"context_line":"            plugin_obj.initial_indexing(index_name\u003dindex_name)"},{"line_number":72,"context_line":"            es_utils.refresh_index(index_name)"},{"line_number":73,"context_line":"        except exceptions.EndpointNotFound:"},{"line_number":74,"context_line":"            # Display a warning, do not propogate."}],"source_content_type":"text/x-python","patch_set":8,"id":"1aa78d24_8e9ac561","line":71,"range":{"start_line":71,"start_character":23,"end_line":71,"end_character":39},"updated":"2016-07-05 18:54:46.000000000","message":"Since we know it\u0027s a possibility the index will disappear underneath this function, it seems like it\u0027d be useful to catch the error and if it looks like it was the result of being terminated by the user, display a message to that effect and ignore it.","commit_id":"0fd53884d5150e7375f5ead2cfad871723ffa162"},{"author":{"_account_id":10063,"name":"Steve McLellan","email":"steven.j.mclellan@gmail.com","username":"sjmc7"},"change_message_id":"c386df55bd6537499a63331e3e2c8c2a85756b8c","unresolved":false,"context_lines":[{"line_number":101,"context_line":"                raise"},{"line_number":102,"context_line":""},{"line_number":103,"context_line":"    # Signal handler to catch interrupts from the user (ctl-c)"},{"line_number":104,"context_line":"    def sig_handler(self, signum, frame):"},{"line_number":105,"context_line":"        \"\"\"When rudely interrupted by the user, we will want to clean up after"},{"line_number":106,"context_line":"           ourselves. We have potentially two pieces of unfinished business."},{"line_number":107,"context_line":"               1. We have running threads. Politely remind them to stop."}],"source_content_type":"text/x-python","patch_set":8,"id":"1aa78d24_739dc258","line":104,"range":{"start_line":104,"start_character":8,"end_line":104,"end_character":19},"updated":"2016-07-05 18:54:46.000000000","message":"No reason to make this a member function and have to remember additional state in \u0027self\u0027; this can be an anonymous function declared in \u0027sync\u0027, and then it\u0027ll have access to all the local state.","commit_id":"0fd53884d5150e7375f5ead2cfad871723ffa162"},{"author":{"_account_id":10063,"name":"Steve McLellan","email":"steven.j.mclellan@gmail.com","username":"sjmc7"},"change_message_id":"c386df55bd6537499a63331e3e2c8c2a85756b8c","unresolved":false,"context_lines":[{"line_number":106,"context_line":"           ourselves. We have potentially two pieces of unfinished business."},{"line_number":107,"context_line":"               1. We have running threads. Politely remind them to stop."},{"line_number":108,"context_line":"               2. We have started new indices in Elasticsearch. Remove them."},{"line_number":109,"context_line":"           One thing to note with regard to Python threads. When we hit this"},{"line_number":110,"context_line":"           handler, any currently runing threads will continue to run. We will"},{"line_number":111,"context_line":"           prevent only future threads from starting. Even though we are"},{"line_number":112,"context_line":"           deleting the newly created indices here, the existing threads will"}],"source_content_type":"text/x-python","patch_set":8,"id":"1aa78d24_d38ab6be","line":109,"range":{"start_line":109,"start_character":29,"end_line":109,"end_character":33},"updated":"2016-07-05 18:54:46.000000000","message":"Then stop the threads or wait for them to finish.","commit_id":"0fd53884d5150e7375f5ead2cfad871723ffa162"},{"author":{"_account_id":10063,"name":"Steve McLellan","email":"steven.j.mclellan@gmail.com","username":"sjmc7"},"change_message_id":"c386df55bd6537499a63331e3e2c8c2a85756b8c","unresolved":false,"context_lines":[{"line_number":115,"context_line":"           errors."},{"line_number":116,"context_line":"        \"\"\""},{"line_number":117,"context_line":"        # Inform the threads to stop."},{"line_number":118,"context_line":"        self.thread_stop \u003d True"},{"line_number":119,"context_line":""},{"line_number":120,"context_line":"        # Remove any newly created Elasticsearch indices."},{"line_number":121,"context_line":"        if self.index_names:"}],"source_content_type":"text/x-python","patch_set":8,"id":"1aa78d24_6e5cb9ed","line":118,"range":{"start_line":118,"start_character":13,"end_line":118,"end_character":24},"updated":"2016-07-05 18:54:46.000000000","message":"Looking at the docs, future.cancel() will do the same as this but more neatly.","commit_id":"0fd53884d5150e7375f5ead2cfad871723ffa162"},{"author":{"_account_id":10063,"name":"Steve McLellan","email":"steven.j.mclellan@gmail.com","username":"sjmc7"},"change_message_id":"c386df55bd6537499a63331e3e2c8c2a85756b8c","unresolved":false,"context_lines":[{"line_number":342,"context_line":""},{"line_number":343,"context_line":"                # Sit back, relax and wait for the threads to complete."},{"line_number":344,"context_line":"                threads_running \u003d True"},{"line_number":345,"context_line":"                while threads_running:"},{"line_number":346,"context_line":"                    # Are any threads still running?"},{"line_number":347,"context_line":"                    threads_running \u003d False"},{"line_number":348,"context_line":"                    for future in futures:"}],"source_content_type":"text/x-python","patch_set":8,"id":"1aa78d24_0e4d356c","line":345,"range":{"start_line":345,"start_character":22,"end_line":345,"end_character":37},"updated":"2016-07-05 18:54:46.000000000","message":"Looking at the documentation for concurrent.futures (https://docs.python.org/3/library/concurrent.futures.html) it looks like waiting is built-in; the context-manager version of TPE blocks on shutdown(), similar to a join(). Obviously be worth testing but might simplify the code.","commit_id":"0fd53884d5150e7375f5ead2cfad871723ffa162"},{"author":{"_account_id":10063,"name":"Steve McLellan","email":"steven.j.mclellan@gmail.com","username":"sjmc7"},"change_message_id":"d80708e6810f4ef418736688b70d15562907a325","unresolved":false,"context_lines":[{"line_number":79,"context_line":"                          \"%(ex)s: %(e)s\") % {\u0027ex\u0027: index_name, \u0027e\u0027: e})"},{"line_number":80,"context_line":"            raise"},{"line_number":81,"context_line":""},{"line_number":82,"context_line":"        if self.cleanup:"},{"line_number":83,"context_line":"            es_utils.alias_error_cleanup(index_names)"},{"line_number":84,"context_line":""},{"line_number":85,"context_line":"    def es_reindex_worker(self, es_reindex, resource_groups, index_names):"}],"source_content_type":"text/x-python","patch_set":9,"id":"1aa78d24_c2250b0b","line":82,"range":{"start_line":82,"start_character":16,"end_line":82,"end_character":23},"updated":"2016-07-08 15:59:27.000000000","message":"Does each thread need to do this? If all doc types are using the same index there\u0027ll potentially be 10 threads all doing the same alias assignment operations. It makes more sense for cleanup to happen outside the threaded code.","commit_id":"c6682e207da102058dd7bf65e1c72a2532919193"},{"author":{"_account_id":10063,"name":"Steve McLellan","email":"steven.j.mclellan@gmail.com","username":"sjmc7"},"change_message_id":"d80708e6810f4ef418736688b70d15562907a325","unresolved":false,"context_lines":[{"line_number":105,"context_line":"                raise"},{"line_number":106,"context_line":""},{"line_number":107,"context_line":"        if self.cleanup:"},{"line_number":108,"context_line":"            es_utils.alias_error_cleanup(index_names)"},{"line_number":109,"context_line":""},{"line_number":110,"context_line":"    @args(\u0027--group\u0027, metavar\u003d\u0027\u003cgroup\u003e\u0027, dest\u003d\u0027group\u0027,"},{"line_number":111,"context_line":"          help\u003d\u0027Index only this Resource Group (or a comma separated list)\u0027)"}],"source_content_type":"text/x-python","patch_set":9,"id":"1aa78d24_4278fbf8","line":108,"range":{"start_line":108,"start_character":20,"end_line":108,"end_character":21},"updated":"2016-07-08 15:59:27.000000000","message":"Same comment as above.","commit_id":"c6682e207da102058dd7bf65e1c72a2532919193"},{"author":{"_account_id":10063,"name":"Steve McLellan","email":"steven.j.mclellan@gmail.com","username":"sjmc7"},"change_message_id":"d80708e6810f4ef418736688b70d15562907a325","unresolved":false,"context_lines":[{"line_number":355,"context_line":""},{"line_number":356,"context_line":"                # Sit back, relax and wait for the threads to complete."},{"line_number":357,"context_line":"                threads_running \u003d True"},{"line_number":358,"context_line":"                while threads_running:"},{"line_number":359,"context_line":"                    # Are any threads still running?"},{"line_number":360,"context_line":"                    threads_running \u003d False"},{"line_number":361,"context_line":"                    for future in futures:"}],"source_content_type":"text/x-python","patch_set":9,"id":"1aa78d24_c2afcb29","line":358,"range":{"start_line":358,"start_character":22,"end_line":358,"end_character":37},"updated":"2016-07-08 15:59:27.000000000","message":"This still seems a bit long winded as mentioned in the previous patchset. The threadpoolexec context manager documentation suggests that it won\u0027t exit until all futures are finished or cancelled.","commit_id":"c6682e207da102058dd7bf65e1c72a2532919193"},{"author":{"_account_id":10063,"name":"Steve McLellan","email":"steven.j.mclellan@gmail.com","username":"sjmc7"},"change_message_id":"d80708e6810f4ef418736688b70d15562907a325","unresolved":false,"context_lines":[{"line_number":366,"context_line":"            except Exception as e:"},{"line_number":367,"context_line":"                # An exception occurred. Inform the remaining threads to stop."},{"line_number":368,"context_line":"                # Start cleaning up ElasticSearch and inform the user."},{"line_number":369,"context_line":"                es_utils.alias_error_cleanup(index_names)"},{"line_number":370,"context_line":"                raise"},{"line_number":371,"context_line":""},{"line_number":372,"context_line":"        # Step #5: Update the \"search\" alias."}],"source_content_type":"text/x-python","patch_set":9,"id":"1aa78d24_c276abd7","line":369,"range":{"start_line":369,"start_character":25,"end_line":369,"end_character":44},"updated":"2016-07-08 15:59:27.000000000","message":"This same function is called in self.plugin_api as well. More may not be better.","commit_id":"c6682e207da102058dd7bf65e1c72a2532919193"},{"author":{"_account_id":10063,"name":"Steve McLellan","email":"steven.j.mclellan@gmail.com","username":"sjmc7"},"change_message_id":"1dfb64f0518f263d1d14cf889a45059ad3de88c3","unresolved":false,"context_lines":[{"line_number":133,"context_line":"            for future in futures:"},{"line_number":134,"context_line":"                future.cancel()"},{"line_number":135,"context_line":""},{"line_number":136,"context_line":"            # Politely wait for the current threads to finish."},{"line_number":137,"context_line":"            wait_for_threads()"},{"line_number":138,"context_line":""},{"line_number":139,"context_line":"            # Rudely remove any newly created Elasticsearch indices."}],"source_content_type":"text/x-python","patch_set":10,"id":"1aa78d24_af425666","line":136,"range":{"start_line":136,"start_character":14,"end_line":136,"end_character":22},"updated":"2016-07-14 18:44:42.000000000","message":"Can you add a log message (at warning level, i think) indicating that there\u0027s been an interrupt but it\u0027s waiting for indexing threads so that it can tidy up properly?","commit_id":"e6e604dc62726733937053d2bdd926d441082d4d"},{"author":{"_account_id":14680,"name":"Rick Aulino","email":"rick.aulino@hpe.com","username":"ricka"},"change_message_id":"c8e03b3793f0cc4d983607e0982ebbcc47458636","unresolved":false,"context_lines":[{"line_number":133,"context_line":"            for future in futures:"},{"line_number":134,"context_line":"                future.cancel()"},{"line_number":135,"context_line":""},{"line_number":136,"context_line":"            # Politely wait for the current threads to finish."},{"line_number":137,"context_line":"            wait_for_threads()"},{"line_number":138,"context_line":""},{"line_number":139,"context_line":"            # Rudely remove any newly created Elasticsearch indices."}],"source_content_type":"text/x-python","patch_set":10,"id":"1aa78d24_524ef2c2","line":136,"range":{"start_line":136,"start_character":14,"end_line":136,"end_character":22},"in_reply_to":"1aa78d24_af425666","updated":"2016-07-14 21:06:48.000000000","message":"Added.","commit_id":"e6e604dc62726733937053d2bdd926d441082d4d"},{"author":{"_account_id":10063,"name":"Steve McLellan","email":"steven.j.mclellan@gmail.com","username":"sjmc7"},"change_message_id":"1dfb64f0518f263d1d14cf889a45059ad3de88c3","unresolved":false,"context_lines":[{"line_number":351,"context_line":"                # Sit back, relax and wait for the threads to complete."},{"line_number":352,"context_line":"                wait_for_threads()"},{"line_number":353,"context_line":"            except Exception as e:"},{"line_number":354,"context_line":"                # An exception occurred. Inform the remaining threads to stop."},{"line_number":355,"context_line":"                # Start cleaning up ElasticSearch and inform the user."},{"line_number":356,"context_line":"                es_utils.alias_error_cleanup(index_names)"},{"line_number":357,"context_line":"                raise"}],"source_content_type":"text/x-python","patch_set":10,"id":"1aa78d24_0f73aa16","line":354,"range":{"start_line":354,"start_character":52,"end_line":354,"end_character":61},"updated":"2016-07-14 18:44:42.000000000","message":"\"Inform the remaining threads to stop\" - this seems to have gone missing. Add call to wait_for_threads() ?","commit_id":"e6e604dc62726733937053d2bdd926d441082d4d"},{"author":{"_account_id":14680,"name":"Rick Aulino","email":"rick.aulino@hpe.com","username":"ricka"},"change_message_id":"c8e03b3793f0cc4d983607e0982ebbcc47458636","unresolved":false,"context_lines":[{"line_number":351,"context_line":"                # Sit back, relax and wait for the threads to complete."},{"line_number":352,"context_line":"                wait_for_threads()"},{"line_number":353,"context_line":"            except Exception as e:"},{"line_number":354,"context_line":"                # An exception occurred. Inform the remaining threads to stop."},{"line_number":355,"context_line":"                # Start cleaning up ElasticSearch and inform the user."},{"line_number":356,"context_line":"                es_utils.alias_error_cleanup(index_names)"},{"line_number":357,"context_line":"                raise"}],"source_content_type":"text/x-python","patch_set":10,"id":"1aa78d24_32e7e6ad","line":354,"range":{"start_line":354,"start_character":52,"end_line":354,"end_character":61},"in_reply_to":"1aa78d24_0f73aa16","updated":"2016-07-14 21:06:48.000000000","message":"Leftover from the pre-thread days. Removed.","commit_id":"e6e604dc62726733937053d2bdd926d441082d4d"},{"author":{"_account_id":10063,"name":"Steve McLellan","email":"steven.j.mclellan@gmail.com","username":"sjmc7"},"change_message_id":"6ac9afad6c71afaf053ddd2d390ec901fc91d4ae","unresolved":false,"context_lines":[{"line_number":58,"context_line":"    def __init__(self):"},{"line_number":59,"context_line":"        utils.register_plugin_opts()"},{"line_number":60,"context_line":""},{"line_number":61,"context_line":"    def plugin_api(self, plugin_obj, index_names):"},{"line_number":62,"context_line":"        \"\"\"Helper to re-index using the plugin API within a thread, allowing"},{"line_number":63,"context_line":"           all plugins to re-index simultaneously. We may need to cleanup."},{"line_number":64,"context_line":"           See sig_handler() for more info."}],"source_content_type":"text/x-python","patch_set":11,"id":"fad55976_00872f10","line":61,"range":{"start_line":61,"start_character":8,"end_line":61,"end_character":18},"updated":"2016-07-15 18:31:04.000000000","message":"This needs to start with an underscore or it gets interpreted as a publically-available command - run searchlight-manage index --help to see what i mean","commit_id":"77f0ccfb4d2f5500706d8ab01c5721c705e42378"},{"author":{"_account_id":14680,"name":"Rick Aulino","email":"rick.aulino@hpe.com","username":"ricka"},"change_message_id":"94a7d98bddb76a7d9c12ce5d20e280d61386e469","unresolved":false,"context_lines":[{"line_number":58,"context_line":"    def __init__(self):"},{"line_number":59,"context_line":"        utils.register_plugin_opts()"},{"line_number":60,"context_line":""},{"line_number":61,"context_line":"    def plugin_api(self, plugin_obj, index_names):"},{"line_number":62,"context_line":"        \"\"\"Helper to re-index using the plugin API within a thread, allowing"},{"line_number":63,"context_line":"           all plugins to re-index simultaneously. We may need to cleanup."},{"line_number":64,"context_line":"           See sig_handler() for more info."}],"source_content_type":"text/x-python","patch_set":11,"id":"fad55976_d867cf23","line":61,"range":{"start_line":61,"start_character":8,"end_line":61,"end_character":18},"in_reply_to":"fad55976_00872f10","updated":"2016-07-15 20:24:31.000000000","message":"Nice catch. It has been underscore-ized.","commit_id":"77f0ccfb4d2f5500706d8ab01c5721c705e42378"},{"author":{"_account_id":10063,"name":"Steve McLellan","email":"steven.j.mclellan@gmail.com","username":"sjmc7"},"change_message_id":"6ac9afad6c71afaf053ddd2d390ec901fc91d4ae","unresolved":false,"context_lines":[{"line_number":78,"context_line":"                          \"%(ex)s: %(e)s\") % {\u0027ex\u0027: index_name, \u0027e\u0027: e})"},{"line_number":79,"context_line":"            raise"},{"line_number":80,"context_line":""},{"line_number":81,"context_line":"    def es_reindex_worker(self, es_reindex, resource_groups, index_names):"},{"line_number":82,"context_line":"        \"\"\"Helper to re-index using the ES reindex helper, allowing all ES"},{"line_number":83,"context_line":"           re-indexes to occur simultaneously. We may need to cleanup. See"},{"line_number":84,"context_line":"           sig_handler() for more info."}],"source_content_type":"text/x-python","patch_set":11,"id":"fad55976_20823301","line":81,"range":{"start_line":81,"start_character":8,"end_line":81,"end_character":25},"updated":"2016-07-15 18:31:04.000000000","message":"Ditto","commit_id":"77f0ccfb4d2f5500706d8ab01c5721c705e42378"},{"author":{"_account_id":14680,"name":"Rick Aulino","email":"rick.aulino@hpe.com","username":"ricka"},"change_message_id":"94a7d98bddb76a7d9c12ce5d20e280d61386e469","unresolved":false,"context_lines":[{"line_number":78,"context_line":"                          \"%(ex)s: %(e)s\") % {\u0027ex\u0027: index_name, \u0027e\u0027: e})"},{"line_number":79,"context_line":"            raise"},{"line_number":80,"context_line":""},{"line_number":81,"context_line":"    def es_reindex_worker(self, es_reindex, resource_groups, index_names):"},{"line_number":82,"context_line":"        \"\"\"Helper to re-index using the ES reindex helper, allowing all ES"},{"line_number":83,"context_line":"           re-indexes to occur simultaneously. We may need to cleanup. See"},{"line_number":84,"context_line":"           sig_handler() for more info."}],"source_content_type":"text/x-python","patch_set":11,"id":"fad55976_386d4b05","line":81,"range":{"start_line":81,"start_character":8,"end_line":81,"end_character":25},"in_reply_to":"fad55976_20823301","updated":"2016-07-15 20:24:31.000000000","message":"_ditto.","commit_id":"77f0ccfb4d2f5500706d8ab01c5721c705e42378"},{"author":{"_account_id":7665,"name":"Travis Tripp","email":"os.travis.tripp@gmail.com","username":"travis.tripp"},"change_message_id":"221c21e7e93567b0407a456057308ab8053ee8b5","unresolved":false,"context_lines":[{"line_number":36,"context_line":""},{"line_number":37,"context_line":"manage_opts \u003d ["},{"line_number":38,"context_line":"    cfg.IntOpt(\u0027workers\u0027, default\u003d3,"},{"line_number":39,"context_line":"               help\u003d\"Maximum number of worker threads for indexing.\")"},{"line_number":40,"context_line":"]"},{"line_number":41,"context_line":""},{"line_number":42,"context_line":"CONF.register_opts(manage_opts, group\u003d\u0027manage\u0027)"}],"source_content_type":"text/x-python","patch_set":12,"id":"dada55a8_ad171350","line":39,"updated":"2016-07-18 20:50:51.000000000","message":"Is 3 just an arbitrary number? Could it be set to the number of enabled plugins as a default? Is their a downside to that?","commit_id":"4628e51d8def05be961ac085a807ac5765c254b2"},{"author":{"_account_id":14680,"name":"Rick Aulino","email":"rick.aulino@hpe.com","username":"ricka"},"change_message_id":"af5d962c5243059da04aad93f3a9c17652ffcc97","unresolved":false,"context_lines":[{"line_number":36,"context_line":""},{"line_number":37,"context_line":"manage_opts \u003d ["},{"line_number":38,"context_line":"    cfg.IntOpt(\u0027workers\u0027, default\u003d3,"},{"line_number":39,"context_line":"               help\u003d\"Maximum number of worker threads for indexing.\")"},{"line_number":40,"context_line":"]"},{"line_number":41,"context_line":""},{"line_number":42,"context_line":"CONF.register_opts(manage_opts, group\u003d\u0027manage\u0027)"}],"source_content_type":"text/x-python","patch_set":12,"id":"dada55a8_31cb78a7","line":39,"in_reply_to":"dada55a8_ad171350","updated":"2016-07-19 02:17:54.000000000","message":"Sort of. I tried different values and the performance wasn\u0027t changing much above 3. Digging into this some more, there are a couple of resource types whose times were overwhelming the others. That was the bottleneck.\n\nAs an example, I had 12 resource types. 8 took well less than a second to re-index. 2 took about 2 seconds. The remaining two took an order of magnitude longer: OS::Glance::Metadef took 15 seconds and OS::Glance::Images took 21 seconds.\n\nSo three threads meant one devoted to OS::Glance::Metadef, one devoted to OS::Glance::Images and one for the rest.\n\nI\u0027m not sure if increasing the default number of threads will buy us much more. But in case there are more resource type hogs in the future, we can bump up the default more than 3.","commit_id":"4628e51d8def05be961ac085a807ac5765c254b2"},{"author":{"_account_id":7665,"name":"Travis Tripp","email":"os.travis.tripp@gmail.com","username":"travis.tripp"},"change_message_id":"221c21e7e93567b0407a456057308ab8053ee8b5","unresolved":false,"context_lines":[{"line_number":67,"context_line":""},{"line_number":68,"context_line":"        try:"},{"line_number":69,"context_line":"            plugin_obj.index_initial_data(index_name\u003dindex_name)"},{"line_number":70,"context_line":"            es_utils.refresh_index(index_name)"},{"line_number":71,"context_line":"        except exceptions.EndpointNotFound:"},{"line_number":72,"context_line":"            # Display a warning, do not propogate."},{"line_number":73,"context_line":"            doc \u003d plugin_obj.get_document_type()"}],"source_content_type":"text/x-python","patch_set":12,"id":"dada55a8_889b1dc9","line":70,"updated":"2016-07-18 20:50:51.000000000","message":"An info level log message here that the plugin is complete could be good. Having a common message to look for wouldn\u0027t hurt.","commit_id":"4628e51d8def05be961ac085a807ac5765c254b2"},{"author":{"_account_id":14680,"name":"Rick Aulino","email":"rick.aulino@hpe.com","username":"ricka"},"change_message_id":"af5d962c5243059da04aad93f3a9c17652ffcc97","unresolved":false,"context_lines":[{"line_number":67,"context_line":""},{"line_number":68,"context_line":"        try:"},{"line_number":69,"context_line":"            plugin_obj.index_initial_data(index_name\u003dindex_name)"},{"line_number":70,"context_line":"            es_utils.refresh_index(index_name)"},{"line_number":71,"context_line":"        except exceptions.EndpointNotFound:"},{"line_number":72,"context_line":"            # Display a warning, do not propogate."},{"line_number":73,"context_line":"            doc \u003d plugin_obj.get_document_type()"}],"source_content_type":"text/x-python","patch_set":12,"id":"dada55a8_018ee377","line":70,"in_reply_to":"dada55a8_889b1dc9","updated":"2016-07-19 02:17:54.000000000","message":"Done. Made the start and end message similar for searching. Same with the ES re-index below.","commit_id":"4628e51d8def05be961ac085a807ac5765c254b2"},{"author":{"_account_id":7665,"name":"Travis Tripp","email":"os.travis.tripp@gmail.com","username":"travis.tripp"},"change_message_id":"221c21e7e93567b0407a456057308ab8053ee8b5","unresolved":false,"context_lines":[{"line_number":98,"context_line":"                es_utils.reindex(src_index\u003dalias_search,"},{"line_number":99,"context_line":"                                 dst_index\u003ddst_index,"},{"line_number":100,"context_line":"                                 type_list\u003des_reindex)"},{"line_number":101,"context_line":"                es_utils.refresh_index(dst_index)"},{"line_number":102,"context_line":"            except Exception as e:"},{"line_number":103,"context_line":"                LOG.error(_LE(\"Failed to setup index extension \""},{"line_number":104,"context_line":"                              \"%(ex)s: %(e)s\") % {\u0027ex\u0027: dst_index, \u0027e\u0027: e})"}],"source_content_type":"text/x-python","patch_set":12,"id":"dada55a8_68a069f5","line":101,"updated":"2016-07-18 20:50:51.000000000","message":"An info level log message here that the plugin is complete could be good. Having a common message to look for wouldn\u0027t hurt.","commit_id":"4628e51d8def05be961ac085a807ac5765c254b2"},{"author":{"_account_id":14680,"name":"Rick Aulino","email":"rick.aulino@hpe.com","username":"ricka"},"change_message_id":"af5d962c5243059da04aad93f3a9c17652ffcc97","unresolved":false,"context_lines":[{"line_number":98,"context_line":"                es_utils.reindex(src_index\u003dalias_search,"},{"line_number":99,"context_line":"                                 dst_index\u003ddst_index,"},{"line_number":100,"context_line":"                                 type_list\u003des_reindex)"},{"line_number":101,"context_line":"                es_utils.refresh_index(dst_index)"},{"line_number":102,"context_line":"            except Exception as e:"},{"line_number":103,"context_line":"                LOG.error(_LE(\"Failed to setup index extension \""},{"line_number":104,"context_line":"                              \"%(ex)s: %(e)s\") % {\u0027ex\u0027: dst_index, \u0027e\u0027: e})"}],"source_content_type":"text/x-python","patch_set":12,"id":"dada55a8_41ee1b23","line":101,"in_reply_to":"dada55a8_68a069f5","updated":"2016-07-19 02:17:54.000000000","message":"Done. See above comment.","commit_id":"4628e51d8def05be961ac085a807ac5765c254b2"},{"author":{"_account_id":16150,"name":"Lei Zhang","email":"lei.a.zhang@intel.com","username":"leizhang"},"change_message_id":"39594cd4c5b4974d24eb1e6a81e5cf89a5c6a864","unresolved":false,"context_lines":[{"line_number":111,"context_line":"    @args(\u0027--force\u0027, dest\u003d\u0027force\u0027, action\u003d\u0027store_true\u0027,"},{"line_number":112,"context_line":"          help\u003d\"Don\u0027t prompt (answer \u0027y\u0027)\")"},{"line_number":113,"context_line":"    def sync(self, group\u003dNone, _type\u003dNone, force\u003dFalse):"},{"line_number":114,"context_line":"        def wait_for_threads():"},{"line_number":115,"context_line":"            \"\"\"Patiently wait for all running threads to complete."},{"line_number":116,"context_line":"            \"\"\""},{"line_number":117,"context_line":"            threads_running \u003d True"}],"source_content_type":"text/x-python","patch_set":12,"id":"dada55a8_a4ca9b4d","line":114,"updated":"2016-07-18 16:24:35.000000000","message":"I\u0027m not quite familiar with concurrent module, but find there is a function does similar things, https://docs.python.org/3/library/concurrent.futures.html#concurrent.futures.wait. Is there any reason to write a new function?","commit_id":"4628e51d8def05be961ac085a807ac5765c254b2"},{"author":{"_account_id":14680,"name":"Rick Aulino","email":"rick.aulino@hpe.com","username":"ricka"},"change_message_id":"c169d53fd317af29eaf1a4ef186e7f62bbe270f2","unresolved":false,"context_lines":[{"line_number":111,"context_line":"    @args(\u0027--force\u0027, dest\u003d\u0027force\u0027, action\u003d\u0027store_true\u0027,"},{"line_number":112,"context_line":"          help\u003d\"Don\u0027t prompt (answer \u0027y\u0027)\")"},{"line_number":113,"context_line":"    def sync(self, group\u003dNone, _type\u003dNone, force\u003dFalse):"},{"line_number":114,"context_line":"        def wait_for_threads():"},{"line_number":115,"context_line":"            \"\"\"Patiently wait for all running threads to complete."},{"line_number":116,"context_line":"            \"\"\""},{"line_number":117,"context_line":"            threads_running \u003d True"}],"source_content_type":"text/x-python","patch_set":12,"id":"dada55a8_f349f7d1","line":114,"in_reply_to":"dada55a8_a4ca9b4d","updated":"2016-07-18 18:02:10.000000000","message":"If we call future.wait(), we will block until all threads are finished. The issue is that we want to catch any interrupts from the user (control-c). That is why we are looping, so the main thread can catch a control-c and cleanup/exit while the other threads are running.\n\nWe could call futures.wait with a timeout call, but we will still need to loop. It would basically be the same loop as below, but calling future.wait() instead of future.done(). In this case we would check if the returned \"not_done\" set is empty vs the return value of future.done().","commit_id":"4628e51d8def05be961ac085a807ac5765c254b2"},{"author":{"_account_id":16150,"name":"Lei Zhang","email":"lei.a.zhang@intel.com","username":"leizhang"},"change_message_id":"013444bab685256d259d544d8866e8ff59c985b4","unresolved":false,"context_lines":[{"line_number":111,"context_line":"    @args(\u0027--force\u0027, dest\u003d\u0027force\u0027, action\u003d\u0027store_true\u0027,"},{"line_number":112,"context_line":"          help\u003d\"Don\u0027t prompt (answer \u0027y\u0027)\")"},{"line_number":113,"context_line":"    def sync(self, group\u003dNone, _type\u003dNone, force\u003dFalse):"},{"line_number":114,"context_line":"        def wait_for_threads():"},{"line_number":115,"context_line":"            \"\"\"Patiently wait for all running threads to complete."},{"line_number":116,"context_line":"            \"\"\""},{"line_number":117,"context_line":"            threads_running \u003d True"}],"source_content_type":"text/x-python","patch_set":12,"id":"dada55a8_1410e2a7","line":114,"in_reply_to":"dada55a8_f349f7d1","updated":"2016-07-19 03:44:56.000000000","message":"Yeah, you are right, learned a new lesson : )","commit_id":"4628e51d8def05be961ac085a807ac5765c254b2"},{"author":{"_account_id":16150,"name":"Lei Zhang","email":"lei.a.zhang@intel.com","username":"leizhang"},"change_message_id":"39594cd4c5b4974d24eb1e6a81e5cf89a5c6a864","unresolved":false,"context_lines":[{"line_number":349,"context_line":"        #           searchlight-specs/specs/newton/"},{"line_number":350,"context_line":"        #             index-performance-enhancement.rst"},{"line_number":351,"context_line":"        max_workers \u003d cfg.CONF.manage.workers"},{"line_number":352,"context_line":"        if max_workers \u003c 1:"},{"line_number":353,"context_line":"            max_workers \u003d 1"},{"line_number":354,"context_line":"        ThreadPoolExec \u003d concurrent.futures.ThreadPoolExecutor"},{"line_number":355,"context_line":"        with ThreadPoolExec(max_workers\u003dmax_workers) as executor:"}],"source_content_type":"text/x-python","patch_set":12,"id":"dada55a8_84e4b70d","line":352,"updated":"2016-07-18 16:24:35.000000000","message":"We can set min value for workers option instead.","commit_id":"4628e51d8def05be961ac085a807ac5765c254b2"},{"author":{"_account_id":14680,"name":"Rick Aulino","email":"rick.aulino@hpe.com","username":"ricka"},"change_message_id":"c169d53fd317af29eaf1a4ef186e7f62bbe270f2","unresolved":false,"context_lines":[{"line_number":349,"context_line":"        #           searchlight-specs/specs/newton/"},{"line_number":350,"context_line":"        #             index-performance-enhancement.rst"},{"line_number":351,"context_line":"        max_workers \u003d cfg.CONF.manage.workers"},{"line_number":352,"context_line":"        if max_workers \u003c 1:"},{"line_number":353,"context_line":"            max_workers \u003d 1"},{"line_number":354,"context_line":"        ThreadPoolExec \u003d concurrent.futures.ThreadPoolExecutor"},{"line_number":355,"context_line":"        with ThreadPoolExec(max_workers\u003dmax_workers) as executor:"}],"source_content_type":"text/x-python","patch_set":12,"id":"dada55a8_b37c9f21","line":352,"in_reply_to":"dada55a8_84e4b70d","updated":"2016-07-18 18:02:10.000000000","message":"Setting a min value will cause the command to fail when the number of workers is set to less than 1. I was thinking that we would still work, just silently using a number of workers that most closely matches the intent. Assuming that the intent of a -1 is to still run the command :)\n\nDo you think this is incorrect behavior? In other words, if the user specifies a number of 0 or -1 that we should fail instead of running. If so, I can add a \"min\" parameter to line 38 above.\n\nOr is the comment more along the lines of the implementation. That we should set the minimum value on line 38, catch the exception and then force it to the minimum value?","commit_id":"4628e51d8def05be961ac085a807ac5765c254b2"},{"author":{"_account_id":16150,"name":"Lei Zhang","email":"lei.a.zhang@intel.com","username":"leizhang"},"change_message_id":"013444bab685256d259d544d8866e8ff59c985b4","unresolved":false,"context_lines":[{"line_number":349,"context_line":"        #           searchlight-specs/specs/newton/"},{"line_number":350,"context_line":"        #             index-performance-enhancement.rst"},{"line_number":351,"context_line":"        max_workers \u003d cfg.CONF.manage.workers"},{"line_number":352,"context_line":"        if max_workers \u003c 1:"},{"line_number":353,"context_line":"            max_workers \u003d 1"},{"line_number":354,"context_line":"        ThreadPoolExec \u003d concurrent.futures.ThreadPoolExecutor"},{"line_number":355,"context_line":"        with ThreadPoolExec(max_workers\u003dmax_workers) as executor:"}],"source_content_type":"text/x-python","patch_set":12,"id":"dada55a8_f72c304c","line":352,"in_reply_to":"dada55a8_b37c9f21","updated":"2016-07-19 03:44:56.000000000","message":"I\u0027m leaning to just adding a \u0027min\u0027 parameter. I think failing when using wrong option value is more explicit. Otherwise users may not even know there are something wrong in their configurations.","commit_id":"4628e51d8def05be961ac085a807ac5765c254b2"},{"author":{"_account_id":14680,"name":"Rick Aulino","email":"rick.aulino@hpe.com","username":"ricka"},"change_message_id":"23854c94c881a3156e0ed9b5bd1ce54afe0e3301","unresolved":false,"context_lines":[{"line_number":349,"context_line":"        #           searchlight-specs/specs/newton/"},{"line_number":350,"context_line":"        #             index-performance-enhancement.rst"},{"line_number":351,"context_line":"        max_workers \u003d cfg.CONF.manage.workers"},{"line_number":352,"context_line":"        if max_workers \u003c 1:"},{"line_number":353,"context_line":"            max_workers \u003d 1"},{"line_number":354,"context_line":"        ThreadPoolExec \u003d concurrent.futures.ThreadPoolExecutor"},{"line_number":355,"context_line":"        with ThreadPoolExec(max_workers\u003dmax_workers) as executor:"}],"source_content_type":"text/x-python","patch_set":12,"id":"dada55a8_16e9e6e6","line":352,"in_reply_to":"dada55a8_f72c304c","updated":"2016-07-19 15:22:01.000000000","message":"Either way is fine by me. Since you are leaning towards catching the exception, allow me to straighten you up and make the code change :)","commit_id":"4628e51d8def05be961ac085a807ac5765c254b2"},{"author":{"_account_id":7665,"name":"Travis Tripp","email":"os.travis.tripp@gmail.com","username":"travis.tripp"},"change_message_id":"58b8f57206b1c433b3e56a6abb60822e026484a2","unresolved":false,"context_lines":[{"line_number":63,"context_line":"        index_name \u003d index_names[gname]"},{"line_number":64,"context_line":"        dtype \u003d plugin_obj.document_type"},{"line_number":65,"context_line":""},{"line_number":66,"context_line":"        LOG.info(_LI(\"API Reindex start %(type)s into %(index_name)s\") %"},{"line_number":67,"context_line":"                 {\u0027type\u0027: dtype, \u0027index_name\u0027: index_name})"},{"line_number":68,"context_line":""},{"line_number":69,"context_line":"        try:"}],"source_content_type":"text/x-python","patch_set":14,"id":"dada55a8_465b5ef6","line":66,"updated":"2016-07-20 20:41:26.000000000","message":"Started (capital)","commit_id":"c575f3ee69e8712829c91777fb962443fcb27add"},{"author":{"_account_id":7665,"name":"Travis Tripp","email":"os.travis.tripp@gmail.com","username":"travis.tripp"},"change_message_id":"58b8f57206b1c433b3e56a6abb60822e026484a2","unresolved":false,"context_lines":[{"line_number":69,"context_line":"        try:"},{"line_number":70,"context_line":"            plugin_obj.index_initial_data(index_name\u003dindex_name)"},{"line_number":71,"context_line":"            es_utils.refresh_index(index_name)"},{"line_number":72,"context_line":""},{"line_number":73,"context_line":"            LOG.info(_LI(\"API Reindex end %(type)s into %(index_name)s\") %"},{"line_number":74,"context_line":"                     {\u0027type\u0027: dtype, \u0027index_name\u0027: index_name})"},{"line_number":75,"context_line":"        except exceptions.EndpointNotFound:"}],"source_content_type":"text/x-python","patch_set":14,"id":"dada55a8_8671d678","line":72,"updated":"2016-07-20 20:41:26.000000000","message":"Complete instead of end.","commit_id":"c575f3ee69e8712829c91777fb962443fcb27add"},{"author":{"_account_id":7665,"name":"Travis Tripp","email":"os.travis.tripp@gmail.com","username":"travis.tripp"},"change_message_id":"58b8f57206b1c433b3e56a6abb60822e026484a2","unresolved":false,"context_lines":[{"line_number":93,"context_line":"            # alias) of tuple."},{"line_number":94,"context_line":"            alias_search \u003d \\"},{"line_number":95,"context_line":"                [a for a in resource_groups if a[0] \u003d\u003d group][0][1]"},{"line_number":96,"context_line":"            LOG.info(_LI(\"ES Reindex start from %(src)s to %(dst)s \""},{"line_number":97,"context_line":"                         \"for types %(types)s\") %"},{"line_number":98,"context_line":"                     {\u0027src\u0027: alias_search, \u0027dst\u0027: index_names[group],"},{"line_number":99,"context_line":"                      \u0027types\u0027: \u0027, \u0027.join(es_reindex)})"}],"source_content_type":"text/x-python","patch_set":14,"id":"dada55a8_26074acb","line":96,"updated":"2016-07-20 20:41:26.000000000","message":"Ditto.","commit_id":"c575f3ee69e8712829c91777fb962443fcb27add"},{"author":{"_account_id":7665,"name":"Travis Tripp","email":"os.travis.tripp@gmail.com","username":"travis.tripp"},"change_message_id":"58b8f57206b1c433b3e56a6abb60822e026484a2","unresolved":false,"context_lines":[{"line_number":103,"context_line":"                                 dst_index\u003ddst_index,"},{"line_number":104,"context_line":"                                 type_list\u003des_reindex)"},{"line_number":105,"context_line":"                es_utils.refresh_index(dst_index)"},{"line_number":106,"context_line":"                LOG.info(_LI(\"ES Reindex end from %(src)s to %(dst)s \""},{"line_number":107,"context_line":"                             \"for types %(types)s\") %"},{"line_number":108,"context_line":"                         {\u0027src\u0027: alias_search, \u0027dst\u0027: index_names[group],"},{"line_number":109,"context_line":"                          \u0027types\u0027: \u0027, \u0027.join(es_reindex)})"}],"source_content_type":"text/x-python","patch_set":14,"id":"dada55a8_660dc2e7","line":106,"updated":"2016-07-20 20:41:26.000000000","message":"ditto","commit_id":"c575f3ee69e8712829c91777fb962443fcb27add"}]}