)]}'
{"releasenotes/notes/bug-1596710-6a5287f0cd77cba0.yaml":[{"author":{"_account_id":15888,"name":"Zhenyu Zheng","email":"zheng.zhenyu@outlook.com","username":"Kevin_Zheng"},"change_message_id":"ac2f6892968c1a48b56f2c6b74e38de4b8a22b1e","unresolved":false,"context_lines":[{"line_number":4,"context_line":"    parameter, searchlight may copy existing Elasticsearch"},{"line_number":5,"context_line":"    data to reindex, if these old indices or type mappings"},{"line_number":6,"context_line":"    are missing, newly created indices will have incomplete"},{"line_number":7,"context_line":"    resource data. "},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"fixes:"},{"line_number":10,"context_line":"  - Fix a bug that incomlete types may be copyed during"}],"source_content_type":"text/x-yaml","patch_set":10,"id":"3a71b18c_1a584bfa","line":7,"range":{"start_line":7,"start_character":18,"end_line":7,"end_character":19},"updated":"2016-12-01 13:55:50.000000000","message":"nit, extra whitespace","commit_id":"fc604e6f4fbaf4495bbb055f279e6371a23ac08d"},{"author":{"_account_id":15888,"name":"Zhenyu Zheng","email":"zheng.zhenyu@outlook.com","username":"Kevin_Zheng"},"change_message_id":"ac2f6892968c1a48b56f2c6b74e38de4b8a22b1e","unresolved":false,"context_lines":[{"line_number":7,"context_line":"    resource data. "},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"fixes:"},{"line_number":10,"context_line":"  - Fix a bug that incomlete types may be copyed during"},{"line_number":11,"context_line":"    reindex. Add checks about data integrity before copying"},{"line_number":12,"context_line":"    existing Elasticsearch data, warn user if there is any"},{"line_number":13,"context_line":"    missing index or type mapping and exit. Users can"}],"source_content_type":"text/x-yaml","patch_set":10,"id":"3a71b18c_3a1427f4","line":10,"range":{"start_line":10,"start_character":19,"end_line":10,"end_character":28},"updated":"2016-12-01 13:55:50.000000000","message":"typo s/incomlete/incomplete","commit_id":"fc604e6f4fbaf4495bbb055f279e6371a23ac08d"}],"searchlight/cmd/manage.py":[{"author":{"_account_id":10063,"name":"Steve McLellan","email":"steven.j.mclellan@gmail.com","username":"sjmc7"},"change_message_id":"6b89ef6e034612edfe1cc6289e7a509cfd3d7671","unresolved":false,"context_lines":[{"line_number":272,"context_line":"                    # Don\u0027t reindex this type"},{"line_number":273,"context_line":"                    plugins_to_index.remove((resource_type, ext))"},{"line_number":274,"context_line":""},{"line_number":275,"context_line":"        non_exist_doc_types \u003d es_utils.find_non_exist_types(es_reindex_mapping)"},{"line_number":276,"context_line":""},{"line_number":277,"context_line":"        if non_exist_doc_types:"},{"line_number":278,"context_line":"            print("}],"source_content_type":"text/x-python","patch_set":4,"id":"7a8ec9b2_a7e1642c","line":275,"range":{"start_line":275,"start_character":8,"end_line":275,"end_character":27},"updated":"2016-09-21 19:14:46.000000000","message":"I think this would be better structured by creating a dict index:[types] (or a list of (index, [types]) pairs), and within find_non_exist_types use the get_mapping API to check whether mappings are declared for the types.","commit_id":"a72c0deee5c3a0b91c3404bce2b9d893bfad1b77"},{"author":{"_account_id":14680,"name":"Rick Aulino","email":"rick.aulino@hpe.com","username":"ricka"},"change_message_id":"3d80acf387aadd2243d1d05948f9ca0da082948e","unresolved":false,"context_lines":[{"line_number":277,"context_line":"        if non_exist_doc_types:"},{"line_number":278,"context_line":"            print("},{"line_number":279,"context_line":"                \"The following types are supposed to be indexed from \""},{"line_number":280,"context_line":"                \"existing Elasticsearch data: %(types)s, but no data \""},{"line_number":281,"context_line":"                \"is found in Elasticsearch. There may be some data \""},{"line_number":282,"context_line":"                \"corruption in these types, you can specify them in \""},{"line_number":283,"context_line":"                \"--type parameter to renindex data by plugin apis, or \""}],"source_content_type":"text/x-python","patch_set":4,"id":"5a8bc5a2_e3f22606","line":280,"updated":"2016-09-22 06:00:26.000000000","message":"The problem isn\u0027t that there is no data (i.e. /searchlight-search/OS::Neutron::Net is empty). The problem is that either the ES index is missing or the ES document_type (SL resource type) is missing.\n\nSee the comment in find_non_exist_types() for thoughts about making this more clear.","commit_id":"a72c0deee5c3a0b91c3404bce2b9d893bfad1b77"},{"author":{"_account_id":10063,"name":"Steve McLellan","email":"steven.j.mclellan@gmail.com","username":"sjmc7"},"change_message_id":"60e0a3e11e7b6a1760b23c76a9c994dc8bfe63b1","unresolved":false,"context_lines":[{"line_number":276,"context_line":""},{"line_number":277,"context_line":"        if missing_index:"},{"line_number":278,"context_line":"            print("},{"line_number":279,"context_line":"                \"Missing indicies when trying to re-index resource information\""},{"line_number":280,"context_line":"                \" from existing Elasticsearch data: %(index)s.\\n\" % {"},{"line_number":281,"context_line":"                    \"index\": \", \".join(missing_index)})"},{"line_number":282,"context_line":""}],"source_content_type":"text/x-python","patch_set":7,"id":"5a74a57a_a682bdc6","line":279,"range":{"start_line":279,"start_character":25,"end_line":279,"end_character":33},"updated":"2016-11-30 16:40:59.000000000","message":"Typo - indices","commit_id":"8911dbd949f3181497df1c07a2860b4333e7fb46"},{"author":{"_account_id":16150,"name":"Lei Zhang","email":"lei.a.zhang@intel.com","username":"leizhang"},"change_message_id":"7de28c2e5d99e92e7fad41cfdd3afcc5dc987002","unresolved":false,"context_lines":[{"line_number":276,"context_line":""},{"line_number":277,"context_line":"        if missing_index:"},{"line_number":278,"context_line":"            print("},{"line_number":279,"context_line":"                \"Missing indicies when trying to re-index resource information\""},{"line_number":280,"context_line":"                \" from existing Elasticsearch data: %(index)s.\\n\" % {"},{"line_number":281,"context_line":"                    \"index\": \", \".join(missing_index)})"},{"line_number":282,"context_line":""}],"source_content_type":"text/x-python","patch_set":7,"id":"5a74a57a_d18afe19","line":279,"range":{"start_line":279,"start_character":25,"end_line":279,"end_character":33},"in_reply_to":"5a74a57a_a682bdc6","updated":"2016-12-01 01:57:46.000000000","message":"Done","commit_id":"8911dbd949f3181497df1c07a2860b4333e7fb46"}],"searchlight/elasticsearch/plugins/utils.py":[{"author":{"_account_id":10063,"name":"Steve McLellan","email":"steven.j.mclellan@gmail.com","username":"sjmc7"},"change_message_id":"6b89ef6e034612edfe1cc6289e7a509cfd3d7671","unresolved":false,"context_lines":[{"line_number":491,"context_line":"    return facet_terms"},{"line_number":492,"context_line":""},{"line_number":493,"context_line":""},{"line_number":494,"context_line":"def find_non_exist_types(index_type_mapping):"},{"line_number":495,"context_line":"    \"\"\"Find doc types belong to non-exist indices\"\"\""},{"line_number":496,"context_line":"    if not index_type_mapping:"},{"line_number":497,"context_line":"        return"}],"source_content_type":"text/x-python","patch_set":4,"id":"7a8ec9b2_e73c4c95","line":494,"range":{"start_line":494,"start_character":4,"end_line":494,"end_character":24},"updated":"2016-09-21 19:14:46.000000000","message":"Maybe change to \u0027find_missing_mappings\u0027","commit_id":"a72c0deee5c3a0b91c3404bce2b9d893bfad1b77"},{"author":{"_account_id":10063,"name":"Steve McLellan","email":"steven.j.mclellan@gmail.com","username":"sjmc7"},"change_message_id":"6b89ef6e034612edfe1cc6289e7a509cfd3d7671","unresolved":false,"context_lines":[{"line_number":500,"context_line":"    types \u003d []"},{"line_number":501,"context_line":""},{"line_number":502,"context_line":"    for index in six.iterkeys(index_type_mapping):"},{"line_number":503,"context_line":"        if not es_engine.indices.exists(index):"},{"line_number":504,"context_line":"            types.extend(index_type_mapping[index])"},{"line_number":505,"context_line":""},{"line_number":506,"context_line":"    return types"}],"source_content_type":"text/x-python","patch_set":4,"id":"7a8ec9b2_c72670b9","line":503,"range":{"start_line":503,"start_character":39,"end_line":503,"end_character":40},"updated":"2016-09-21 19:14:46.000000000","message":"Having no documents in the index I don\u0027t think is a problem, but the check needs to make sure that the type exists in the mapping for the index.","commit_id":"a72c0deee5c3a0b91c3404bce2b9d893bfad1b77"},{"author":{"_account_id":4428,"name":"Liyingjun","email":"yinjalee@163.com","username":"liyingjun"},"change_message_id":"1daa8b72e54b741455053f94635f4309636c59fe","unresolved":false,"context_lines":[{"line_number":500,"context_line":"    types \u003d []"},{"line_number":501,"context_line":""},{"line_number":502,"context_line":"    for index in six.iterkeys(index_type_mapping):"},{"line_number":503,"context_line":"        if not es_engine.indices.exists(index):"},{"line_number":504,"context_line":"            types.extend(index_type_mapping[index])"},{"line_number":505,"context_line":""},{"line_number":506,"context_line":"    return types"}],"source_content_type":"text/x-python","patch_set":4,"id":"7a8ec9b2_fe42cf66","line":503,"updated":"2016-09-19 09:09:43.000000000","message":"This doesn\u0027t cover the case when the index exists but there is no documents in the index, right?","commit_id":"a72c0deee5c3a0b91c3404bce2b9d893bfad1b77"},{"author":{"_account_id":14680,"name":"Rick Aulino","email":"rick.aulino@hpe.com","username":"ricka"},"change_message_id":"3d80acf387aadd2243d1d05948f9ca0da082948e","unresolved":false,"context_lines":[{"line_number":500,"context_line":"    types \u003d []"},{"line_number":501,"context_line":""},{"line_number":502,"context_line":"    for index in six.iterkeys(index_type_mapping):"},{"line_number":503,"context_line":"        if not es_engine.indices.exists(index):"},{"line_number":504,"context_line":"            types.extend(index_type_mapping[index])"},{"line_number":505,"context_line":""},{"line_number":506,"context_line":"    return types"}],"source_content_type":"text/x-python","patch_set":4,"id":"5a8bc5a2_03aab2d4","line":503,"range":{"start_line":503,"start_character":39,"end_line":503,"end_character":40},"in_reply_to":"7a8ec9b2_c72670b9","updated":"2016-09-22 06:00:26.000000000","message":"It seems there are three cases here. Assume we have the following document\nstructures:\n\n/searchlight/OS::Neutron::Net/11111111-1111-1111-1111-111111111111\n/searchlight/OS::Cinder::Volume/22222222-2222-2222-2222-222222222222\n\nCase #1: The index does not exist. There are no documents for any resource type\ngroups using searchlight. This is an error.\n\nCase #2: The document type (OS::Neutron::Net) does not exist. Even the documents\nwould exist for OS::Cinder::Volume, they would not exist for OS::Neutron::Net.\nThis is an error.\n\nCase #3: All document types are missing. We cannot determine if this is an\nerror. There are valid cases where a particular resource type would not\nhave any resources.\n\nThe check on line 498 handles case #1. We still need to handle case #2.\nChanging the check to (where doc_type is plugin.get_document_type()):\n\n   for index in six.iterkeys(index_type_mapping):\n       for doc in index_type_mapping[index]:\n           try:\n                mapping \u003d \n                   es_engine.indices.get_mapping(index,doc)\n                if not mapping:\n                    types.append(doc_type)\n           expect elasticsearch.exceptions.NotFound:\n               types.append(doc_type)\n\nShould work for both cases. We may want to differentiate between case #1 and case #2 to make it clearer to the user what the issue is. In this case, we may want to keep bad indices (in the exception handler) separate form the bad document types (empty mapping).","commit_id":"a72c0deee5c3a0b91c3404bce2b9d893bfad1b77"},{"author":{"_account_id":10063,"name":"Steve McLellan","email":"steven.j.mclellan@gmail.com","username":"sjmc7"},"change_message_id":"60e0a3e11e7b6a1760b23c76a9c994dc8bfe63b1","unresolved":false,"context_lines":[{"line_number":504,"context_line":"        for doc_type in index_type_mapping[index]:"},{"line_number":505,"context_line":"            try:"},{"line_number":506,"context_line":"                mapping \u003d es_engine.indices.get_mapping(index, doc_type)"},{"line_number":507,"context_line":"                if doc_type \u003d\u003d \u0027OS::Glance::Image\u0027:"},{"line_number":508,"context_line":"                    import ipdb"},{"line_number":509,"context_line":"                    ipdb.set_trace()"},{"line_number":510,"context_line":"                if not mapping:"}],"source_content_type":"text/x-python","patch_set":7,"id":"5a74a57a_33c4313f","line":507,"range":{"start_line":507,"start_character":28,"end_line":507,"end_character":30},"updated":"2016-11-30 16:40:59.000000000","message":"Left over from debugging?","commit_id":"8911dbd949f3181497df1c07a2860b4333e7fb46"},{"author":{"_account_id":16150,"name":"Lei Zhang","email":"lei.a.zhang@intel.com","username":"leizhang"},"change_message_id":"7de28c2e5d99e92e7fad41cfdd3afcc5dc987002","unresolved":false,"context_lines":[{"line_number":504,"context_line":"        for doc_type in index_type_mapping[index]:"},{"line_number":505,"context_line":"            try:"},{"line_number":506,"context_line":"                mapping \u003d es_engine.indices.get_mapping(index, doc_type)"},{"line_number":507,"context_line":"                if doc_type \u003d\u003d \u0027OS::Glance::Image\u0027:"},{"line_number":508,"context_line":"                    import ipdb"},{"line_number":509,"context_line":"                    ipdb.set_trace()"},{"line_number":510,"context_line":"                if not mapping:"}],"source_content_type":"text/x-python","patch_set":7,"id":"5a74a57a_11ae3698","line":507,"range":{"start_line":507,"start_character":28,"end_line":507,"end_character":30},"in_reply_to":"5a74a57a_33c4313f","updated":"2016-12-01 01:57:46.000000000","message":"Yeah, good catch, forget to clean up these debugging code.","commit_id":"8911dbd949f3181497df1c07a2860b4333e7fb46"}]}