)]}'
{"specs/newton/searchlight-client-library.rst":[{"author":{"_account_id":7665,"name":"Travis Tripp","email":"os.travis.tripp@gmail.com","username":"travis.tripp"},"change_message_id":"24540317542b9de54d77c0bb29ecc4c3fefdf033","unresolved":false,"context_lines":[{"line_number":6,"context_line":""},{"line_number":7,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":8,"context_line":"Searchlight Client-Library"},{"line_number":9,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":10,"context_line":""},{"line_number":11,"context_line":"SearchLight indexes data in elasticsearch for initial load by calling"},{"line_number":12,"context_line":"the API from source services(glance, nova etc). For keeping the data"}],"source_content_type":"text/x-rst","patch_set":2,"id":"5a18252c_637afcf2","line":9,"updated":"2016-04-13 19:28:54.000000000","message":"Please correct capitalization of service names.","commit_id":"b7a3d5f110ca5027eba306fb9a7c05b06731f934"},{"author":{"_account_id":8959,"name":"Lakshmi N Sampath","email":"lakshmi.sampath@hp.com","username":"lakshmi"},"change_message_id":"643f1bfbe554233c64df13b3d4c45235c2e39e50","unresolved":false,"context_lines":[{"line_number":6,"context_line":""},{"line_number":7,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":8,"context_line":"Searchlight Client-Library"},{"line_number":9,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":10,"context_line":""},{"line_number":11,"context_line":"SearchLight indexes data in elasticsearch for initial load by calling"},{"line_number":12,"context_line":"the API from source services(glance, nova etc). For keeping the data"}],"source_content_type":"text/x-rst","patch_set":2,"id":"5a18252c_7e10d3a8","line":9,"in_reply_to":"5a18252c_637afcf2","updated":"2016-04-13 19:45:30.000000000","message":"Done","commit_id":"b7a3d5f110ca5027eba306fb9a7c05b06731f934"},{"author":{"_account_id":7665,"name":"Travis Tripp","email":"os.travis.tripp@gmail.com","username":"travis.tripp"},"change_message_id":"24540317542b9de54d77c0bb29ecc4c3fefdf033","unresolved":false,"context_lines":[{"line_number":9,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":10,"context_line":""},{"line_number":11,"context_line":"SearchLight indexes data in elasticsearch for initial load by calling"},{"line_number":12,"context_line":"the API from source services(glance, nova etc). For keeping the data"},{"line_number":13,"context_line":"in sync with ongoing changes in the source system, it listens to"},{"line_number":14,"context_line":"notifications sent by the services and updates elasticsearch for changes."},{"line_number":15,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"5a18252c_c30f902c","line":12,"updated":"2016-04-13 19:28:54.000000000","message":"nit: space between services and parenthesis.","commit_id":"b7a3d5f110ca5027eba306fb9a7c05b06731f934"},{"author":{"_account_id":8959,"name":"Lakshmi N Sampath","email":"lakshmi.sampath@hp.com","username":"lakshmi"},"change_message_id":"643f1bfbe554233c64df13b3d4c45235c2e39e50","unresolved":false,"context_lines":[{"line_number":9,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":10,"context_line":""},{"line_number":11,"context_line":"SearchLight indexes data in elasticsearch for initial load by calling"},{"line_number":12,"context_line":"the API from source services(glance, nova etc). For keeping the data"},{"line_number":13,"context_line":"in sync with ongoing changes in the source system, it listens to"},{"line_number":14,"context_line":"notifications sent by the services and updates elasticsearch for changes."},{"line_number":15,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"5a18252c_de276710","line":12,"in_reply_to":"5a18252c_c30f902c","updated":"2016-04-13 19:45:30.000000000","message":"Done","commit_id":"b7a3d5f110ca5027eba306fb9a7c05b06731f934"},{"author":{"_account_id":7665,"name":"Travis Tripp","email":"os.travis.tripp@gmail.com","username":"travis.tripp"},"change_message_id":"24540317542b9de54d77c0bb29ecc4c3fefdf033","unresolved":false,"context_lines":[{"line_number":11,"context_line":"SearchLight indexes data in elasticsearch for initial load by calling"},{"line_number":12,"context_line":"the API from source services(glance, nova etc). For keeping the data"},{"line_number":13,"context_line":"in sync with ongoing changes in the source system, it listens to"},{"line_number":14,"context_line":"notifications sent by the services and updates elasticsearch for changes."},{"line_number":15,"context_line":""},{"line_number":16,"context_line":"Source service has to either send notifications or call searchlight"},{"line_number":17,"context_line":"for ongoing changes on source data."}],"source_content_type":"text/x-rst","patch_set":2,"id":"5a18252c_a321b4a1","line":14,"updated":"2016-04-13 19:28:54.000000000","message":"I think drop \"for changes\"","commit_id":"b7a3d5f110ca5027eba306fb9a7c05b06731f934"},{"author":{"_account_id":8959,"name":"Lakshmi N Sampath","email":"lakshmi.sampath@hp.com","username":"lakshmi"},"change_message_id":"643f1bfbe554233c64df13b3d4c45235c2e39e50","unresolved":false,"context_lines":[{"line_number":11,"context_line":"SearchLight indexes data in elasticsearch for initial load by calling"},{"line_number":12,"context_line":"the API from source services(glance, nova etc). For keeping the data"},{"line_number":13,"context_line":"in sync with ongoing changes in the source system, it listens to"},{"line_number":14,"context_line":"notifications sent by the services and updates elasticsearch for changes."},{"line_number":15,"context_line":""},{"line_number":16,"context_line":"Source service has to either send notifications or call searchlight"},{"line_number":17,"context_line":"for ongoing changes on source data."}],"source_content_type":"text/x-rst","patch_set":2,"id":"5a18252c_9e84cfd4","line":14,"in_reply_to":"5a18252c_a321b4a1","updated":"2016-04-13 19:45:30.000000000","message":"Done","commit_id":"b7a3d5f110ca5027eba306fb9a7c05b06731f934"},{"author":{"_account_id":7665,"name":"Travis Tripp","email":"os.travis.tripp@gmail.com","username":"travis.tripp"},"change_message_id":"24540317542b9de54d77c0bb29ecc4c3fefdf033","unresolved":false,"context_lines":[{"line_number":14,"context_line":"notifications sent by the services and updates elasticsearch for changes."},{"line_number":15,"context_line":""},{"line_number":16,"context_line":"Source service has to either send notifications or call searchlight"},{"line_number":17,"context_line":"for ongoing changes on source data."},{"line_number":18,"context_line":""},{"line_number":19,"context_line":"Problem Description"},{"line_number":20,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":2,"id":"5a18252c_6375dc96","line":17,"updated":"2016-04-13 19:28:54.000000000","message":"How about, \"With this proposal, the source service will be able to either send notifications or use a new client library to directly index the data by bypassing additional middleware.\"","commit_id":"b7a3d5f110ca5027eba306fb9a7c05b06731f934"},{"author":{"_account_id":8959,"name":"Lakshmi N Sampath","email":"lakshmi.sampath@hp.com","username":"lakshmi"},"change_message_id":"643f1bfbe554233c64df13b3d4c45235c2e39e50","unresolved":false,"context_lines":[{"line_number":14,"context_line":"notifications sent by the services and updates elasticsearch for changes."},{"line_number":15,"context_line":""},{"line_number":16,"context_line":"Source service has to either send notifications or call searchlight"},{"line_number":17,"context_line":"for ongoing changes on source data."},{"line_number":18,"context_line":""},{"line_number":19,"context_line":"Problem Description"},{"line_number":20,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":2,"id":"5a18252c_3ef2db6f","line":17,"in_reply_to":"5a18252c_6375dc96","updated":"2016-04-13 19:45:30.000000000","message":"Done.  Clarified on middleware since it might be confused with wsgi middleware","commit_id":"b7a3d5f110ca5027eba306fb9a7c05b06731f934"},{"author":{"_account_id":7665,"name":"Travis Tripp","email":"os.travis.tripp@gmail.com","username":"travis.tripp"},"change_message_id":"24540317542b9de54d77c0bb29ecc4c3fefdf033","unresolved":false,"context_lines":[{"line_number":20,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":21,"context_line":""},{"line_number":22,"context_line":"Most of the services in openstack have notification events which are sent"},{"line_number":23,"context_line":"in near realtime for the ongoing changes to service data."},{"line_number":24,"context_line":""},{"line_number":25,"context_line":"For services which don\u0027t generate any notifications, searchlight cannot"},{"line_number":26,"context_line":"keep the data in elasticsearch updated with latest changes in source service."}],"source_content_type":"text/x-rst","patch_set":2,"id":"5a18252c_c322f000","line":23,"updated":"2016-04-13 19:28:54.000000000","message":"These services use the OSLO Messaging library.","commit_id":"b7a3d5f110ca5027eba306fb9a7c05b06731f934"},{"author":{"_account_id":8959,"name":"Lakshmi N Sampath","email":"lakshmi.sampath@hp.com","username":"lakshmi"},"change_message_id":"643f1bfbe554233c64df13b3d4c45235c2e39e50","unresolved":false,"context_lines":[{"line_number":20,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":21,"context_line":""},{"line_number":22,"context_line":"Most of the services in openstack have notification events which are sent"},{"line_number":23,"context_line":"in near realtime for the ongoing changes to service data."},{"line_number":24,"context_line":""},{"line_number":25,"context_line":"For services which don\u0027t generate any notifications, searchlight cannot"},{"line_number":26,"context_line":"keep the data in elasticsearch updated with latest changes in source service."}],"source_content_type":"text/x-rst","patch_set":2,"id":"5a18252c_fe54e321","line":23,"in_reply_to":"5a18252c_c322f000","updated":"2016-04-13 19:45:30.000000000","message":"Done","commit_id":"b7a3d5f110ca5027eba306fb9a7c05b06731f934"},{"author":{"_account_id":7665,"name":"Travis Tripp","email":"os.travis.tripp@gmail.com","username":"travis.tripp"},"change_message_id":"24540317542b9de54d77c0bb29ecc4c3fefdf033","unresolved":false,"context_lines":[{"line_number":49,"context_line":""},{"line_number":50,"context_line":"Option 2 and 3 will have the disadvantage of not utilizing the full"},{"line_number":51,"context_line":"capabilities of searchlight like zero down time reindexing."},{"line_number":52,"context_line":""},{"line_number":53,"context_line":""},{"line_number":54,"context_line":"POC:"},{"line_number":55,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"5a18252c_bec6eb46","line":52,"updated":"2016-04-13 19:28:54.000000000","message":"I started writing inline suggestions and then decided to just provide a suggested wording improvement for the whole section\n\nThe following options have been identified for services which don\u0027t generate notifications.\n\nChange the service to send notifications using the OSLO messaging libraries.\nCreate a Searchlight client library which can be used by the service to directly update ElasticSearch. This library will need to stay aligned with its associated Searchlight API plugin to ensure that the data mappings, resource types, and target indices stay in sync.\nCreate a client library which is similar to option 2 but does not add any additional dependencies.  The library will not have any dependencies and will use http client to directly communicate with ElasticSearch for sending updates (rather than the ElasticSearch python client library).\nOption 1 is the recommended option for resiliency. Without it, the source service must take ownership of handling ElasticSearch downtime as well as handling all data consistency checks.\n\nIf notifications cannot be sent due to some restrictions on source service, the next best option would be 2 and then 3.\n\nOption 2 and 3 will have the disadvantage of not utilizing the full capabilities of searchlight like zero down time reindexing [0].\n\n[0] Searchlight provides a feature called zero downtime re-indexing where an administrator can on-demand rebuild the indexes from various sources without any noticeable disruption to end users.\n\nhttps://specs.openstack.org/openstack/searchlight-specs/specs/mitaka/zero-downtime-reindexing.html","commit_id":"b7a3d5f110ca5027eba306fb9a7c05b06731f934"},{"author":{"_account_id":8959,"name":"Lakshmi N Sampath","email":"lakshmi.sampath@hp.com","username":"lakshmi"},"change_message_id":"643f1bfbe554233c64df13b3d4c45235c2e39e50","unresolved":false,"context_lines":[{"line_number":49,"context_line":""},{"line_number":50,"context_line":"Option 2 and 3 will have the disadvantage of not utilizing the full"},{"line_number":51,"context_line":"capabilities of searchlight like zero down time reindexing."},{"line_number":52,"context_line":""},{"line_number":53,"context_line":""},{"line_number":54,"context_line":"POC:"},{"line_number":55,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"5a18252c_1eac3f33","line":52,"in_reply_to":"5a18252c_bec6eb46","updated":"2016-04-13 19:45:30.000000000","message":"Done.","commit_id":"b7a3d5f110ca5027eba306fb9a7c05b06731f934"},{"author":{"_account_id":17363,"name":"Timur Alperovich","email":"timur@timuralp.com","username":"timuralp"},"change_message_id":"092f9ff50f969a75f19adb5da6f8b3e58189cf25","unresolved":false,"context_lines":[{"line_number":35,"context_line":""},{"line_number":36,"context_line":"1. Change the service to send notifications using the OSLO messaging libraries."},{"line_number":37,"context_line":""},{"line_number":38,"context_line":"2. Create a Searchlight client library which can be used by the service to directly"},{"line_number":39,"context_line":"   update ElasticSearch. This library will need to stay aligned with its associated"},{"line_number":40,"context_line":"   Searchlight API plugin to ensure that the data mappings, resource types, and"},{"line_number":41,"context_line":"   target indices stay in sync."}],"source_content_type":"text/x-rst","patch_set":4,"id":"5a18252c_0f97bd3a","line":38,"updated":"2016-04-14 13:35:14.000000000","message":"I wanted to share two points about possible ways to implement this.\nThe first uses a middleware to propagate updates to ES. The middleware can use green threads and minimally affect the Swift request processing. It is then added to the pipeline on every proxy node.\nThe second one that may be interesting is through this patch: https://review.openstack.org/#/c/304332/. It exposes the get_items_since() API on the container server, which would allow a process to look through all of the entries in the container database and propagate those updates to ES. It would allow for building a more efficient indexer and is the container_sync approach John referred to during the meeting.","commit_id":"f28f4cb93170ec723e770492734ebe2c907cc165"},{"author":{"_account_id":8959,"name":"Lakshmi N Sampath","email":"lakshmi.sampath@hp.com","username":"lakshmi"},"change_message_id":"3c708344410ddafab6b8bd785df28b2b88ff23b5","unresolved":false,"context_lines":[{"line_number":35,"context_line":""},{"line_number":36,"context_line":"1. Change the service to send notifications using the OSLO messaging libraries."},{"line_number":37,"context_line":""},{"line_number":38,"context_line":"2. Create a Searchlight client library which can be used by the service to directly"},{"line_number":39,"context_line":"   update ElasticSearch. This library will need to stay aligned with its associated"},{"line_number":40,"context_line":"   Searchlight API plugin to ensure that the data mappings, resource types, and"},{"line_number":41,"context_line":"   target indices stay in sync."}],"source_content_type":"text/x-rst","patch_set":4,"id":"5a18252c_00965e3e","line":38,"in_reply_to":"5a18252c_0f97bd3a","updated":"2016-04-14 17:33:56.000000000","message":"Calling Searchlight from Swift WSGI middleware was used in notification patch but the problem was that response headers in the pipeline didn\u0027t have all the fields required for indexing in searchlight, so within the middleware a HEAD request was issued to get the details.  I will document that as an approach here.","commit_id":"f28f4cb93170ec723e770492734ebe2c907cc165"},{"author":{"_account_id":7665,"name":"Travis Tripp","email":"os.travis.tripp@gmail.com","username":"travis.tripp"},"change_message_id":"ee1ea1b0081401fced5c90d58005eb842bf3bd1b","unresolved":false,"context_lines":[{"line_number":58,"context_line":""},{"line_number":59,"context_line":"Option 1 POC : https://review.openstack.org/#/c/249471/"},{"line_number":60,"context_line":""},{"line_number":61,"context_line":"Option 2 POC : https://review.openstack.org/#/c/305309/"},{"line_number":62,"context_line":""},{"line_number":63,"context_line":"Alternatives"},{"line_number":64,"context_line":"------------"}],"source_content_type":"text/x-rst","patch_set":4,"id":"5a18252c_1b644ce2","line":61,"updated":"2016-04-13 20:48:07.000000000","message":"This is option 3 POC (no dependency on ES).","commit_id":"f28f4cb93170ec723e770492734ebe2c907cc165"},{"author":{"_account_id":8959,"name":"Lakshmi N Sampath","email":"lakshmi.sampath@hp.com","username":"lakshmi"},"change_message_id":"3c708344410ddafab6b8bd785df28b2b88ff23b5","unresolved":false,"context_lines":[{"line_number":58,"context_line":""},{"line_number":59,"context_line":"Option 1 POC : https://review.openstack.org/#/c/249471/"},{"line_number":60,"context_line":""},{"line_number":61,"context_line":"Option 2 POC : https://review.openstack.org/#/c/305309/"},{"line_number":62,"context_line":""},{"line_number":63,"context_line":"Alternatives"},{"line_number":64,"context_line":"------------"}],"source_content_type":"text/x-rst","patch_set":4,"id":"5a18252c_409cd65c","line":61,"in_reply_to":"5a18252c_1b644ce2","updated":"2016-04-14 17:33:56.000000000","message":"Done. Got confused while turning around the numbers","commit_id":"f28f4cb93170ec723e770492734ebe2c907cc165"},{"author":{"_account_id":7665,"name":"Travis Tripp","email":"os.travis.tripp@gmail.com","username":"travis.tripp"},"change_message_id":"10621b53c0b3eb13ffacb903ae32542d422dcb85","unresolved":false,"context_lines":[{"line_number":72,"context_line":""},{"line_number":73,"context_line":"References"},{"line_number":74,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":75,"context_line":""},{"line_number":76,"context_line":"[0] Searchlight provides a feature called zero downtime re-indexing where an"},{"line_number":77,"context_line":"    administrator can on-demand rebuild the indexes from various sources without any"},{"line_number":78,"context_line":"    noticeable disruption to end users."}],"source_content_type":"text/x-rst","patch_set":4,"id":"5a18252c_b178d92c","line":75,"updated":"2016-04-13 21:47:12.000000000","message":"Better add http://docs.openstack.org/developer/swift/overview_container_sync.html#what-s-going-on-behind-the-scenes-in-the-cluster","commit_id":"f28f4cb93170ec723e770492734ebe2c907cc165"},{"author":{"_account_id":8959,"name":"Lakshmi N Sampath","email":"lakshmi.sampath@hp.com","username":"lakshmi"},"change_message_id":"3c708344410ddafab6b8bd785df28b2b88ff23b5","unresolved":false,"context_lines":[{"line_number":72,"context_line":""},{"line_number":73,"context_line":"References"},{"line_number":74,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":75,"context_line":""},{"line_number":76,"context_line":"[0] Searchlight provides a feature called zero downtime re-indexing where an"},{"line_number":77,"context_line":"    administrator can on-demand rebuild the indexes from various sources without any"},{"line_number":78,"context_line":"    noticeable disruption to end users."}],"source_content_type":"text/x-rst","patch_set":4,"id":"5a18252c_00bdbeb6","line":75,"in_reply_to":"5a18252c_b178d92c","updated":"2016-04-14 17:33:56.000000000","message":"Done","commit_id":"f28f4cb93170ec723e770492734ebe2c907cc165"}]}
