)]}'
{"doc/source/userdoc/edp.rst":[{"author":{"_account_id":13919,"name":"Evgeny Sikachev","email":"esikachov@gmail.com","username":"esikachev"},"change_message_id":"2ccb889c007867f60b5da4ae98a1b9f2211bc9f1","unresolved":false,"context_lines":[{"line_number":20,"context_line":"Interfaces"},{"line_number":21,"context_line":"----------"},{"line_number":22,"context_line":""},{"line_number":23,"context_line":"The EDP features can be used from the Sahara web UI which is described in the :doc:`../horizon/dashboard.user.guide`."},{"line_number":24,"context_line":""},{"line_number":25,"context_line":"The EDP features also can be used directly by a client through the `REST api \u003chttp://developer.openstack.org/api-ref-data-processing-v1.1.html\u003e`_"},{"line_number":26,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"da20952f_c2fad474","line":23,"range":{"start_line":23,"start_character":38,"end_line":23,"end_character":39},"updated":"2015-09-07 13:48:46.000000000","message":"s","commit_id":"52aed8adcbcbfc31f328158e85ba07acf5b35cc8"},{"author":{"_account_id":13919,"name":"Evgeny Sikachev","email":"esikachov@gmail.com","username":"esikachev"},"change_message_id":"2ccb889c007867f60b5da4ae98a1b9f2211bc9f1","unresolved":false,"context_lines":[{"line_number":44,"context_line":""},{"line_number":45,"context_line":"A :dfn:`Job Binary` object stores a URL to a single script or Jar file and any credentials needed to retrieve the file.  The file itself may be stored in the Sahara internal database or in Swift."},{"line_number":46,"context_line":""},{"line_number":47,"context_line":"Files in the Sahara database are stored as raw bytes in a :dfn:`Job Binary Internal` object.  This object\u0027s sole purpose is to store a file for later retrieval.  No extra credentials need to be supplied for files stored internally."},{"line_number":48,"context_line":""},{"line_number":49,"context_line":"Sahara requires credentials (username and password) to access files stored in Swift unless Swift proxy users are configured as described in :doc:`../userdoc/advanced.configuration.guide`. The Swift service must be running in the same OpenStack installation referenced by Sahara."},{"line_number":50,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"da20952f_a2eb20a3","line":47,"range":{"start_line":47,"start_character":13,"end_line":47,"end_character":14},"updated":"2015-09-07 13:48:46.000000000","message":"s","commit_id":"52aed8adcbcbfc31f328158e85ba07acf5b35cc8"},{"author":{"_account_id":13919,"name":"Evgeny Sikachev","email":"esikachov@gmail.com","username":"esikachev"},"change_message_id":"2ccb889c007867f60b5da4ae98a1b9f2211bc9f1","unresolved":false,"context_lines":[{"line_number":79,"context_line":""},{"line_number":80,"context_line":"A :dfn:`Data Source` object stores a URL which designates the location of input or output data and any credentials needed to access the location."},{"line_number":81,"context_line":""},{"line_number":82,"context_line":"Sahara supports data sources in Swift. The Swift service must be running in the same OpenStack installation referenced by Sahara."},{"line_number":83,"context_line":""},{"line_number":84,"context_line":"Sahara also supports data sources in HDFS. Any HDFS instance running on a Sahara cluster in the same OpenStack installation is accessible without manual configuration. Other instances of HDFS may be used as well provided that the URL is resolvable from the node executing the job."},{"line_number":85,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"da20952f_6230a856","line":82,"range":{"start_line":82,"start_character":122,"end_line":82,"end_character":123},"updated":"2015-09-07 13:48:46.000000000","message":"s","commit_id":"52aed8adcbcbfc31f328158e85ba07acf5b35cc8"},{"author":{"_account_id":13919,"name":"Evgeny Sikachev","email":"esikachov@gmail.com","username":"esikachev"},"change_message_id":"2ccb889c007867f60b5da4ae98a1b9f2211bc9f1","unresolved":false,"context_lines":[{"line_number":81,"context_line":""},{"line_number":82,"context_line":"Sahara supports data sources in Swift. The Swift service must be running in the same OpenStack installation referenced by Sahara."},{"line_number":83,"context_line":""},{"line_number":84,"context_line":"Sahara also supports data sources in HDFS. Any HDFS instance running on a Sahara cluster in the same OpenStack installation is accessible without manual configuration. Other instances of HDFS may be used as well provided that the URL is resolvable from the node executing the job."},{"line_number":85,"context_line":""},{"line_number":86,"context_line":"Some job types require the use of data source objects to specify input and output when a job is launched. For example, when running a Pig job the UI will prompt the user for input and output data source objects."},{"line_number":87,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"da20952f_0233ec62","line":84,"range":{"start_line":84,"start_character":74,"end_line":84,"end_character":75},"updated":"2015-09-07 13:48:46.000000000","message":"s","commit_id":"52aed8adcbcbfc31f328158e85ba07acf5b35cc8"},{"author":{"_account_id":13919,"name":"Evgeny Sikachev","email":"esikachov@gmail.com","username":"esikachev"},"change_message_id":"2ccb889c007867f60b5da4ae98a1b9f2211bc9f1","unresolved":false,"context_lines":[{"line_number":85,"context_line":""},{"line_number":86,"context_line":"Some job types require the use of data source objects to specify input and output when a job is launched. For example, when running a Pig job the UI will prompt the user for input and output data source objects."},{"line_number":87,"context_line":""},{"line_number":88,"context_line":"Other job types like Java or Spark do not require the user to specify data sources. For these job types, data paths are passed as arguments. For convenience, Sahara allows data source objects to be"},{"line_number":89,"context_line":"referenced by name or id. The section `Using Data Source References as Arguments`_ gives further details."},{"line_number":90,"context_line":""},{"line_number":91,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"da20952f_621748a3","line":88,"range":{"start_line":88,"start_character":158,"end_line":88,"end_character":159},"updated":"2015-09-07 13:48:46.000000000","message":"s","commit_id":"52aed8adcbcbfc31f328158e85ba07acf5b35cc8"},{"author":{"_account_id":13919,"name":"Evgeny Sikachev","email":"esikachov@gmail.com","username":"esikachev"},"change_message_id":"2ccb889c007867f60b5da4ae98a1b9f2211bc9f1","unresolved":false,"context_lines":[{"line_number":103,"context_line":"General Workflow"},{"line_number":104,"context_line":"----------------"},{"line_number":105,"context_line":""},{"line_number":106,"context_line":"The general workflow for defining and executing a job in Sahara is essentially the same whether using the web UI or the REST API."},{"line_number":107,"context_line":""},{"line_number":108,"context_line":"1. Launch a cluster from Sahara if there is not one already available"},{"line_number":109,"context_line":"2. Create all of the Job Binaries needed to run the job, stored in the Sahara database or in Swift"}],"source_content_type":"text/x-rst","patch_set":1,"id":"da20952f_020a0c79","line":106,"range":{"start_line":106,"start_character":57,"end_line":106,"end_character":58},"updated":"2015-09-07 13:48:46.000000000","message":"s","commit_id":"52aed8adcbcbfc31f328158e85ba07acf5b35cc8"},{"author":{"_account_id":13919,"name":"Evgeny Sikachev","email":"esikachov@gmail.com","username":"esikachev"},"change_message_id":"2ccb889c007867f60b5da4ae98a1b9f2211bc9f1","unresolved":false,"context_lines":[{"line_number":196,"context_line":"++++++++++++++++++++++++++++++++++"},{"line_number":197,"context_line":""},{"line_number":198,"context_line":"In order to better document your job for cluster operators (or for yourself in the future)"},{"line_number":199,"context_line":"Sahara allows the addition of an interface (or method signature) to your job template. A"},{"line_number":200,"context_line":"sample interface for the Teragen hadoop example might be:"},{"line_number":201,"context_line":""},{"line_number":202,"context_line":"      +---------------+--------------+------------------+-------------+----------+---------------------+"}],"source_content_type":"text/x-rst","patch_set":1,"id":"da20952f_a2980076","line":199,"range":{"start_line":199,"start_character":0,"end_line":199,"end_character":1},"updated":"2015-09-07 13:48:46.000000000","message":"s","commit_id":"52aed8adcbcbfc31f328158e85ba07acf5b35cc8"},{"author":{"_account_id":13953,"name":"Elise Gafford","email":"egafford@redhat.com","username":"egafford"},"change_message_id":"20f60add21a6c5699eddd7515cc6f16a08a5321c","unresolved":false,"context_lines":[{"line_number":196,"context_line":"++++++++++++++++++++++++++++++++++"},{"line_number":197,"context_line":""},{"line_number":198,"context_line":"In order to better document your job for cluster operators (or for yourself in the future)"},{"line_number":199,"context_line":"Sahara allows the addition of an interface (or method signature) to your job template. A"},{"line_number":200,"context_line":"sample interface for the Teragen hadoop example might be:"},{"line_number":201,"context_line":""},{"line_number":202,"context_line":"      +---------------+--------------+------------------+-------------+----------+---------------------+"}],"source_content_type":"text/x-rst","patch_set":1,"id":"da20952f_a4a8d978","line":199,"in_reply_to":"da20952f_a2980076","updated":"2015-09-08 19:24:12.000000000","message":"Done; the rest of these I fixed in a separate patch (on which I\u0027ve rebased this,) but you\u0027re completely right; I didn\u0027t make this a dependency of that patch, so the \u0027Sahara\u0027 would still\u0027ve been there. Nice catch; thanks.","commit_id":"52aed8adcbcbfc31f328158e85ba07acf5b35cc8"},{"author":{"_account_id":13919,"name":"Evgeny Sikachev","email":"esikachov@gmail.com","username":"esikachev"},"change_message_id":"2ccb889c007867f60b5da4ae98a1b9f2211bc9f1","unresolved":false,"context_lines":[{"line_number":259,"context_line":"Generation of Swift Properties for Data Sources"},{"line_number":260,"context_line":"+++++++++++++++++++++++++++++++++++++++++++++++"},{"line_number":261,"context_line":""},{"line_number":262,"context_line":"If Swift proxy users are not configured (see :doc:`../userdoc/advanced.configuration.guide`) and a job is run with data source objects containing Swift paths, Sahara will automatically generate Swift username and password configuration values based on the credentials in the data sources.  If the input and output data sources are both in Swift, it is expected that they specify the same credentials."},{"line_number":263,"context_line":""},{"line_number":264,"context_line":"The Swift credentials may be set explicitly with the following configuration values:"},{"line_number":265,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"da20952f_c26f3429","line":262,"range":{"start_line":262,"start_character":159,"end_line":262,"end_character":160},"updated":"2015-09-07 13:48:46.000000000","message":"s","commit_id":"52aed8adcbcbfc31f328158e85ba07acf5b35cc8"},{"author":{"_account_id":13919,"name":"Evgeny Sikachev","email":"esikachov@gmail.com","username":"esikachev"},"change_message_id":"2ccb889c007867f60b5da4ae98a1b9f2211bc9f1","unresolved":false,"context_lines":[{"line_number":347,"context_line":"`this example \u003chttps://github.com/openstack/sahara/blob/master/etc/edp-examples/edp-java/README.rst\u003e`_"},{"line_number":348,"context_line":"to explicitly load the values."},{"line_number":349,"context_line":""},{"line_number":350,"context_line":"The following special configuration values are read by Sahara and affect how Java jobs are run:"},{"line_number":351,"context_line":""},{"line_number":352,"context_line":"* ``edp.java.main_class`` (required) Specifies the full name of the class"},{"line_number":353,"context_line":"  containing ``main(String[] args)``"}],"source_content_type":"text/x-rst","patch_set":1,"id":"da20952f_a2700009","line":350,"range":{"start_line":350,"start_character":55,"end_line":350,"end_character":56},"updated":"2015-09-07 13:48:46.000000000","message":"s","commit_id":"52aed8adcbcbfc31f328158e85ba07acf5b35cc8"},{"author":{"_account_id":13919,"name":"Evgeny Sikachev","email":"esikachov@gmail.com","username":"esikachev"},"change_message_id":"2ccb889c007867f60b5da4ae98a1b9f2211bc9f1","unresolved":false,"context_lines":[{"line_number":365,"context_line":"  special handling of configuration values and exit conditions. The default is"},{"line_number":366,"context_line":"  **False**."},{"line_number":367,"context_line":""},{"line_number":368,"context_line":"  If this configuration value is set to **True**, Sahara will modify"},{"line_number":369,"context_line":"  the job\u0027s Hadoop configuration before invoking the specified **main** method."},{"line_number":370,"context_line":"  Any configuration values specified during job launch (excluding those"},{"line_number":371,"context_line":"  beginning with **edp.**) will be automatically set in the job\u0027s Hadoop"}],"source_content_type":"text/x-rst","patch_set":1,"id":"da20952f_62650843","line":368,"range":{"start_line":368,"start_character":50,"end_line":368,"end_character":51},"updated":"2015-09-07 13:48:46.000000000","message":"s","commit_id":"52aed8adcbcbfc31f328158e85ba07acf5b35cc8"},{"author":{"_account_id":13919,"name":"Evgeny Sikachev","email":"esikachov@gmail.com","username":"esikachev"},"change_message_id":"2ccb889c007867f60b5da4ae98a1b9f2211bc9f1","unresolved":false,"context_lines":[{"line_number":428,"context_line":"The **edp-spark** example bundled with Sahara contains a Spark program for estimating Pi."},{"line_number":429,"context_line":""},{"line_number":430,"context_line":""},{"line_number":431,"context_line":"Special Sahara URLs"},{"line_number":432,"context_line":"--------------------"},{"line_number":433,"context_line":""},{"line_number":434,"context_line":"Sahara uses custom URLs to refer to objects stored in Swift or the Sahara internal database. These URLs are not meant to be used"}],"source_content_type":"text/x-rst","patch_set":1,"id":"da20952f_e2d838f1","line":431,"range":{"start_line":431,"start_character":8,"end_line":431,"end_character":9},"updated":"2015-09-07 13:48:46.000000000","message":"s","commit_id":"52aed8adcbcbfc31f328158e85ba07acf5b35cc8"},{"author":{"_account_id":13919,"name":"Evgeny Sikachev","email":"esikachov@gmail.com","username":"esikachev"},"change_message_id":"2ccb889c007867f60b5da4ae98a1b9f2211bc9f1","unresolved":false,"context_lines":[{"line_number":431,"context_line":"Special Sahara URLs"},{"line_number":432,"context_line":"--------------------"},{"line_number":433,"context_line":""},{"line_number":434,"context_line":"Sahara uses custom URLs to refer to objects stored in Swift or the Sahara internal database. These URLs are not meant to be used"},{"line_number":435,"context_line":"outside of Sahara."},{"line_number":436,"context_line":""},{"line_number":437,"context_line":"Sahara Swift URLs passed to running jobs as input or output sources include a \".sahara\" suffix on the container, for example:"}],"source_content_type":"text/x-rst","patch_set":1,"id":"da20952f_62cc2835","line":434,"range":{"start_line":434,"start_character":67,"end_line":434,"end_character":68},"updated":"2015-09-07 13:48:46.000000000","message":"s","commit_id":"52aed8adcbcbfc31f328158e85ba07acf5b35cc8"},{"author":{"_account_id":13919,"name":"Evgeny Sikachev","email":"esikachov@gmail.com","username":"esikachev"},"change_message_id":"2ccb889c007867f60b5da4ae98a1b9f2211bc9f1","unresolved":false,"context_lines":[{"line_number":453,"context_line":"EDP Requirements"},{"line_number":454,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":455,"context_line":""},{"line_number":456,"context_line":"The OpenStack installation and the cluster launched from Sahara must meet the following minimum requirements in order for EDP to function:"},{"line_number":457,"context_line":""},{"line_number":458,"context_line":"OpenStack Services"},{"line_number":459,"context_line":"------------------"}],"source_content_type":"text/x-rst","patch_set":1,"id":"da20952f_824cdcc8","line":456,"range":{"start_line":456,"start_character":57,"end_line":456,"end_character":58},"updated":"2015-09-07 13:48:46.000000000","message":"s","commit_id":"52aed8adcbcbfc31f328158e85ba07acf5b35cc8"},{"author":{"_account_id":13919,"name":"Evgeny Sikachev","email":"esikachov@gmail.com","username":"esikachev"},"change_message_id":"2ccb889c007867f60b5da4ae98a1b9f2211bc9f1","unresolved":false,"context_lines":[{"line_number":475,"context_line":"-----------------"},{"line_number":476,"context_line":""},{"line_number":477,"context_line":"Requirements for EDP support depend on the EDP job type and plugin used for the cluster."},{"line_number":478,"context_line":"For example a Vanilla Sahara cluster must run at least one instance of these processes"},{"line_number":479,"context_line":"to support EDP:"},{"line_number":480,"context_line":""},{"line_number":481,"context_line":"* For Hadoop version 1:"}],"source_content_type":"text/x-rst","patch_set":1,"id":"da20952f_e2511863","line":478,"range":{"start_line":478,"start_character":22,"end_line":478,"end_character":23},"updated":"2015-09-07 13:48:46.000000000","message":"s","commit_id":"52aed8adcbcbfc31f328158e85ba07acf5b35cc8"},{"author":{"_account_id":13919,"name":"Evgeny Sikachev","email":"esikachov@gmail.com","username":"esikachev"},"change_message_id":"2ccb889c007867f60b5da4ae98a1b9f2211bc9f1","unresolved":false,"context_lines":[{"line_number":511,"context_line":""},{"line_number":512,"context_line":"Two config parameters control the behaviour of periodic clusters:"},{"line_number":513,"context_line":""},{"line_number":514,"context_line":" * periodic_enable - if set to \u0027False\u0027, Sahara will do nothing to a transient"},{"line_number":515,"context_line":"   cluster once the job it was created for is completed. If it is set to"},{"line_number":516,"context_line":"   \u0027True\u0027, then the behaviour depends on the value of the next parameter."},{"line_number":517,"context_line":" * use_identity_api_v3 - set it to \u0027False\u0027 if your OpenStack installation"}],"source_content_type":"text/x-rst","patch_set":1,"id":"da20952f_225b9087","line":514,"range":{"start_line":514,"start_character":40,"end_line":514,"end_character":41},"updated":"2015-09-07 13:48:46.000000000","message":"s","commit_id":"52aed8adcbcbfc31f328158e85ba07acf5b35cc8"},{"author":{"_account_id":10670,"name":"Michael McCune","email":"elmiko@redhat.com","username":"mimccune"},"change_message_id":"06dc90f4d2eb1e798bb81aa835ff9d07f39f2f3d","unresolved":false,"context_lines":[{"line_number":291,"context_line":"      | Output Path   | args         | 2                | data_source | false    | hdfs://ip:port/path |"},{"line_number":292,"context_line":"      +---------------+--------------+------------------+-------------+----------+---------------------+"},{"line_number":293,"context_line":"      | Mapper Count  | configs      | mapred.map.tasks | number      | false    | ``unset``           |"},{"line_number":294,"context_line":"      +---------------+--------------+------------------+-------------+----------+---------------------+"},{"line_number":295,"context_line":""},{"line_number":296,"context_line":"A \"Description\" field may also be added to each interface argument."},{"line_number":297,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"da20952f_dfaa78d2","line":294,"updated":"2015-09-08 19:43:42.000000000","message":"can we show an example of how to create the interface map in json?\n\ni\u0027m just having trouble putting together the details here with the explanation later for how to create this with mapping types, location, etc.","commit_id":"d8eaae08155224112b6d8baf4b23501f99c1c914"},{"author":{"_account_id":13953,"name":"Elise Gafford","email":"egafford@redhat.com","username":"egafford"},"change_message_id":"060999a8a283d26b0c0aaf7710e2739b7b9ae789","unresolved":false,"context_lines":[{"line_number":291,"context_line":"      | Output Path   | args         | 2                | data_source | false    | hdfs://ip:port/path |"},{"line_number":292,"context_line":"      +---------------+--------------+------------------+-------------+----------+---------------------+"},{"line_number":293,"context_line":"      | Mapper Count  | configs      | mapred.map.tasks | number      | false    | ``unset``           |"},{"line_number":294,"context_line":"      +---------------+--------------+------------------+-------------+----------+---------------------+"},{"line_number":295,"context_line":""},{"line_number":296,"context_line":"A \"Description\" field may also be added to each interface argument."},{"line_number":297,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"da20952f_bf7a2424","line":294,"in_reply_to":"da20952f_dfaa78d2","updated":"2015-09-08 19:54:09.000000000","message":"Sure; I\u0027ll add an example.","commit_id":"d8eaae08155224112b6d8baf4b23501f99c1c914"},{"author":{"_account_id":10670,"name":"Michael McCune","email":"elmiko@redhat.com","username":"mimccune"},"change_message_id":"06dc90f4d2eb1e798bb81aa835ff9d07f39f2f3d","unresolved":false,"context_lines":[{"line_number":295,"context_line":""},{"line_number":296,"context_line":"A \"Description\" field may also be added to each interface argument."},{"line_number":297,"context_line":""},{"line_number":298,"context_line":"Creating this interface would allow you to specify configuration for any"},{"line_number":299,"context_line":"execution of the job template by passing an \"interface\" map similar to:"},{"line_number":300,"context_line":""},{"line_number":301,"context_line":".. sourcecode:: json"}],"source_content_type":"text/x-rst","patch_set":2,"id":"da20952f_4410ddab","line":298,"updated":"2015-09-08 19:43:42.000000000","message":"s/specify configuration/specify a configuration/","commit_id":"d8eaae08155224112b6d8baf4b23501f99c1c914"},{"author":{"_account_id":10670,"name":"Michael McCune","email":"elmiko@redhat.com","username":"mimccune"},"change_message_id":"06dc90f4d2eb1e798bb81aa835ff9d07f39f2f3d","unresolved":false,"context_lines":[{"line_number":304,"context_line":"        \"Rows\": \"1000000\","},{"line_number":305,"context_line":"        \"Mapper Count\": \"3\","},{"line_number":306,"context_line":"        \"Output Path\": \"hdfs://mycluster:8020/user/myuser/teragen-output\""},{"line_number":307,"context_line":"    }"},{"line_number":308,"context_line":""},{"line_number":309,"context_line":"The specified arguments would be automatically placed into the args, configs,"},{"line_number":310,"context_line":"and params for the job, according to the mapping type and location fields of"}],"source_content_type":"text/x-rst","patch_set":2,"id":"da20952f_a453f938","line":307,"updated":"2015-09-08 19:43:42.000000000","message":"should we include \"Example Class\" in this representation?","commit_id":"d8eaae08155224112b6d8baf4b23501f99c1c914"},{"author":{"_account_id":13953,"name":"Elise Gafford","email":"egafford@redhat.com","username":"egafford"},"change_message_id":"060999a8a283d26b0c0aaf7710e2739b7b9ae789","unresolved":false,"context_lines":[{"line_number":304,"context_line":"        \"Rows\": \"1000000\","},{"line_number":305,"context_line":"        \"Mapper Count\": \"3\","},{"line_number":306,"context_line":"        \"Output Path\": \"hdfs://mycluster:8020/user/myuser/teragen-output\""},{"line_number":307,"context_line":"    }"},{"line_number":308,"context_line":""},{"line_number":309,"context_line":"The specified arguments would be automatically placed into the args, configs,"},{"line_number":310,"context_line":"and params for the job, according to the mapping type and location fields of"}],"source_content_type":"text/x-rst","patch_set":2,"id":"da20952f_bf93045c","line":307,"in_reply_to":"da20952f_a453f938","updated":"2015-09-08 19:54:09.000000000","message":"I actually specifically wanted not to include that one, to suggest that one use of interface args is to specify configurations that are effectively constant (like Java classes usually will be.)","commit_id":"d8eaae08155224112b6d8baf4b23501f99c1c914"}]}