)]}'
{"conf.d/spark.yaml.example":[{"author":{"_account_id":11809,"name":"Craig Bryant","email":"craig.bryant@hpe.com","username":"craig-bryant"},"change_message_id":"e47345e14db3a7a0c7ea424ec01676d5e5508557","unresolved":false,"context_lines":[{"line_number":2,"context_line":""},{"line_number":3,"context_line":"instances:"},{"line_number":4,"context_line":"  - history_server: http://spark-history.mycluster.com.br:18080/api/v1"},{"line_number":5,"context_line":"  # min_date: 2016-12-24T12:30:01"}],"source_content_type":"application/octet-stream","patch_set":6,"id":"3a98d1f4_25ad0d9b","line":5,"range":{"start_line":5,"start_character":4,"end_line":5,"end_character":12},"updated":"2016-10-03 22:31:51.000000000","message":"Not sure what this is for?\n\nYou should add an entry for this plugin in docs/Plugins.md to explain the options","commit_id":"8bcd14d11b183f34aef0ba612c9151f900e3960f"}],"docs/Plugins.md":[{"author":{"_account_id":11809,"name":"Craig Bryant","email":"craig.bryant@hpe.com","username":"craig-bryant"},"change_message_id":"821c43c89ef17a36deeffd0d147cdacea7f77ea3","unresolved":false,"context_lines":[{"line_number":66,"context_line":"  - [RedisDB](#redisdb)"},{"line_number":67,"context_line":"  - [Riak](#riak)"},{"line_number":68,"context_line":"  - [SQLServer](#sqlserver)"},{"line_number":69,"context_line":"  - [SolidFire](#solidfire)"},{"line_number":70,"context_line":"  - [Spark](#spark)"},{"line_number":71,"context_line":"  - [Supervisord](#supervisord)"},{"line_number":72,"context_line":"  - [Swift Diags](#swift-diags)"}],"source_content_type":"text/x-markdown","patch_set":16,"id":"ba5201f7_b63d37bd","line":69,"range":{"start_line":69,"start_character":0,"end_line":69,"end_character":27},"updated":"2017-01-04 16:33:21.000000000","message":"Why did you move this out of alphabetical order?","commit_id":"2abc1510768266a830259a1ac1474cf88b9e48de"},{"author":{"_account_id":11809,"name":"Craig Bryant","email":"craig.bryant@hpe.com","username":"craig-bryant"},"change_message_id":"821c43c89ef17a36deeffd0d147cdacea7f77ea3","unresolved":false,"context_lines":[{"line_number":1450,"context_line":"| solidfire.data_reduction_factor     | service\u003dsolidfire, cluster | Aggregate data reduction efficiency, thin_prov * dedup * compression |"},{"line_number":1451,"context_line":""},{"line_number":1452,"context_line":"## Spark"},{"line_number":1453,"context_line":"This plugin provides metrics for Spark history server. It consumes [Spark history server API](https://spark.apache.org/docs/latest/monitoring.html) retriving applications since last Agent check, or using `min_date` in configuration file. It executes `GET` requests on `http://\u003cserver-url\u003e:18080/api/v1`."},{"line_number":1454,"context_line":""},{"line_number":1455,"context_line":"### Sample config"},{"line_number":1456,"context_line":"```"}],"source_content_type":"text/x-markdown","patch_set":16,"id":"ba5201f7_b66457e5","line":1453,"range":{"start_line":1453,"start_character":148,"end_line":1453,"end_character":157},"updated":"2017-01-04 16:33:21.000000000","message":"spelling","commit_id":"2abc1510768266a830259a1ac1474cf88b9e48de"},{"author":{"_account_id":11809,"name":"Craig Bryant","email":"craig.bryant@hpe.com","username":"craig-bryant"},"change_message_id":"821c43c89ef17a36deeffd0d147cdacea7f77ea3","unresolved":false,"context_lines":[{"line_number":1462,"context_line":"    min_date: 2016-12-31T12:00 #Uses minData parameter on API"},{"line_number":1463,"context_line":"```"},{"line_number":1464,"context_line":""},{"line_number":1465,"context_line":"The Spark history server check return the following metrics"},{"line_number":1466,"context_line":""},{"line_number":1467,"context_line":"| Metric Name                                       | Dimensions                                                     | Semantics                                                                                                     |"},{"line_number":1468,"context_line":"| -----------                                       | ----------                                                     | ---------                                                                                                     |"}],"source_content_type":"text/x-markdown","patch_set":16,"id":"ba5201f7_f65edf10","line":1465,"range":{"start_line":1465,"start_character":31,"end_line":1465,"end_character":37},"updated":"2017-01-04 16:33:21.000000000","message":"returns","commit_id":"2abc1510768266a830259a1ac1474cf88b9e48de"}],"monasca_agent/collector/checks_d/spark.py":[{"author":{"_account_id":11809,"name":"Craig Bryant","email":"craig.bryant@hpe.com","username":"craig-bryant"},"change_message_id":"eca78b05919426fb84f7a18d65966f88dd0897ae","unresolved":false,"context_lines":[{"line_number":9,"context_line":""},{"line_number":10,"context_line":""},{"line_number":11,"context_line":"METRICS \u003d {"},{"line_number":12,"context_line":"\t\u0027applications\u0027: { },"},{"line_number":13,"context_line":"\t\u0027jobs\u0027: {"},{"line_number":14,"context_line":"\t    \"name\" : \"saveAsTextFile at Eclat.scala:47\","},{"line_number":15,"context_line":"\t    \"submissionTime\" : \"2015-11-10T01:45:53.677GMT\","},{"line_number":16,"context_line":"\t    \"completionTime\" : \"2015-11-10T01:54:26.040GMT\","},{"line_number":17,"context_line":"\t    \"status\" : \"FAILED\","},{"line_number":18,"context_line":"\t    \"numTasks\" : 4096,"},{"line_number":19,"context_line":"\t    \"numActiveTasks\" : 72,"},{"line_number":20,"context_line":"\t    \"numCompletedTasks\" : 633,"},{"line_number":21,"context_line":"\t    \"numSkippedTasks\" : 633,"},{"line_number":22,"context_line":"\t    \"numFailedTasks\" : 0,"},{"line_number":23,"context_line":"\t    \"numActiveStages\" : 0,"},{"line_number":24,"context_line":"\t    \"numCompletedStages\" : 0,"},{"line_number":25,"context_line":"\t    \"numSkippedStages\" : 1,"},{"line_number":26,"context_line":"\t    \"numFailedStages\" : 1"},{"line_number":27,"context_line":"\t},"},{"line_number":28,"context_line":"\t\u0027stages\u0027: {"},{"line_number":29,"context_line":"\t    \"status\" : \"COMPLETE\","},{"line_number":30,"context_line":"\t    \"numActiveTasks\" : 0,"},{"line_number":31,"context_line":"\t    \"numCompleteTasks\" : 2048,"},{"line_number":32,"context_line":"\t    \"numFailedTasks\" : 0,"},{"line_number":33,"context_line":"\t    \"executorRunTime\" : 1162600,"},{"line_number":34,"context_line":"\t    \"inputBytes\" : 0,"},{"line_number":35,"context_line":"\t    \"inputRecords\" : 0,"},{"line_number":36,"context_line":"\t    \"outputBytes\" : 0,"},{"line_number":37,"context_line":"\t    \"outputRecords\" : 0,"},{"line_number":38,"context_line":"\t    \"shuffleReadBytes\" : 1694241535,"},{"line_number":39,"context_line":"\t    \"shuffleReadRecords\" : 334012466,"},{"line_number":40,"context_line":"\t    \"shuffleWriteBytes\" : 0,"},{"line_number":41,"context_line":"\t    \"shuffleWriteRecords\" : 0,"},{"line_number":42,"context_line":"\t    \"memoryBytesSpilled\" : 0,"},{"line_number":43,"context_line":"\t    \"diskBytesSpilled\" : 0,"},{"line_number":44,"context_line":"\t},"},{"line_number":45,"context_line":"\t\u0027executors\u0027: {"},{"line_number":46,"context_line":"\t    \"id\" : \"3\","},{"line_number":47,"context_line":"\t    \"hostPort\" : \"phocus6.lvad.speed.dcc.ufmg.br:57542\","},{"line_number":48,"context_line":"\t    \"rddBlocks\" : 173,"},{"line_number":49,"context_line":"\t    \"memoryUsed\" : 5742522541,"},{"line_number":50,"context_line":"\t    \"diskUsed\" : 767937430,"},{"line_number":51,"context_line":"\t    \"activeTasks\" : 7,"},{"line_number":52,"context_line":"\t    \"failedTasks\" : 0,"},{"line_number":53,"context_line":"\t    \"completedTasks\" : 978,"},{"line_number":54,"context_line":"\t    \"totalTasks\" : 985,"},{"line_number":55,"context_line":"\t    \"totalDuration\" : 4630287,"},{"line_number":56,"context_line":"\t    \"totalInputBytes\" : 1691145317,"},{"line_number":57,"context_line":"\t    \"totalShuffleRead\" : 337939892,"},{"line_number":58,"context_line":"\t    \"totalShuffleWrite\" : 1047933195,"},{"line_number":59,"context_line":"\t    \"maxMemory\" : 7779731374,"},{"line_number":60,"context_line":"\t    \"executorLogs\" : {"},{"line_number":61,"context_line":"\t\t\"stdout\" :"},{"line_number":62,"context_line":"\t\t\"http://phocus6.lvad.speed.dcc.ufmg.br:8081/logPage/?appId\u003dapp-20151109234335-0000\u0026executorId\u003d3\u0026logType\u003dstdout\","},{"line_number":63,"context_line":"\t\t\"stderr\" :"},{"line_number":64,"context_line":"\t\t\"http://phocus6.lvad.speed.dcc.ufmg.br:8081/logPage/?appId\u003dapp-20151109234335-0000\u0026executorId\u003d3\u0026logType\u003dstderr\""},{"line_number":65,"context_line":""},{"line_number":66,"context_line":"\t\t}"},{"line_number":67,"context_line":"\t},"},{"line_number":68,"context_line":"\t\u0027storages\u0027: {"},{"line_number":69,"context_line":"\t\t\"id\" : 2,"},{"line_number":70,"context_line":"\t\t\"name\" : \"transactions_rdd\","},{"line_number":71,"context_line":"\t\t\"numPartitions\" : 2048,"},{"line_number":72,"context_line":"\t\t\"numCachedPartitions\" : 1471,"},{"line_number":73,"context_line":"\t\t\"storageLevel\" : \"Memory Serialized 1x Replicated\","},{"line_number":74,"context_line":"\t\t\"memoryUsed\" : 3764947734,"},{"line_number":75,"context_line":"\t\t\"diskUsed\" : 0"},{"line_number":76,"context_line":"\t}"},{"line_number":77,"context_line":"}"},{"line_number":78,"context_line":""},{"line_number":79,"context_line":"class Spark(AgentCheck):"}],"source_content_type":"text/x-python","patch_set":3,"id":"3a98d1f4_df0b9bd4","line":76,"range":{"start_line":12,"start_character":0,"end_line":76,"end_character":2},"updated":"2016-09-30 21:24:23.000000000","message":"Replace the tabs with spaces","commit_id":"176f57d2e17744ea589ad019b6a15b8ebfbba961"},{"author":{"_account_id":5559,"name":"gui maluf balzana","display_name":"Gui Maluf Balzana","email":"guilherme.balzana@canonical.com","username":"guimalufb"},"change_message_id":"7ba2083036ce6421468c65d50818d2776f5d41cc","unresolved":false,"context_lines":[{"line_number":9,"context_line":""},{"line_number":10,"context_line":""},{"line_number":11,"context_line":"METRICS \u003d {"},{"line_number":12,"context_line":"\t\u0027applications\u0027: { },"},{"line_number":13,"context_line":"\t\u0027jobs\u0027: {"},{"line_number":14,"context_line":"\t    \"name\" : \"saveAsTextFile at Eclat.scala:47\","},{"line_number":15,"context_line":"\t    \"submissionTime\" : \"2015-11-10T01:45:53.677GMT\","},{"line_number":16,"context_line":"\t    \"completionTime\" : \"2015-11-10T01:54:26.040GMT\","},{"line_number":17,"context_line":"\t    \"status\" : \"FAILED\","},{"line_number":18,"context_line":"\t    \"numTasks\" : 4096,"},{"line_number":19,"context_line":"\t    \"numActiveTasks\" : 72,"},{"line_number":20,"context_line":"\t    \"numCompletedTasks\" : 633,"},{"line_number":21,"context_line":"\t    \"numSkippedTasks\" : 633,"},{"line_number":22,"context_line":"\t    \"numFailedTasks\" : 0,"},{"line_number":23,"context_line":"\t    \"numActiveStages\" : 0,"},{"line_number":24,"context_line":"\t    \"numCompletedStages\" : 0,"},{"line_number":25,"context_line":"\t    \"numSkippedStages\" : 1,"},{"line_number":26,"context_line":"\t    \"numFailedStages\" : 1"},{"line_number":27,"context_line":"\t},"},{"line_number":28,"context_line":"\t\u0027stages\u0027: {"},{"line_number":29,"context_line":"\t    \"status\" : \"COMPLETE\","},{"line_number":30,"context_line":"\t    \"numActiveTasks\" : 0,"},{"line_number":31,"context_line":"\t    \"numCompleteTasks\" : 2048,"},{"line_number":32,"context_line":"\t    \"numFailedTasks\" : 0,"},{"line_number":33,"context_line":"\t    \"executorRunTime\" : 1162600,"},{"line_number":34,"context_line":"\t    \"inputBytes\" : 0,"},{"line_number":35,"context_line":"\t    \"inputRecords\" : 0,"},{"line_number":36,"context_line":"\t    \"outputBytes\" : 0,"},{"line_number":37,"context_line":"\t    \"outputRecords\" : 0,"},{"line_number":38,"context_line":"\t    \"shuffleReadBytes\" : 1694241535,"},{"line_number":39,"context_line":"\t    \"shuffleReadRecords\" : 334012466,"},{"line_number":40,"context_line":"\t    \"shuffleWriteBytes\" : 0,"},{"line_number":41,"context_line":"\t    \"shuffleWriteRecords\" : 0,"},{"line_number":42,"context_line":"\t    \"memoryBytesSpilled\" : 0,"},{"line_number":43,"context_line":"\t    \"diskBytesSpilled\" : 0,"},{"line_number":44,"context_line":"\t},"},{"line_number":45,"context_line":"\t\u0027executors\u0027: {"},{"line_number":46,"context_line":"\t    \"id\" : \"3\","},{"line_number":47,"context_line":"\t    \"hostPort\" : \"phocus6.lvad.speed.dcc.ufmg.br:57542\","},{"line_number":48,"context_line":"\t    \"rddBlocks\" : 173,"},{"line_number":49,"context_line":"\t    \"memoryUsed\" : 5742522541,"},{"line_number":50,"context_line":"\t    \"diskUsed\" : 767937430,"},{"line_number":51,"context_line":"\t    \"activeTasks\" : 7,"},{"line_number":52,"context_line":"\t    \"failedTasks\" : 0,"},{"line_number":53,"context_line":"\t    \"completedTasks\" : 978,"},{"line_number":54,"context_line":"\t    \"totalTasks\" : 985,"},{"line_number":55,"context_line":"\t    \"totalDuration\" : 4630287,"},{"line_number":56,"context_line":"\t    \"totalInputBytes\" : 1691145317,"},{"line_number":57,"context_line":"\t    \"totalShuffleRead\" : 337939892,"},{"line_number":58,"context_line":"\t    \"totalShuffleWrite\" : 1047933195,"},{"line_number":59,"context_line":"\t    \"maxMemory\" : 7779731374,"},{"line_number":60,"context_line":"\t    \"executorLogs\" : {"},{"line_number":61,"context_line":"\t\t\"stdout\" :"},{"line_number":62,"context_line":"\t\t\"http://phocus6.lvad.speed.dcc.ufmg.br:8081/logPage/?appId\u003dapp-20151109234335-0000\u0026executorId\u003d3\u0026logType\u003dstdout\","},{"line_number":63,"context_line":"\t\t\"stderr\" :"},{"line_number":64,"context_line":"\t\t\"http://phocus6.lvad.speed.dcc.ufmg.br:8081/logPage/?appId\u003dapp-20151109234335-0000\u0026executorId\u003d3\u0026logType\u003dstderr\""},{"line_number":65,"context_line":""},{"line_number":66,"context_line":"\t\t}"},{"line_number":67,"context_line":"\t},"},{"line_number":68,"context_line":"\t\u0027storages\u0027: {"},{"line_number":69,"context_line":"\t\t\"id\" : 2,"},{"line_number":70,"context_line":"\t\t\"name\" : \"transactions_rdd\","},{"line_number":71,"context_line":"\t\t\"numPartitions\" : 2048,"},{"line_number":72,"context_line":"\t\t\"numCachedPartitions\" : 1471,"},{"line_number":73,"context_line":"\t\t\"storageLevel\" : \"Memory Serialized 1x Replicated\","},{"line_number":74,"context_line":"\t\t\"memoryUsed\" : 3764947734,"},{"line_number":75,"context_line":"\t\t\"diskUsed\" : 0"},{"line_number":76,"context_line":"\t}"},{"line_number":77,"context_line":"}"},{"line_number":78,"context_line":""},{"line_number":79,"context_line":"class Spark(AgentCheck):"}],"source_content_type":"text/x-python","patch_set":3,"id":"3a98d1f4_4a150b3e","line":76,"range":{"start_line":12,"start_character":0,"end_line":76,"end_character":2},"in_reply_to":"3a98d1f4_df0b9bd4","updated":"2016-09-30 21:49:35.000000000","message":"this is just trash... :/ I\u0027m still thinking how to get/translate metrics to monasca/influxdb/grafana name","commit_id":"176f57d2e17744ea589ad019b6a15b8ebfbba961"},{"author":{"_account_id":11809,"name":"Craig Bryant","email":"craig.bryant@hpe.com","username":"craig-bryant"},"change_message_id":"eca78b05919426fb84f7a18d65966f88dd0897ae","unresolved":false,"context_lines":[{"line_number":113,"context_line":"                       duration,"},{"line_number":114,"context_line":"                       dimensions\u003ddimensions,"},{"line_number":115,"context_line":"                       timestamp\u003dstartTime,"},{"line_number":116,"context_line":"                       value_meta\u003d{\u0027app-id\u0027: app[\u0027id\u0027]})"},{"line_number":117,"context_line":""},{"line_number":118,"context_line":"            for job in app[\u0027jobs\u0027]:"},{"line_number":119,"context_line":"                duration \u003d self._duration_time(job[\u0027submissionTime\u0027], job[\u0027completionTime\u0027])"}],"source_content_type":"text/x-python","patch_set":3,"id":"3a98d1f4_1fe0a3f9","line":116,"range":{"start_line":116,"start_character":45,"end_line":116,"end_character":54},"updated":"2016-09-30 21:24:23.000000000","message":"What does this value look like? Does it vary with each restart of the app?","commit_id":"176f57d2e17744ea589ad019b6a15b8ebfbba961"},{"author":{"_account_id":5559,"name":"gui maluf balzana","display_name":"Gui Maluf Balzana","email":"guilherme.balzana@canonical.com","username":"guimalufb"},"change_message_id":"7ba2083036ce6421468c65d50818d2776f5d41cc","unresolved":false,"context_lines":[{"line_number":113,"context_line":"                       duration,"},{"line_number":114,"context_line":"                       dimensions\u003ddimensions,"},{"line_number":115,"context_line":"                       timestamp\u003dstartTime,"},{"line_number":116,"context_line":"                       value_meta\u003d{\u0027app-id\u0027: app[\u0027id\u0027]})"},{"line_number":117,"context_line":""},{"line_number":118,"context_line":"            for job in app[\u0027jobs\u0027]:"},{"line_number":119,"context_line":"                duration \u003d self._duration_time(job[\u0027submissionTime\u0027], job[\u0027completionTime\u0027])"}],"source_content_type":"text/x-python","patch_set":3,"id":"3a98d1f4_ea889773","line":116,"range":{"start_line":116,"start_character":45,"end_line":116,"end_character":54},"in_reply_to":"3a98d1f4_1fe0a3f9","updated":"2016-09-30 21:49:35.000000000","message":"I\u0027m really appreciating your comments! :D\n\nThis is the identifier of each run on spark\nI think will fit better as a dimension instead of value_meta.\n\nCause I want to be able to filter data by correlated applications","commit_id":"176f57d2e17744ea589ad019b6a15b8ebfbba961"},{"author":{"_account_id":11809,"name":"Craig Bryant","email":"craig.bryant@hpe.com","username":"craig-bryant"},"change_message_id":"e47345e14db3a7a0c7ea424ec01676d5e5508557","unresolved":false,"context_lines":[{"line_number":65,"context_line":"    \"\"\"Extract metrics from Spark monitor REST API\"\"\""},{"line_number":66,"context_line":""},{"line_number":67,"context_line":"    def check(self, instance):"},{"line_number":68,"context_line":"        if  self.instance_count() \u003e 0:"},{"line_number":69,"context_line":"            config_min_date \u003d instance.get(\u0027min_date\u0027, None)"},{"line_number":70,"context_line":"            config_history_server \u003d instance.get(\u0027history_server\u0027, None)"},{"line_number":71,"context_line":"            if config_history_server is None:"}],"source_content_type":"text/x-python","patch_set":6,"id":"3a98d1f4_85bd0104","line":68,"range":{"start_line":68,"start_character":10,"end_line":68,"end_character":12},"updated":"2016-10-03 22:31:51.000000000","message":"You have two spaces here","commit_id":"8bcd14d11b183f34aef0ba612c9151f900e3960f"},{"author":{"_account_id":11809,"name":"Craig Bryant","email":"craig.bryant@hpe.com","username":"craig-bryant"},"change_message_id":"e47345e14db3a7a0c7ea424ec01676d5e5508557","unresolved":false,"context_lines":[{"line_number":71,"context_line":"            if config_history_server is None:"},{"line_number":72,"context_line":"                raise Exception(\u0027A Spark history_server must be specified\u0027)"},{"line_number":73,"context_line":""},{"line_number":74,"context_line":"        if config_min_date is None:"},{"line_number":75,"context_line":"            # Plugin config check_freq should have precedence over AgentConfig"},{"line_number":76,"context_line":"            check_freq \u003d self.init_config.get(\u0027check_freq\u0027,"},{"line_number":77,"context_line":"                                              self.agent_config.get(\u0027check_freq\u0027, 0))"}],"source_content_type":"text/x-python","patch_set":6,"id":"3a98d1f4_a54f5d32","line":74,"range":{"start_line":74,"start_character":11,"end_line":74,"end_character":26},"updated":"2016-10-03 22:31:51.000000000","message":"config_min_date will not be defined at this point if self.instance_count() is 0","commit_id":"8bcd14d11b183f34aef0ba612c9151f900e3960f"},{"author":{"_account_id":11809,"name":"Craig Bryant","email":"craig.bryant@hpe.com","username":"craig-bryant"},"change_message_id":"e47345e14db3a7a0c7ea424ec01676d5e5508557","unresolved":false,"context_lines":[{"line_number":81,"context_line":"            last_check \u003d config_min_date"},{"line_number":82,"context_line":""},{"line_number":83,"context_line":"        self.log.info(\"Fetching data since %s\" % last_check)"},{"line_number":84,"context_line":"        dimensions \u003d self._set_dimensions({\u0027component\u0027: \u0027spark-history\u0027,"},{"line_number":85,"context_line":"                                          \u0027service\u0027: \u0027spark\u0027},"},{"line_number":86,"context_line":"                                          instance)"},{"line_number":87,"context_line":""}],"source_content_type":"text/x-python","patch_set":6,"id":"3a98d1f4_05981169","line":84,"range":{"start_line":84,"start_character":26,"end_line":84,"end_character":41},"updated":"2016-10-03 22:31:51.000000000","message":"If you do allow multiple instances, then you should add a dimension for history_server so the user can separate the metrics for each history_server","commit_id":"8bcd14d11b183f34aef0ba612c9151f900e3960f"},{"author":{"_account_id":11809,"name":"Craig Bryant","email":"craig.bryant@hpe.com","username":"craig-bryant"},"change_message_id":"e47345e14db3a7a0c7ea424ec01676d5e5508557","unresolved":false,"context_lines":[{"line_number":104,"context_line":"                       value_meta\u003dvalue_meta)"},{"line_number":105,"context_line":""},{"line_number":106,"context_line":"            for job in app[\u0027jobs\u0027]:"},{"line_number":107,"context_line":"                value_meta.update({\u0027job-id\u0027: job[\u0027jobId\u0027]})"},{"line_number":108,"context_line":""},{"line_number":109,"context_line":"                duration \u003d self._duration_time(job[\u0027submissionTime\u0027], job[\u0027completionTime\u0027])"},{"line_number":110,"context_line":"                job_submission_time \u003d self._unix_timestamp(job[\u0027submissionTime\u0027])"}],"source_content_type":"text/x-python","patch_set":6,"id":"3a98d1f4_e57ec5e2","line":107,"range":{"start_line":107,"start_character":34,"end_line":107,"end_character":58},"updated":"2016-10-03 22:31:51.000000000","message":"This is probably not a good use of value_meta. How is the user going to be able to find the metrics associated with a particular job-id? You can\u0027t search on value_meta.\n\nYou also need to think about the value of each metric. This looks like it could create lots and lots of metrics and measurements. Are they really valuable? They do take up space in the metrics database\n\nWould some summary metrics be better?","commit_id":"8bcd14d11b183f34aef0ba612c9151f900e3960f"},{"author":{"_account_id":11809,"name":"Craig Bryant","email":"craig.bryant@hpe.com","username":"craig-bryant"},"change_message_id":"e47345e14db3a7a0c7ea424ec01676d5e5508557","unresolved":false,"context_lines":[{"line_number":145,"context_line":"        return applications"},{"line_number":146,"context_line":""},{"line_number":147,"context_line":"    def _get_api_request(self, url, path\u003d\u0027\u0027, params\u003d{}):"},{"line_number":148,"context_line":"        response \u003d requests.get(url\u003durl + path, params\u003dparams)"},{"line_number":149,"context_line":"        if not response.ok:"},{"line_number":150,"context_line":"            return {}"},{"line_number":151,"context_line":"        return response.json()"}],"source_content_type":"text/x-python","patch_set":6,"id":"3a98d1f4_255c2d7c","line":148,"range":{"start_line":148,"start_character":28,"end_line":148,"end_character":31},"updated":"2016-10-03 22:31:51.000000000","message":"What if the server is down? This will throw an exception, right? You need to handle that and not just let it bubble up to the collector","commit_id":"8bcd14d11b183f34aef0ba612c9151f900e3960f"},{"author":{"_account_id":16168,"name":"Tomasz Trębski","email":"kornicameister@gmail.com","username":"trebskit"},"change_message_id":"caf1d2abd3554ccd9e299a80623593be2118c665","unresolved":false,"context_lines":[{"line_number":107,"context_line":""},{"line_number":108,"context_line":"            for job in app[\u0027jobs\u0027]:"},{"line_number":109,"context_line":"                job_dimensions \u003d self._update_dimension(app_dimensions,"},{"line_number":110,"context_line":"                                                        {\u0027job_id\u0027: str(job[\u0027jobId\u0027])})"},{"line_number":111,"context_line":""},{"line_number":112,"context_line":"                duration \u003d self._duration_time(job[\u0027submissionTime\u0027], job[\u0027completionTime\u0027])"},{"line_number":113,"context_line":"                job_submission_time \u003d self._unix_timestamp(job[\u0027submissionTime\u0027])"}],"source_content_type":"text/x-python","patch_set":15,"id":"fa6399be_86aa4718","line":110,"updated":"2016-10-11 05:14:02.000000000","message":"Any details about jobId. Name suggests it can generate multiple distinct dimensions. Does every job run generates unique_id, what happens if spark gets restarted - the counter starts back from 0 ?","commit_id":"a2b281348b5903d6e9a54f6575aaf3a1eeadbfff"},{"author":{"_account_id":11809,"name":"Craig Bryant","email":"craig.bryant@hpe.com","username":"craig-bryant"},"change_message_id":"821c43c89ef17a36deeffd0d147cdacea7f77ea3","unresolved":false,"context_lines":[{"line_number":107,"context_line":""},{"line_number":108,"context_line":"            for job in app[\u0027jobs\u0027]:"},{"line_number":109,"context_line":"                job_dimensions \u003d self._update_dimension(app_dimensions,"},{"line_number":110,"context_line":"                                                        {\u0027job_id\u0027: str(job[\u0027jobId\u0027])})"},{"line_number":111,"context_line":""},{"line_number":112,"context_line":"                duration \u003d self._duration_time(job[\u0027submissionTime\u0027], job[\u0027completionTime\u0027])"},{"line_number":113,"context_line":"                job_submission_time \u003d self._unix_timestamp(job[\u0027submissionTime\u0027])"}],"source_content_type":"text/x-python","patch_set":16,"id":"ba5201f7_1619a388","line":110,"range":{"start_line":110,"start_character":67,"end_line":110,"end_character":84},"updated":"2017-01-04 16:33:21.000000000","message":"Tomasz already asked about jobid but you didn\u0027t answer it. If jobid is generated when spark gets restarted or the app is restarted using it as a dimension will make these metrics useless.","commit_id":"2abc1510768266a830259a1ac1474cf88b9e48de"},{"author":{"_account_id":5559,"name":"gui maluf balzana","display_name":"Gui Maluf Balzana","email":"guilherme.balzana@canonical.com","username":"guimalufb"},"change_message_id":"ac649575f1009d6d83856680dc69c7d35be29c23","unresolved":false,"context_lines":[{"line_number":107,"context_line":""},{"line_number":108,"context_line":"            for job in app[\u0027jobs\u0027]:"},{"line_number":109,"context_line":"                job_dimensions \u003d self._update_dimension(app_dimensions,"},{"line_number":110,"context_line":"                                                        {\u0027job_id\u0027: str(job[\u0027jobId\u0027])})"},{"line_number":111,"context_line":""},{"line_number":112,"context_line":"                duration \u003d self._duration_time(job[\u0027submissionTime\u0027], job[\u0027completionTime\u0027])"},{"line_number":113,"context_line":"                job_submission_time \u003d self._unix_timestamp(job[\u0027submissionTime\u0027])"}],"source_content_type":"text/x-python","patch_set":16,"id":"1a430d35_e3e4b60e","line":110,"range":{"start_line":110,"start_character":67,"end_line":110,"end_character":84},"in_reply_to":"1a430d35_83ca12a0","updated":"2017-02-03 18:37:17.000000000","message":"Done","commit_id":"2abc1510768266a830259a1ac1474cf88b9e48de"},{"author":{"_account_id":11809,"name":"Craig Bryant","email":"craig.bryant@hpe.com","username":"craig-bryant"},"change_message_id":"bb703c49b1a673f989933cacca4b74cc9b192f87","unresolved":false,"context_lines":[{"line_number":107,"context_line":""},{"line_number":108,"context_line":"            for job in app[\u0027jobs\u0027]:"},{"line_number":109,"context_line":"                job_dimensions \u003d self._update_dimension(app_dimensions,"},{"line_number":110,"context_line":"                                                        {\u0027job_id\u0027: str(job[\u0027jobId\u0027])})"},{"line_number":111,"context_line":""},{"line_number":112,"context_line":"                duration \u003d self._duration_time(job[\u0027submissionTime\u0027], job[\u0027completionTime\u0027])"},{"line_number":113,"context_line":"                job_submission_time \u003d self._unix_timestamp(job[\u0027submissionTime\u0027])"}],"source_content_type":"text/x-python","patch_set":16,"id":"1a430d35_83ca12a0","line":110,"range":{"start_line":110,"start_character":67,"end_line":110,"end_character":84},"in_reply_to":"1a430d35_8c2091f9","updated":"2017-02-03 18:28:21.000000000","message":"The dimensions are used for finding a metric, think about trying to graph it in grafana: what would you use to identify the correct metric to graph? So, the dimensions should be meaningful to the user and name is the better choice.","commit_id":"2abc1510768266a830259a1ac1474cf88b9e48de"},{"author":{"_account_id":5559,"name":"gui maluf balzana","display_name":"Gui Maluf Balzana","email":"guilherme.balzana@canonical.com","username":"guimalufb"},"change_message_id":"279229cb830dd3aa5ac9ade35d1c258dec8f104f","unresolved":false,"context_lines":[{"line_number":107,"context_line":""},{"line_number":108,"context_line":"            for job in app[\u0027jobs\u0027]:"},{"line_number":109,"context_line":"                job_dimensions \u003d self._update_dimension(app_dimensions,"},{"line_number":110,"context_line":"                                                        {\u0027job_id\u0027: str(job[\u0027jobId\u0027])})"},{"line_number":111,"context_line":""},{"line_number":112,"context_line":"                duration \u003d self._duration_time(job[\u0027submissionTime\u0027], job[\u0027completionTime\u0027])"},{"line_number":113,"context_line":"                job_submission_time \u003d self._unix_timestamp(job[\u0027submissionTime\u0027])"}],"source_content_type":"text/x-python","patch_set":16,"id":"1a430d35_8c2091f9","line":110,"range":{"start_line":110,"start_character":67,"end_line":110,"end_character":84},"in_reply_to":"9a57fde8_d48db37c","updated":"2017-02-03 12:35:48.000000000","message":"Sorry for taking so much time to reply. I\u0027m stuck with other tasks :/\n\nYou\u0027re right, `name` field is more meaningful but I tought dimensions should be an UUID-like field.\n\nI\u0027ll include or swap to `name` field.","commit_id":"2abc1510768266a830259a1ac1474cf88b9e48de"},{"author":{"_account_id":5559,"name":"gui maluf balzana","display_name":"Gui Maluf Balzana","email":"guilherme.balzana@canonical.com","username":"guimalufb"},"change_message_id":"f2b4fc35c0d66522e68e1a6bc71a5a93de001a14","unresolved":false,"context_lines":[{"line_number":107,"context_line":""},{"line_number":108,"context_line":"            for job in app[\u0027jobs\u0027]:"},{"line_number":109,"context_line":"                job_dimensions \u003d self._update_dimension(app_dimensions,"},{"line_number":110,"context_line":"                                                        {\u0027job_id\u0027: str(job[\u0027jobId\u0027])})"},{"line_number":111,"context_line":""},{"line_number":112,"context_line":"                duration \u003d self._duration_time(job[\u0027submissionTime\u0027], job[\u0027completionTime\u0027])"},{"line_number":113,"context_line":"                job_submission_time \u003d self._unix_timestamp(job[\u0027submissionTime\u0027])"}],"source_content_type":"text/x-python","patch_set":16,"id":"ba5201f7_828aadbd","line":110,"range":{"start_line":110,"start_character":67,"end_line":110,"end_character":84},"in_reply_to":"ba5201f7_1619a388","updated":"2017-01-04 17:51:59.000000000","message":"I\u0027d answered that point but it was lost as a draft comment.\n\nJobID is strictly related with app_id, since it\u0027s hierarchically organized.\n\n    App has many Jobs\n       Jobs has many Stages\n           Stages has many tasks\n\nI want to be able to retrieve data from a specific job group event this job been different among apps.","commit_id":"2abc1510768266a830259a1ac1474cf88b9e48de"},{"author":{"_account_id":5559,"name":"gui maluf balzana","display_name":"Gui Maluf Balzana","email":"guilherme.balzana@canonical.com","username":"guimalufb"},"change_message_id":"9c77001f967583a3695c801cd9ead0bced387dee","unresolved":false,"context_lines":[{"line_number":107,"context_line":""},{"line_number":108,"context_line":"            for job in app[\u0027jobs\u0027]:"},{"line_number":109,"context_line":"                job_dimensions \u003d self._update_dimension(app_dimensions,"},{"line_number":110,"context_line":"                                                        {\u0027job_id\u0027: str(job[\u0027jobId\u0027])})"},{"line_number":111,"context_line":""},{"line_number":112,"context_line":"                duration \u003d self._duration_time(job[\u0027submissionTime\u0027], job[\u0027completionTime\u0027])"},{"line_number":113,"context_line":"                job_submission_time \u003d self._unix_timestamp(job[\u0027submissionTime\u0027])"}],"source_content_type":"text/x-python","patch_set":16,"id":"ba5201f7_7f537fe2","line":110,"range":{"start_line":110,"start_character":67,"end_line":110,"end_character":84},"in_reply_to":"ba5201f7_1bc37c39","updated":"2017-01-05 13:05:50.000000000","message":"I\u0027ve included a fixtures data at\nhttps://review.openstack.org/#/c/377698/16/tests/checks_d/fixtures/test_spark.json\n\napp_id looks more like this: application_1473948406615_0046","commit_id":"2abc1510768266a830259a1ac1474cf88b9e48de"},{"author":{"_account_id":11809,"name":"Craig Bryant","email":"craig.bryant@hpe.com","username":"craig-bryant"},"change_message_id":"f7b0d959ec931449c18e0247ff828cc0a82d2073","unresolved":false,"context_lines":[{"line_number":107,"context_line":""},{"line_number":108,"context_line":"            for job in app[\u0027jobs\u0027]:"},{"line_number":109,"context_line":"                job_dimensions \u003d self._update_dimension(app_dimensions,"},{"line_number":110,"context_line":"                                                        {\u0027job_id\u0027: str(job[\u0027jobId\u0027])})"},{"line_number":111,"context_line":""},{"line_number":112,"context_line":"                duration \u003d self._duration_time(job[\u0027submissionTime\u0027], job[\u0027completionTime\u0027])"},{"line_number":113,"context_line":"                job_submission_time \u003d self._unix_timestamp(job[\u0027submissionTime\u0027])"}],"source_content_type":"text/x-python","patch_set":16,"id":"ba5201f7_1bc37c39","line":110,"range":{"start_line":110,"start_character":67,"end_line":110,"end_character":84},"in_reply_to":"ba5201f7_237664a4","updated":"2017-01-05 00:27:49.000000000","message":"Yes, that is clear about jobid. For app_id, does it really look like 100, 200? How is the user supposed to map 200 to the correct application?","commit_id":"2abc1510768266a830259a1ac1474cf88b9e48de"},{"author":{"_account_id":11809,"name":"Craig Bryant","email":"craig.bryant@hpe.com","username":"craig-bryant"},"change_message_id":"abc86c5cbcbdd3e8f4ce6a47aa770234799de5ad","unresolved":false,"context_lines":[{"line_number":107,"context_line":""},{"line_number":108,"context_line":"            for job in app[\u0027jobs\u0027]:"},{"line_number":109,"context_line":"                job_dimensions \u003d self._update_dimension(app_dimensions,"},{"line_number":110,"context_line":"                                                        {\u0027job_id\u0027: str(job[\u0027jobId\u0027])})"},{"line_number":111,"context_line":""},{"line_number":112,"context_line":"                duration \u003d self._duration_time(job[\u0027submissionTime\u0027], job[\u0027completionTime\u0027])"},{"line_number":113,"context_line":"                job_submission_time \u003d self._unix_timestamp(job[\u0027submissionTime\u0027])"}],"source_content_type":"text/x-python","patch_set":16,"id":"9a57fde8_d48db37c","line":110,"range":{"start_line":110,"start_character":67,"end_line":110,"end_character":84},"in_reply_to":"ba5201f7_7f537fe2","updated":"2017-01-12 17:42:15.000000000","message":"Given the app_id looks like that, why are you using it instead of the name field? That is what the user will be looking for, not the App Id.","commit_id":"2abc1510768266a830259a1ac1474cf88b9e48de"},{"author":{"_account_id":11809,"name":"Craig Bryant","email":"craig.bryant@hpe.com","username":"craig-bryant"},"change_message_id":"45daaf530fbf43329f796428317db78066d936be","unresolved":false,"context_lines":[{"line_number":107,"context_line":""},{"line_number":108,"context_line":"            for job in app[\u0027jobs\u0027]:"},{"line_number":109,"context_line":"                job_dimensions \u003d self._update_dimension(app_dimensions,"},{"line_number":110,"context_line":"                                                        {\u0027job_id\u0027: str(job[\u0027jobId\u0027])})"},{"line_number":111,"context_line":""},{"line_number":112,"context_line":"                duration \u003d self._duration_time(job[\u0027submissionTime\u0027], job[\u0027completionTime\u0027])"},{"line_number":113,"context_line":"                job_submission_time \u003d self._unix_timestamp(job[\u0027submissionTime\u0027])"}],"source_content_type":"text/x-python","patch_set":16,"id":"ba5201f7_a0c6ca97","line":110,"range":{"start_line":110,"start_character":67,"end_line":110,"end_character":84},"in_reply_to":"ba5201f7_828aadbd","updated":"2017-01-04 17:59:48.000000000","message":"Does jobid change when Spark is restarted? What does it look like?","commit_id":"2abc1510768266a830259a1ac1474cf88b9e48de"},{"author":{"_account_id":5559,"name":"gui maluf balzana","display_name":"Gui Maluf Balzana","email":"guilherme.balzana@canonical.com","username":"guimalufb"},"change_message_id":"cf7632e6f971bc3235300ae6e3accf7fa6f324a3","unresolved":false,"context_lines":[{"line_number":107,"context_line":""},{"line_number":108,"context_line":"            for job in app[\u0027jobs\u0027]:"},{"line_number":109,"context_line":"                job_dimensions \u003d self._update_dimension(app_dimensions,"},{"line_number":110,"context_line":"                                                        {\u0027job_id\u0027: str(job[\u0027jobId\u0027])})"},{"line_number":111,"context_line":""},{"line_number":112,"context_line":"                duration \u003d self._duration_time(job[\u0027submissionTime\u0027], job[\u0027completionTime\u0027])"},{"line_number":113,"context_line":"                job_submission_time \u003d self._unix_timestamp(job[\u0027submissionTime\u0027])"}],"source_content_type":"text/x-python","patch_set":16,"id":"ba5201f7_237664a4","line":110,"range":{"start_line":110,"start_character":67,"end_line":110,"end_character":84},"in_reply_to":"ba5201f7_a0c6ca97","updated":"2017-01-04 19:13:05.000000000","message":"No, \njobId is always the same within an application.\n\nIt\u0027ll exists equals jobsId but binding to an app_id.\ne.g. app_id: 100, jobId: 1, 2, 3, 4\n app_id: 200, jobId: 1, 2, 3, 4\n\nDoesn\u0027t make sense filter jobId 1, but make sense jobId from app_id 200\n\nAm I been clear enough?","commit_id":"2abc1510768266a830259a1ac1474cf88b9e48de"},{"author":{"_account_id":11809,"name":"Craig Bryant","email":"craig.bryant@hpe.com","username":"craig-bryant"},"change_message_id":"821c43c89ef17a36deeffd0d147cdacea7f77ea3","unresolved":false,"context_lines":[{"line_number":144,"context_line":"    def _proccess_metrics(self, data, metrics, name, dimensions, timestamp, value_meta):"},{"line_number":145,"context_line":"        for spark_metric, metric in metrics.iteritems():"},{"line_number":146,"context_line":"            metric_name, metric_type \u003d metric"},{"line_number":147,"context_line":"            if metric_type \u003d\u003d GAUGE:"},{"line_number":148,"context_line":"                self.gauge(\u0027spark.application.%s.%s\u0027 % (name, metric_name),"},{"line_number":149,"context_line":"                           value\u003ddata[spark_metric],"},{"line_number":150,"context_line":"                           dimensions\u003ddimensions,"}],"source_content_type":"text/x-python","patch_set":16,"id":"ba5201f7_76ebcfb0","line":147,"range":{"start_line":147,"start_character":15,"end_line":147,"end_character":36},"updated":"2017-01-04 16:33:21.000000000","message":"What is the point of having metric_type? It is always GAUGE and your ignores it if it is not. Code would be clearer without it.","commit_id":"2abc1510768266a830259a1ac1474cf88b9e48de"},{"author":{"_account_id":5559,"name":"gui maluf balzana","display_name":"Gui Maluf Balzana","email":"guilherme.balzana@canonical.com","username":"guimalufb"},"change_message_id":"f2b4fc35c0d66522e68e1a6bc71a5a93de001a14","unresolved":false,"context_lines":[{"line_number":144,"context_line":"    def _proccess_metrics(self, data, metrics, name, dimensions, timestamp, value_meta):"},{"line_number":145,"context_line":"        for spark_metric, metric in metrics.iteritems():"},{"line_number":146,"context_line":"            metric_name, metric_type \u003d metric"},{"line_number":147,"context_line":"            if metric_type \u003d\u003d GAUGE:"},{"line_number":148,"context_line":"                self.gauge(\u0027spark.application.%s.%s\u0027 % (name, metric_name),"},{"line_number":149,"context_line":"                           value\u003ddata[spark_metric],"},{"line_number":150,"context_line":"                           dimensions\u003ddimensions,"}],"source_content_type":"text/x-python","patch_set":16,"id":"ba5201f7_0264fd1f","line":147,"range":{"start_line":147,"start_character":15,"end_line":147,"end_character":36},"in_reply_to":"ba5201f7_76ebcfb0","updated":"2017-01-04 17:51:59.000000000","message":"I was just following similar patterns of other plugins.\nI\u0027ll clean up","commit_id":"2abc1510768266a830259a1ac1474cf88b9e48de"},{"author":{"_account_id":16168,"name":"Tomasz Trębski","email":"kornicameister@gmail.com","username":"trebskit"},"change_message_id":"c4a03ad52ac967ef30c98d324cf809c415fe5c46","unresolved":false,"context_lines":[{"line_number":64,"context_line":""},{"line_number":65,"context_line":"    def check(self, instance):"},{"line_number":66,"context_line":"        if self.instance_count() \u003e 0:"},{"line_number":67,"context_line":"            config_history_server \u003d instance.get(\u0027history_server\u0027, None)"},{"line_number":68,"context_line":"            if config_history_server is None:"},{"line_number":69,"context_line":"                raise Exception(\u0027A Spark history_server must be specified\u0027)"},{"line_number":70,"context_line":""}],"source_content_type":"text/x-python","patch_set":18,"id":"1a430d35_3b587a32","line":67,"updated":"2017-02-06 07:07:05.000000000","message":"You write down history_server_url down in spark detection plugin but here expect history_server","commit_id":"9e502d3bbc9215da95dc91e0460d2bc3e26999b1"},{"author":{"_account_id":5559,"name":"gui maluf balzana","display_name":"Gui Maluf Balzana","email":"guilherme.balzana@canonical.com","username":"guimalufb"},"change_message_id":"6f5a7ea787fb20147d800c3278c89b7a5f37466d","unresolved":false,"context_lines":[{"line_number":64,"context_line":""},{"line_number":65,"context_line":"    def check(self, instance):"},{"line_number":66,"context_line":"        if self.instance_count() \u003e 0:"},{"line_number":67,"context_line":"            config_history_server \u003d instance.get(\u0027history_server\u0027, None)"},{"line_number":68,"context_line":"            if config_history_server is None:"},{"line_number":69,"context_line":"                raise Exception(\u0027A Spark history_server must be specified\u0027)"},{"line_number":70,"context_line":""}],"source_content_type":"text/x-python","patch_set":18,"id":"1a430d35_5a19a087","line":67,"in_reply_to":"1a430d35_3b587a32","updated":"2017-02-06 19:02:41.000000000","message":"Done","commit_id":"9e502d3bbc9215da95dc91e0460d2bc3e26999b1"},{"author":{"_account_id":16168,"name":"Tomasz Trębski","email":"kornicameister@gmail.com","username":"trebskit"},"change_message_id":"dc05a787b3725187c44ab828e51a5c705bb2f2d4","unresolved":false,"context_lines":[{"line_number":90,"context_line":"        data \u003d self._fetch_data(url\u003dconfig_history_server, minDate\u003dlast_check)"},{"line_number":91,"context_line":"        self._collect_metrics(data, dimensions)"},{"line_number":92,"context_line":""},{"line_number":93,"context_line":"    def _collect_metrics(self, data, dimensions, value_meta\u003d{}):"},{"line_number":94,"context_line":"        for app in data:"},{"line_number":95,"context_line":"            app_dimensions \u003d self._update_dimension(dimensions, {\u0027app_id\u0027: app[\u0027id\u0027],"},{"line_number":96,"context_line":"                                                                 \u0027app_name\u0027: app[\u0027name\u0027]})"}],"source_content_type":"text/x-python","patch_set":18,"id":"1a430d35_41d2af01","line":93,"updated":"2017-02-06 11:44:04.000000000","message":"Mutable values shouldn\u0027t be used as default values IMHO","commit_id":"9e502d3bbc9215da95dc91e0460d2bc3e26999b1"},{"author":{"_account_id":16168,"name":"Tomasz Trębski","email":"kornicameister@gmail.com","username":"trebskit"},"change_message_id":"ebc3f9d3b280d0247320f1fb99980e3578efb7d6","unresolved":false,"context_lines":[{"line_number":90,"context_line":"        data \u003d self._fetch_data(url\u003dconfig_history_server, minDate\u003dlast_check)"},{"line_number":91,"context_line":"        self._collect_metrics(data, dimensions)"},{"line_number":92,"context_line":""},{"line_number":93,"context_line":"    def _collect_metrics(self, data, dimensions, value_meta\u003d{}):"},{"line_number":94,"context_line":"        for app in data:"},{"line_number":95,"context_line":"            app_dimensions \u003d self._update_dimension(dimensions, {\u0027app_id\u0027: app[\u0027id\u0027],"},{"line_number":96,"context_line":"                                                                 \u0027app_name\u0027: app[\u0027name\u0027]})"}],"source_content_type":"text/x-python","patch_set":18,"id":"1a430d35_21df3a0e","line":93,"in_reply_to":"1a430d35_156e0b95","updated":"2017-02-07 05:58:41.000000000","message":"I think that\u0027s the best way to show it\n\n  def test(el, con\u003d{}):\n    con[el] \u003d el\n    return con\n \n  # call\n  [\u0027{}\\n\u0027.format(test(i)) for i in range(1,10)]\n\nnow, normally the output I\u0027d expect is something along\n\n  Out 1:\n  {1:1},\n  {2:2},\n  {3:3}\n  ...\n\nso, each time function get\u0027s call a new dict will be created and returned having only one element, however mutable arguments in python are evaluated at the moment when function is defined not called, so the output here is something close to:\n\n  Out 2:\n  [\u0027{1: 1}\\n\u0027,\n   \u0027{1: 1, 2: 2}\\n\u0027,\n   \u0027{1: 1, 2: 2, 3: 3}\\n\u0027,\n   \u0027{1: 1, 2: 2, 3: 3, 4: 4}\\n\u0027,\n   \u0027{1: 1, 2: 2, 3: 3, 4: 4, 5: 5}\\n\u0027,\n   \u0027{1: 1, 2: 2, 3: 3, 4: 4, 5: 5, 6: 6}\\n\u0027,\n   \u0027{1: 1, 2: 2, 3: 3, 4: 4, 5: 5, 6: 6, 7: 7}\\n\u0027,\n   \u0027{1: 1, 2: 2, 3: 3, 4: 4, 5: 5, 6: 6, 7: 7, 8: 8}\\n\u0027,\n   \u0027{1: 1, 2: 2, 3: 3, 4: 4, 5: 5, 6: 6, 7: 7, 8: 8, 9: 9}\\n\u0027]\n\nwhich might be desired in some cases like for example maintaining state between calls.\n\nJust to supplement what was written already, function defined like this:\n  \n  def test(el, con\u003dNone):\n    if con is None:\n      con \u003d {}\n    con[el] \u003d el\n    return con\n\nwill give you results of Out 1","commit_id":"9e502d3bbc9215da95dc91e0460d2bc3e26999b1"},{"author":{"_account_id":5559,"name":"gui maluf balzana","display_name":"Gui Maluf Balzana","email":"guilherme.balzana@canonical.com","username":"guimalufb"},"change_message_id":"cf010e1d915e4b1a539348cd3bdd1b2cb3ca8bf0","unresolved":false,"context_lines":[{"line_number":90,"context_line":"        data \u003d self._fetch_data(url\u003dconfig_history_server, minDate\u003dlast_check)"},{"line_number":91,"context_line":"        self._collect_metrics(data, dimensions)"},{"line_number":92,"context_line":""},{"line_number":93,"context_line":"    def _collect_metrics(self, data, dimensions, value_meta\u003d{}):"},{"line_number":94,"context_line":"        for app in data:"},{"line_number":95,"context_line":"            app_dimensions \u003d self._update_dimension(dimensions, {\u0027app_id\u0027: app[\u0027id\u0027],"},{"line_number":96,"context_line":"                                                                 \u0027app_name\u0027: app[\u0027name\u0027]})"}],"source_content_type":"text/x-python","patch_set":18,"id":"1a430d35_085a3a2a","line":93,"in_reply_to":"1a430d35_21df3a0e","updated":"2017-02-07 21:11:26.000000000","message":"Done","commit_id":"9e502d3bbc9215da95dc91e0460d2bc3e26999b1"},{"author":{"_account_id":5559,"name":"gui maluf balzana","display_name":"Gui Maluf Balzana","email":"guilherme.balzana@canonical.com","username":"guimalufb"},"change_message_id":"6f5a7ea787fb20147d800c3278c89b7a5f37466d","unresolved":false,"context_lines":[{"line_number":90,"context_line":"        data \u003d self._fetch_data(url\u003dconfig_history_server, minDate\u003dlast_check)"},{"line_number":91,"context_line":"        self._collect_metrics(data, dimensions)"},{"line_number":92,"context_line":""},{"line_number":93,"context_line":"    def _collect_metrics(self, data, dimensions, value_meta\u003d{}):"},{"line_number":94,"context_line":"        for app in data:"},{"line_number":95,"context_line":"            app_dimensions \u003d self._update_dimension(dimensions, {\u0027app_id\u0027: app[\u0027id\u0027],"},{"line_number":96,"context_line":"                                                                 \u0027app_name\u0027: app[\u0027name\u0027]})"}],"source_content_type":"text/x-python","patch_set":18,"id":"1a430d35_156e0b95","line":93,"in_reply_to":"1a430d35_41d2af01","updated":"2017-02-06 19:02:41.000000000","message":"Could you clarify this point?\nI can\u0027t understand why do not user value_meta\u003d{} or params\u003d{}","commit_id":"9e502d3bbc9215da95dc91e0460d2bc3e26999b1"},{"author":{"_account_id":16168,"name":"Tomasz Trębski","email":"kornicameister@gmail.com","username":"trebskit"},"change_message_id":"6733100d3ae023795d0cc18731fec58813f7b6c6","unresolved":false,"context_lines":[{"line_number":164,"context_line":""},{"line_number":165,"context_line":"        return applications"},{"line_number":166,"context_line":""},{"line_number":167,"context_line":"    def _get_api_request(self, url, path\u003d\u0027\u0027, params\u003d{}):"},{"line_number":168,"context_line":"        try:"},{"line_number":169,"context_line":"            response \u003d requests.get(url\u003durl + path, params\u003dparams)"},{"line_number":170,"context_line":"            response.raise_for_status()"}],"source_content_type":"text/x-python","patch_set":18,"id":"1a430d35_fb0512c2","line":167,"updated":"2017-02-06 07:10:53.000000000","message":"I\u0027d suggest logging down arguments of this method for debugging. Tried run this locally, got error on _get_stages and I couldn\u0027t actually figure out what might have gone wrong without having really easy way to reproduce the request.","commit_id":"9e502d3bbc9215da95dc91e0460d2bc3e26999b1"},{"author":{"_account_id":5559,"name":"gui maluf balzana","display_name":"Gui Maluf Balzana","email":"guilherme.balzana@canonical.com","username":"guimalufb"},"change_message_id":"6f5a7ea787fb20147d800c3278c89b7a5f37466d","unresolved":false,"context_lines":[{"line_number":164,"context_line":""},{"line_number":165,"context_line":"        return applications"},{"line_number":166,"context_line":""},{"line_number":167,"context_line":"    def _get_api_request(self, url, path\u003d\u0027\u0027, params\u003d{}):"},{"line_number":168,"context_line":"        try:"},{"line_number":169,"context_line":"            response \u003d requests.get(url\u003durl + path, params\u003dparams)"},{"line_number":170,"context_line":"            response.raise_for_status()"}],"source_content_type":"text/x-python","patch_set":18,"id":"1a430d35_5506d3ce","line":167,"in_reply_to":"1a430d35_81dc9713","updated":"2017-02-06 19:02:41.000000000","message":"Could you clarify this point?\nI can\u0027t understand why do not user value_meta\u003d{} or params\u003d{}","commit_id":"9e502d3bbc9215da95dc91e0460d2bc3e26999b1"},{"author":{"_account_id":5559,"name":"gui maluf balzana","display_name":"Gui Maluf Balzana","email":"guilherme.balzana@canonical.com","username":"guimalufb"},"change_message_id":"6f5a7ea787fb20147d800c3278c89b7a5f37466d","unresolved":false,"context_lines":[{"line_number":164,"context_line":""},{"line_number":165,"context_line":"        return applications"},{"line_number":166,"context_line":""},{"line_number":167,"context_line":"    def _get_api_request(self, url, path\u003d\u0027\u0027, params\u003d{}):"},{"line_number":168,"context_line":"        try:"},{"line_number":169,"context_line":"            response \u003d requests.get(url\u003durl + path, params\u003dparams)"},{"line_number":170,"context_line":"            response.raise_for_status()"}],"source_content_type":"text/x-python","patch_set":18,"id":"1a430d35_953cbbe3","line":167,"in_reply_to":"1a430d35_fb0512c2","updated":"2017-02-06 19:02:41.000000000","message":"Done","commit_id":"9e502d3bbc9215da95dc91e0460d2bc3e26999b1"},{"author":{"_account_id":16168,"name":"Tomasz Trębski","email":"kornicameister@gmail.com","username":"trebskit"},"change_message_id":"dc05a787b3725187c44ab828e51a5c705bb2f2d4","unresolved":false,"context_lines":[{"line_number":164,"context_line":""},{"line_number":165,"context_line":"        return applications"},{"line_number":166,"context_line":""},{"line_number":167,"context_line":"    def _get_api_request(self, url, path\u003d\u0027\u0027, params\u003d{}):"},{"line_number":168,"context_line":"        try:"},{"line_number":169,"context_line":"            response \u003d requests.get(url\u003durl + path, params\u003dparams)"},{"line_number":170,"context_line":"            response.raise_for_status()"}],"source_content_type":"text/x-python","patch_set":18,"id":"1a430d35_81dc9713","line":167,"in_reply_to":"1a430d35_fb0512c2","updated":"2017-02-06 11:44:04.000000000","message":"Mutable values shouldn\u0027t be used as default values IMHO","commit_id":"9e502d3bbc9215da95dc91e0460d2bc3e26999b1"},{"author":{"_account_id":16168,"name":"Tomasz Trębski","email":"kornicameister@gmail.com","username":"trebskit"},"change_message_id":"dc05a787b3725187c44ab828e51a5c705bb2f2d4","unresolved":false,"context_lines":[{"line_number":192,"context_line":"    def _get_stages(self, url, application_id):"},{"line_number":193,"context_line":"        return self._get_api_request(url\u003durl,"},{"line_number":194,"context_line":"                                     path\u003d\u0027/applications/\u0027 + application_id + \u0027/stages\u0027)"},{"line_number":195,"context_line":""},{"line_number":196,"context_line":"    def _get_stage(self, url, application_id, stage_id, status\u003d\u0027\u0027):"},{"line_number":197,"context_line":"        params \u003d {\u0027status\u0027: status}"},{"line_number":198,"context_line":"        return self._get_api_request(url\u003durl,"}],"source_content_type":"text/x-python","patch_set":18,"id":"1a430d35_21020370","line":195,"updated":"2017-02-06 11:44:04.000000000","message":"Why status here has a value and the one in _get_jobs does not ?","commit_id":"9e502d3bbc9215da95dc91e0460d2bc3e26999b1"},{"author":{"_account_id":5559,"name":"gui maluf balzana","display_name":"Gui Maluf Balzana","email":"guilherme.balzana@canonical.com","username":"guimalufb"},"change_message_id":"6f5a7ea787fb20147d800c3278c89b7a5f37466d","unresolved":false,"context_lines":[{"line_number":192,"context_line":"    def _get_stages(self, url, application_id):"},{"line_number":193,"context_line":"        return self._get_api_request(url\u003durl,"},{"line_number":194,"context_line":"                                     path\u003d\u0027/applications/\u0027 + application_id + \u0027/stages\u0027)"},{"line_number":195,"context_line":""},{"line_number":196,"context_line":"    def _get_stage(self, url, application_id, stage_id, status\u003d\u0027\u0027):"},{"line_number":197,"context_line":"        params \u003d {\u0027status\u0027: status}"},{"line_number":198,"context_line":"        return self._get_api_request(url\u003durl,"}],"source_content_type":"text/x-python","patch_set":18,"id":"1a430d35_394ea16f","line":195,"in_reply_to":"1a430d35_21020370","updated":"2017-02-06 19:02:41.000000000","message":"No specific reason\nDone","commit_id":"9e502d3bbc9215da95dc91e0460d2bc3e26999b1"},{"author":{"_account_id":16168,"name":"Tomasz Trębski","email":"kornicameister@gmail.com","username":"trebskit"},"change_message_id":"42ff6a5ec406d6bbb1824cf862df20cb61e335d4","unresolved":false,"context_lines":[{"line_number":170,"context_line":"            response.raise_for_status()"},{"line_number":171,"context_line":"            return response.json()"},{"line_number":172,"context_line":"        except requests.exceptions.RequestException as e:"},{"line_number":173,"context_line":"            self.log.error(e)"},{"line_number":174,"context_line":"            self.log.error(\u0027Failed to GET %s with %s\u0027 % (url+path, params))"},{"line_number":175,"context_line":"            return {}"},{"line_number":176,"context_line":""}],"source_content_type":"text/x-python","patch_set":19,"id":"1a430d35_01eebeb3","line":173,"updated":"2017-02-07 06:00:32.000000000","message":"Not really sure if a method is available, but could you check if you can change self.log.error to self.log.exception + move the error message above the exception ?","commit_id":"a05fdf9ddf8e2dc32dc34d152f209f3eff0d4a65"},{"author":{"_account_id":5559,"name":"gui maluf balzana","display_name":"Gui Maluf Balzana","email":"guilherme.balzana@canonical.com","username":"guimalufb"},"change_message_id":"940813607ece111b90d448733147830e0b52ec62","unresolved":false,"context_lines":[{"line_number":170,"context_line":"            response.raise_for_status()"},{"line_number":171,"context_line":"            return response.json()"},{"line_number":172,"context_line":"        except requests.exceptions.RequestException as e:"},{"line_number":173,"context_line":"            self.log.error(e)"},{"line_number":174,"context_line":"            self.log.error(\u0027Failed to GET %s with %s\u0027 % (url+path, params))"},{"line_number":175,"context_line":"            return {}"},{"line_number":176,"context_line":""}],"source_content_type":"text/x-python","patch_set":19,"id":"1a430d35_284d363e","line":173,"in_reply_to":"1a430d35_01eebeb3","updated":"2017-02-07 21:11:42.000000000","message":"Done","commit_id":"a05fdf9ddf8e2dc32dc34d152f209f3eff0d4a65"}],"monasca_setup/detection/plugins/spark.py":[{"author":{"_account_id":16168,"name":"Tomasz Trębski","email":"kornicameister@gmail.com","username":"trebskit"},"change_message_id":"caf1d2abd3554ccd9e299a80623593be2118c665","unresolved":false,"context_lines":[{"line_number":22,"context_line":"        port or process is running"},{"line_number":23,"context_line":"        \"\"\""},{"line_number":24,"context_line":""},{"line_number":25,"context_line":"        port_listen \u003d find_addr_listening_on_port(\u002718080\u0027) is not None"},{"line_number":26,"context_line":"        process_exists \u003d find_process_cmdline(_SPARK_JAVA_CLASS) is not None"},{"line_number":27,"context_line":"        has_dependencies \u003d self.dependencies_installed()"},{"line_number":28,"context_line":""}],"source_content_type":"text/x-python","patch_set":15,"id":"fa6399be_668613bb","line":25,"updated":"2016-10-11 05:14:02.000000000","message":"This could be part of the argument passed to the plugin during manual call of monasca-setup. So you could have 18080 by default but leave some room for adjusting the plugin.","commit_id":"a2b281348b5903d6e9a54f6575aaf3a1eeadbfff"},{"author":{"_account_id":20033,"name":"Artur Basiak","email":"artur.basiak@ts.fujitsu.com","username":"abasiak"},"change_message_id":"0a8b9140557fab3ffdabec172b95d19c1f069567","unresolved":false,"context_lines":[{"line_number":21,"context_line":"        port or process is running"},{"line_number":22,"context_line":"        \"\"\""},{"line_number":23,"context_line":""},{"line_number":24,"context_line":"        port_listening \u003d utils.find_addr_listening_on_port(18080) is not None"},{"line_number":25,"context_line":"        process_exists \u003d utils.find_process_cmdline(_SPARK_JAVA_CLASS) is not None"},{"line_number":26,"context_line":"        has_dependencies \u003d self.dependencies_installed()"},{"line_number":27,"context_line":""}],"source_content_type":"text/x-python","patch_set":16,"id":"7a3c09a3_dce88f19","line":24,"range":{"start_line":24,"start_character":59,"end_line":24,"end_character":64},"updated":"2017-01-13 06:22:21.000000000","message":"Shouldn\u0027t this be configurable? You can move this to variable and also pass it via monasca-setup --detection_args_json IMHO.","commit_id":"2abc1510768266a830259a1ac1474cf88b9e48de"},{"author":{"_account_id":5559,"name":"gui maluf balzana","display_name":"Gui Maluf Balzana","email":"guilherme.balzana@canonical.com","username":"guimalufb"},"change_message_id":"279229cb830dd3aa5ac9ade35d1c258dec8f104f","unresolved":false,"context_lines":[{"line_number":21,"context_line":"        port or process is running"},{"line_number":22,"context_line":"        \"\"\""},{"line_number":23,"context_line":""},{"line_number":24,"context_line":"        port_listening \u003d utils.find_addr_listening_on_port(18080) is not None"},{"line_number":25,"context_line":"        process_exists \u003d utils.find_process_cmdline(_SPARK_JAVA_CLASS) is not None"},{"line_number":26,"context_line":"        has_dependencies \u003d self.dependencies_installed()"},{"line_number":27,"context_line":""}],"source_content_type":"text/x-python","patch_set":16,"id":"1a430d35_ecc035e3","line":24,"range":{"start_line":24,"start_character":59,"end_line":24,"end_character":64},"in_reply_to":"7a3c09a3_dce88f19","updated":"2017-02-03 12:35:48.000000000","message":"I was simplifying the detection plugin.\nI\u0027ll parameterize it","commit_id":"2abc1510768266a830259a1ac1474cf88b9e48de"}]}