)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"792b870132c0c4ffdbc825a67f16780f972c584f","unresolved":false,"context_lines":[{"line_number":10,"context_line":"non-obvious failure modes (the NUMATopologyFilter in particular comes"},{"line_number":11,"context_line":"to mind). It is possible to have a situation where a given instances"},{"line_number":12,"context_line":"fails to schedule, and it is not immediately apparent as to what"},{"line_number":13,"context_line":"exactly caused the failure."},{"line_number":14,"context_line":""},{"line_number":15,"context_line":"Accordingly, it is proposed that we allow for optional detailed"},{"line_number":16,"context_line":"messages about precisely *why* a scheduler filter is failing."}],"source_content_type":"text/x-gerrit-commit-message","patch_set":11,"id":"dab17558_cbb21be5","line":13,"updated":"2016-05-13 14:16:51.000000000","message":"Yup! :(","commit_id":"f9e3e35dc1f9ae3f7b0b01eecf54d706ef736b13"}],"specs/newton/approved/improve-sched-error-reporting.rst":[{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"b9996b57634e7fb24b66322f1aaf3e0d955b3afd","unresolved":false,"context_lines":[{"line_number":46,"context_line":"we propose that the exact problem should be indicated in the log:"},{"line_number":47,"context_line":""},{"line_number":48,"context_line":"*\"%(host)s, %(node)s CPUs avail(%d) \u003c required(%d)\"*"},{"line_number":49,"context_line":""},{"line_number":50,"context_line":"Use Cases"},{"line_number":51,"context_line":"---------"},{"line_number":52,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"1a122d0e_75f409f0","line":49,"updated":"2016-04-28 20:45:30.000000000","message":"Like we discussed, I\u0027m totally in favor of a change helping logging like this but...","commit_id":"a1f60b6d27bbde839725ffd5db819b293a77bfbe"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"b9996b57634e7fb24b66322f1aaf3e0d955b3afd","unresolved":false,"context_lines":[{"line_number":57,"context_line":"Proposed change"},{"line_number":58,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":59,"context_line":""},{"line_number":60,"context_line":"The host_passes() routine in BaseHostFilter will be extended to take an"},{"line_number":61,"context_line":"additional \"info\" argument which will be used to track the reason for why a"},{"line_number":62,"context_line":"particular filter failed a particular compute node.  This argument will be a"},{"line_number":63,"context_line":"dict, with the keys being the names of filters, and the values being"}],"source_content_type":"text/x-rst","patch_set":1,"id":"1a122d0e_955605cc","line":60,"updated":"2016-04-28 20:45:30.000000000","message":"... please don\u0027t provide this implementation details.\n\nLike we discussed during the Summit, some consensus was about providing a pile of logs back to the scheduler BaseFilterHandler (probably using the RequestContext object) so that the BaseHandler could get it and depile it in case of an error.\n\nLater mid-term, we discussed on this kind of change should maybe be ported to oslo.logging instead.","commit_id":"a1f60b6d27bbde839725ffd5db819b293a77bfbe"},{"author":{"_account_id":11303,"name":"Markus Zoeller (markus_z)","email":"mzoeller@linux.vnet.ibm.com","username":"mzoeller"},"change_message_id":"490eebf1b7cce1ffb2daeaf23e61f5b6ee250a23","unresolved":false,"context_lines":[{"line_number":59,"context_line":""},{"line_number":60,"context_line":"The host_passes() routine in BaseHostFilter will be extended to take an"},{"line_number":61,"context_line":"additional \"info\" argument which will be used to track the reason for why a"},{"line_number":62,"context_line":"particular filter failed a particular compute node.  This argument will be a"},{"line_number":63,"context_line":"dict, with the keys being the names of filters, and the values being"},{"line_number":64,"context_line":"themselves dicts, where each dict in turn has keys of compute node names and"},{"line_number":65,"context_line":"values of strings describing the reason why that compute node failed the"},{"line_number":66,"context_line":"filter.  (In many cases this could just be the same message as is logged in the"},{"line_number":67,"context_line":"existing debug log message.)"},{"line_number":68,"context_line":""},{"line_number":69,"context_line":"By accumulating this information rather than logging it, we can log the"}],"source_content_type":"text/x-rst","patch_set":1,"id":"1a122d0e_420cd9eb","line":66,"range":{"start_line":62,"start_character":53,"end_line":66,"end_character":7},"updated":"2016-04-27 19:40:12.000000000","message":"Why not having a class for that?","commit_id":"a1f60b6d27bbde839725ffd5db819b293a77bfbe"},{"author":{"_account_id":8768,"name":"Chris Friesen","email":"chris.friesen@windriver.com","username":"cbf123"},"change_message_id":"c4d5da48900b47c8e8cfab24903622cc3740f251","unresolved":false,"context_lines":[{"line_number":59,"context_line":""},{"line_number":60,"context_line":"The host_passes() routine in BaseHostFilter will be extended to take an"},{"line_number":61,"context_line":"additional \"info\" argument which will be used to track the reason for why a"},{"line_number":62,"context_line":"particular filter failed a particular compute node.  This argument will be a"},{"line_number":63,"context_line":"dict, with the keys being the names of filters, and the values being"},{"line_number":64,"context_line":"themselves dicts, where each dict in turn has keys of compute node names and"},{"line_number":65,"context_line":"values of strings describing the reason why that compute node failed the"},{"line_number":66,"context_line":"filter.  (In many cases this could just be the same message as is logged in the"},{"line_number":67,"context_line":"existing debug log message.)"},{"line_number":68,"context_line":""},{"line_number":69,"context_line":"By accumulating this information rather than logging it, we can log the"}],"source_content_type":"text/x-rst","patch_set":1,"id":"1a122d0e_1b5c087c","line":66,"range":{"start_line":62,"start_character":53,"end_line":66,"end_character":7},"in_reply_to":"1a122d0e_420cd9eb","updated":"2016-04-27 22:01:31.000000000","message":"Yes, it might make sense to make it a class.  At a minimum there would be a helper function.","commit_id":"a1f60b6d27bbde839725ffd5db819b293a77bfbe"},{"author":{"_account_id":11564,"name":"Chris Dent","email":"cdent@anticdent.org","username":"chdent"},"change_message_id":"7680b7855215fce1fca1247359011f305bc55719","unresolved":false,"context_lines":[{"line_number":64,"context_line":"themselves dicts, where each dict in turn has keys of compute node names and"},{"line_number":65,"context_line":"values of strings describing the reason why that compute node failed the"},{"line_number":66,"context_line":"filter.  (In many cases this could just be the same message as is logged in the"},{"line_number":67,"context_line":"existing debug log message.)"},{"line_number":68,"context_line":""},{"line_number":69,"context_line":"By accumulating this information rather than logging it, we can log the"},{"line_number":70,"context_line":"information in FilterScheduler._schedule() only if no valid hosts are returned."}],"source_content_type":"text/x-rst","patch_set":1,"id":"1a122d0e_47457915","line":67,"updated":"2016-04-18 11:12:30.000000000","message":"Is the idea to create `info` in _schedule and pass it down the stack to `host_passes` or to create it lower down and return it along with the hosts when `get_filtered_hosts` is called?\n\nAnd since we\u0027re in a weird loop there in which the selected hosts are winnowed each time round the loop, how is `info` itself getting winnowed?\n\nAlso, it seems to me that unless the number of candidate hosts starts out small (perhaps a result of in db filtering happening before out of db filtering) the size and complexity of `info` gets interesting at scale.","commit_id":"a1f60b6d27bbde839725ffd5db819b293a77bfbe"},{"author":{"_account_id":8768,"name":"Chris Friesen","email":"chris.friesen@windriver.com","username":"cbf123"},"change_message_id":"7c25e42cb5d3903ada835951b35ac4b18c25671d","unresolved":false,"context_lines":[{"line_number":64,"context_line":"themselves dicts, where each dict in turn has keys of compute node names and"},{"line_number":65,"context_line":"values of strings describing the reason why that compute node failed the"},{"line_number":66,"context_line":"filter.  (In many cases this could just be the same message as is logged in the"},{"line_number":67,"context_line":"existing debug log message.)"},{"line_number":68,"context_line":""},{"line_number":69,"context_line":"By accumulating this information rather than logging it, we can log the"},{"line_number":70,"context_line":"information in FilterScheduler._schedule() only if no valid hosts are returned."}],"source_content_type":"text/x-rst","patch_set":1,"id":"1a122d0e_512a852d","line":67,"in_reply_to":"1a122d0e_47457915","updated":"2016-04-21 19:26:11.000000000","message":"Our current implementation passes down a dict that each filter can fill in.  I suppose the filters could also create an info report and return it with the hosts.  I\u0027m fine either way.\n\nI agree that it could get interesting at scale, which is why I thought we\u0027d specify which filters should provide the extra information.  Presumably we\u0027d also run those filters later on in the filter list, to minimize the number of complicated tests that need to be done.\n\nI don\u0027t really see any way around the possible scaling issues while still preserving enough information to figure out why something failed to schedule.  I\u0027m open to suggestions though.","commit_id":"a1f60b6d27bbde839725ffd5db819b293a77bfbe"},{"author":{"_account_id":11303,"name":"Markus Zoeller (markus_z)","email":"mzoeller@linux.vnet.ibm.com","username":"mzoeller"},"change_message_id":"490eebf1b7cce1ffb2daeaf23e61f5b6ee250a23","unresolved":false,"context_lines":[{"line_number":71,"context_line":"Since the logging happens only on failure, it is proposed that we log at the"},{"line_number":72,"context_line":"\"info\" level rather than the \"debug\" level."},{"line_number":73,"context_line":""},{"line_number":74,"context_line":"In order to further reduce the additional logging, it is proposed that a new"},{"line_number":75,"context_line":"\"scheduler_enable_log_filters\" config option be added.  This will consist of a"},{"line_number":76,"context_line":"list of filter names (analogous to \"scheduler_default_filters\") for which we"},{"line_number":77,"context_line":"want to enable the additional logging.  In this way we can avoid extra logging"},{"line_number":78,"context_line":"for cases which fail for trivial reasons like lack of CPU/RAM/disk.  I propose"}],"source_content_type":"text/x-rst","patch_set":1,"id":"1a122d0e_e2bfe5cd","line":75,"range":{"start_line":74,"start_character":0,"end_line":75,"end_character":53},"updated":"2016-04-27 19:40:12.000000000","message":"I can imagine that this will cause confusion when sometimes a info log will happen and sometimes not. Having the behavior consistent over the filters sounds more pleasant to me. I\u0027d prefer behavior consistency over (a tiny?) logging amount reduction.","commit_id":"a1f60b6d27bbde839725ffd5db819b293a77bfbe"},{"author":{"_account_id":11303,"name":"Markus Zoeller (markus_z)","email":"mzoeller@linux.vnet.ibm.com","username":"mzoeller"},"change_message_id":"e6ada8cac7a8c0df4381a4155a0f690fdfb03eff","unresolved":false,"context_lines":[{"line_number":71,"context_line":"Since the logging happens only on failure, it is proposed that we log at the"},{"line_number":72,"context_line":"\"info\" level rather than the \"debug\" level."},{"line_number":73,"context_line":""},{"line_number":74,"context_line":"In order to further reduce the additional logging, it is proposed that a new"},{"line_number":75,"context_line":"\"scheduler_enable_log_filters\" config option be added.  This will consist of a"},{"line_number":76,"context_line":"list of filter names (analogous to \"scheduler_default_filters\") for which we"},{"line_number":77,"context_line":"want to enable the additional logging.  In this way we can avoid extra logging"},{"line_number":78,"context_line":"for cases which fail for trivial reasons like lack of CPU/RAM/disk.  I propose"}],"source_content_type":"text/x-rst","patch_set":1,"id":"1a122d0e_1c3342f7","line":75,"range":{"start_line":74,"start_character":0,"end_line":75,"end_character":53},"in_reply_to":"1a122d0e_bb9074ee","updated":"2016-04-28 19:05:22.000000000","message":"OK, I\u0027m convinced.","commit_id":"a1f60b6d27bbde839725ffd5db819b293a77bfbe"},{"author":{"_account_id":8768,"name":"Chris Friesen","email":"chris.friesen@windriver.com","username":"cbf123"},"change_message_id":"c4d5da48900b47c8e8cfab24903622cc3740f251","unresolved":false,"context_lines":[{"line_number":71,"context_line":"Since the logging happens only on failure, it is proposed that we log at the"},{"line_number":72,"context_line":"\"info\" level rather than the \"debug\" level."},{"line_number":73,"context_line":""},{"line_number":74,"context_line":"In order to further reduce the additional logging, it is proposed that a new"},{"line_number":75,"context_line":"\"scheduler_enable_log_filters\" config option be added.  This will consist of a"},{"line_number":76,"context_line":"list of filter names (analogous to \"scheduler_default_filters\") for which we"},{"line_number":77,"context_line":"want to enable the additional logging.  In this way we can avoid extra logging"},{"line_number":78,"context_line":"for cases which fail for trivial reasons like lack of CPU/RAM/disk.  I propose"}],"source_content_type":"text/x-rst","patch_set":1,"id":"1a122d0e_bb9074ee","line":75,"range":{"start_line":74,"start_character":0,"end_line":75,"end_character":53},"in_reply_to":"1a122d0e_e2bfe5cd","updated":"2016-04-27 22:01:31.000000000","message":"The config option would be set by the admin, so I don\u0027t think there would be any confusion.  If they want consistency they can just specify all the filters.\n\nAs an example, I don\u0027t think it makes sense to log all the compute nodes that failed simple filters like CPU/RAM/disk, especially since we\u0027re looking at moving some of the simple filters into DB queries at which point they wouldn\u0027t be logged anyways.","commit_id":"a1f60b6d27bbde839725ffd5db819b293a77bfbe"},{"author":{"_account_id":6589,"name":"Jesse Keating","email":"jkeating@j2solutions.net","username":"jesse-keating"},"change_message_id":"de181caea23ea2b20974bac5233278d0d1008d56","unresolved":false,"context_lines":[{"line_number":65,"context_line":"Second, we will accumulate debug logs in memory and then emit them only if the"},{"line_number":66,"context_line":"overall scheduling operation fails (i.e. exception.NoValidHost is raised)."},{"line_number":67,"context_line":"Since the logging would happen only on failure, it is proposed that we log at"},{"line_number":68,"context_line":"the \"info\" level rather than the \"debug\" level."},{"line_number":69,"context_line":""},{"line_number":70,"context_line":"In order to further reduce the additional logging, it is proposed that a new"},{"line_number":71,"context_line":"\"scheduler_suppress_log_filters\" config option be added.  This will consist of a"}],"source_content_type":"text/x-rst","patch_set":2,"id":"1a122d0e_bad649c1","line":68,"updated":"2016-05-02 17:00:39.000000000","message":"There is definitely a use case where operators will want to see why a particular host is not chosen, even if some other host ultimately is chosen. In this case, there is no eventual NoValidHost exception, and so with your current implementation the scheduling decisions will be lost. I\u0027d prefer there be a way to still retain these bits of information so that an operator can clearly trace why machine B was chosen over machine A to satisfy the scheduling requirements.\n\nWhether this is made available with yet another config option, or the decision is made to not use NoValidHost as a deciding factor on when to log, and instead just log always to the appropriate level, I care not. I just care that as an Operator I have available to me the entire debugging information for every scheduler decision, be it ultimately successful or not.","commit_id":"7aa9249af88900f02848489716c113a83b39b91a"}],"specs/newton/approved/improve-sched-logging.rst":[{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"8dff7dc0513f37cbb44725a10b57daa68e6695d4","unresolved":false,"context_lines":[{"line_number":63,"context_line":"First, we will attempt to ensure that all scheduler filter failures have a"},{"line_number":64,"context_line":"useful log message describing why the filter failed for that host."},{"line_number":65,"context_line":""},{"line_number":66,"context_line":"Second, when the log level is set higher than \"debug\", we will accumulate"},{"line_number":67,"context_line":"debug logs in memory and then emit them only if the overall scheduling"},{"line_number":68,"context_line":"operation fails (i.e. exception.NoValidHost is raised).  Since the logging"},{"line_number":69,"context_line":"would happen only on failure, it is proposed that we log at the \"info\" level"},{"line_number":70,"context_line":"rather than the \"debug\" level.  When the log level is set to \"debug\", we will"},{"line_number":71,"context_line":"emit logs for the success case as well, but subject to the suppression filter"},{"line_number":72,"context_line":"specified below."},{"line_number":73,"context_line":""},{"line_number":74,"context_line":"In order to further reduce the logging, it is proposed that a new"},{"line_number":75,"context_line":"\"scheduler_suppress_log_filters\" config option be added.  This will consist of"}],"source_content_type":"text/x-rst","patch_set":7,"id":"1a122d0e_c07146e0","line":72,"range":{"start_line":66,"start_character":0,"end_line":72,"end_character":16},"updated":"2016-05-04 15:55:30.000000000","message":"I also struggle to understand how this will be implemented without hacks. Would you happen to have any PoC code I could look through?","commit_id":"9b600b58da53198468622a5c4486c4be99faa918"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"8dff7dc0513f37cbb44725a10b57daa68e6695d4","unresolved":false,"context_lines":[{"line_number":63,"context_line":"First, we will attempt to ensure that all scheduler filter failures have a"},{"line_number":64,"context_line":"useful log message describing why the filter failed for that host."},{"line_number":65,"context_line":""},{"line_number":66,"context_line":"Second, when the log level is set higher than \"debug\", we will accumulate"},{"line_number":67,"context_line":"debug logs in memory and then emit them only if the overall scheduling"},{"line_number":68,"context_line":"operation fails (i.e. exception.NoValidHost is raised).  Since the logging"},{"line_number":69,"context_line":"would happen only on failure, it is proposed that we log at the \"info\" level"},{"line_number":70,"context_line":"rather than the \"debug\" level.  When the log level is set to \"debug\", we will"},{"line_number":71,"context_line":"emit logs for the success case as well, but subject to the suppression filter"},{"line_number":72,"context_line":"specified below."},{"line_number":73,"context_line":""},{"line_number":74,"context_line":"In order to further reduce the logging, it is proposed that a new"},{"line_number":75,"context_line":"\"scheduler_suppress_log_filters\" config option be added.  This will consist of"}],"source_content_type":"text/x-rst","patch_set":7,"id":"1a122d0e_2093dafb","line":72,"range":{"start_line":66,"start_character":0,"end_line":72,"end_character":16},"updated":"2016-05-04 15:55:30.000000000","message":"I assume you\u0027re talking about the logs that the scheduler emits when making decisions (does it fit on this node?). You could do with being more concrete here as we do in fact log at ERROR level for failed case, but the messages are simply too generic (as you say).","commit_id":"9b600b58da53198468622a5c4486c4be99faa918"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"9d5ae9741a3992fd75b4ee4fbdc8d1626e2872d2","unresolved":false,"context_lines":[{"line_number":63,"context_line":"First, we will attempt to ensure that all scheduler filter failures have a"},{"line_number":64,"context_line":"useful log message describing why the filter failed for that host."},{"line_number":65,"context_line":""},{"line_number":66,"context_line":"Second, when the log level is set higher than \"debug\", we will accumulate"},{"line_number":67,"context_line":"debug logs in memory and then emit them only if the overall scheduling"},{"line_number":68,"context_line":"operation fails (i.e. exception.NoValidHost is raised).  Since the logging"},{"line_number":69,"context_line":"would happen only on failure, it is proposed that we log at the \"info\" level"},{"line_number":70,"context_line":"rather than the \"debug\" level.  When the log level is set to \"debug\", we will"},{"line_number":71,"context_line":"emit logs for the success case as well, but subject to the suppression filter"},{"line_number":72,"context_line":"specified below."},{"line_number":73,"context_line":""},{"line_number":74,"context_line":"In order to further reduce the logging, it is proposed that a new"},{"line_number":75,"context_line":"\"scheduler_suppress_log_filters\" config option be added.  This will consist of"}],"source_content_type":"text/x-rst","patch_set":7,"id":"1a122d0e_7ae9fab6","line":72,"range":{"start_line":66,"start_character":0,"end_line":72,"end_character":16},"in_reply_to":"1a122d0e_0bd6bff0","updated":"2016-05-04 16:41:10.000000000","message":"Lovely - cheers :)","commit_id":"9b600b58da53198468622a5c4486c4be99faa918"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"9d5ae9741a3992fd75b4ee4fbdc8d1626e2872d2","unresolved":false,"context_lines":[{"line_number":63,"context_line":"First, we will attempt to ensure that all scheduler filter failures have a"},{"line_number":64,"context_line":"useful log message describing why the filter failed for that host."},{"line_number":65,"context_line":""},{"line_number":66,"context_line":"Second, when the log level is set higher than \"debug\", we will accumulate"},{"line_number":67,"context_line":"debug logs in memory and then emit them only if the overall scheduling"},{"line_number":68,"context_line":"operation fails (i.e. exception.NoValidHost is raised).  Since the logging"},{"line_number":69,"context_line":"would happen only on failure, it is proposed that we log at the \"info\" level"},{"line_number":70,"context_line":"rather than the \"debug\" level.  When the log level is set to \"debug\", we will"},{"line_number":71,"context_line":"emit logs for the success case as well, but subject to the suppression filter"},{"line_number":72,"context_line":"specified below."},{"line_number":73,"context_line":""},{"line_number":74,"context_line":"In order to further reduce the logging, it is proposed that a new"},{"line_number":75,"context_line":"\"scheduler_suppress_log_filters\" config option be added.  This will consist of"}],"source_content_type":"text/x-rst","patch_set":7,"id":"1a122d0e_7a00da66","line":72,"range":{"start_line":66,"start_character":0,"end_line":72,"end_character":16},"in_reply_to":"1a122d0e_0be07f95","updated":"2016-05-04 16:41:10.000000000","message":"OK, good to see you have some plan in place. You don\u0027t have to include specifics here, but the above might be helpful if you feel like adding it.","commit_id":"9b600b58da53198468622a5c4486c4be99faa918"},{"author":{"_account_id":8768,"name":"Chris Friesen","email":"chris.friesen@windriver.com","username":"cbf123"},"change_message_id":"0a31a4f56f7c1ae00a24346c9cc60b08923a7edd","unresolved":false,"context_lines":[{"line_number":63,"context_line":"First, we will attempt to ensure that all scheduler filter failures have a"},{"line_number":64,"context_line":"useful log message describing why the filter failed for that host."},{"line_number":65,"context_line":""},{"line_number":66,"context_line":"Second, when the log level is set higher than \"debug\", we will accumulate"},{"line_number":67,"context_line":"debug logs in memory and then emit them only if the overall scheduling"},{"line_number":68,"context_line":"operation fails (i.e. exception.NoValidHost is raised).  Since the logging"},{"line_number":69,"context_line":"would happen only on failure, it is proposed that we log at the \"info\" level"},{"line_number":70,"context_line":"rather than the \"debug\" level.  When the log level is set to \"debug\", we will"},{"line_number":71,"context_line":"emit logs for the success case as well, but subject to the suppression filter"},{"line_number":72,"context_line":"specified below."},{"line_number":73,"context_line":""},{"line_number":74,"context_line":"In order to further reduce the logging, it is proposed that a new"},{"line_number":75,"context_line":"\"scheduler_suppress_log_filters\" config option be added.  This will consist of"}],"source_content_type":"text/x-rst","patch_set":7,"id":"1a122d0e_0bd6bff0","line":72,"range":{"start_line":66,"start_character":0,"end_line":72,"end_character":16},"in_reply_to":"1a122d0e_2093dafb","updated":"2016-05-04 16:19:38.000000000","message":"Yes, I\u0027m talking about the logs from the scheduler filters.  We currently log the fact that it failed at ERROR level, but the detailed logs from each filter is only available at the DEBUG level (if at all).  I\u0027ll try to reword this.","commit_id":"9b600b58da53198468622a5c4486c4be99faa918"},{"author":{"_account_id":8768,"name":"Chris Friesen","email":"chris.friesen@windriver.com","username":"cbf123"},"change_message_id":"0a31a4f56f7c1ae00a24346c9cc60b08923a7edd","unresolved":false,"context_lines":[{"line_number":63,"context_line":"First, we will attempt to ensure that all scheduler filter failures have a"},{"line_number":64,"context_line":"useful log message describing why the filter failed for that host."},{"line_number":65,"context_line":""},{"line_number":66,"context_line":"Second, when the log level is set higher than \"debug\", we will accumulate"},{"line_number":67,"context_line":"debug logs in memory and then emit them only if the overall scheduling"},{"line_number":68,"context_line":"operation fails (i.e. exception.NoValidHost is raised).  Since the logging"},{"line_number":69,"context_line":"would happen only on failure, it is proposed that we log at the \"info\" level"},{"line_number":70,"context_line":"rather than the \"debug\" level.  When the log level is set to \"debug\", we will"},{"line_number":71,"context_line":"emit logs for the success case as well, but subject to the suppression filter"},{"line_number":72,"context_line":"specified below."},{"line_number":73,"context_line":""},{"line_number":74,"context_line":"In order to further reduce the logging, it is proposed that a new"},{"line_number":75,"context_line":"\"scheduler_suppress_log_filters\" config option be added.  This will consist of"}],"source_content_type":"text/x-rst","patch_set":7,"id":"1a122d0e_0be07f95","line":72,"range":{"start_line":66,"start_character":0,"end_line":72,"end_character":16},"in_reply_to":"1a122d0e_c07146e0","updated":"2016-05-04 16:19:38.000000000","message":"Nope. :)  Our current implementation passes in a dict as an extra argument, but that\u0027s hacky.  The discussions I had with johnthetubaguy at the summit basically settled on a scheduler-filter-specific logging function.  Basically if the underlying logging level was set to debug it would emit the logs as usual, while if it as set higher it would store the logs in memory and then emit them on NoValidHost or else throw them out on successful selection of a host.\n\nLonger term we had talked about adding something to oslo that would let you do this more generically, where you could specify the beginning of an \"interesting\" code block, then accumulate debug logs, and at the end of the \"interesting\" code block you either emit all the logs (on failure) or throw them out (on success).","commit_id":"9b600b58da53198468622a5c4486c4be99faa918"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"8dff7dc0513f37cbb44725a10b57daa68e6695d4","unresolved":false,"context_lines":[{"line_number":126,"context_line":"Developer impact"},{"line_number":127,"context_line":"----------------"},{"line_number":128,"context_line":""},{"line_number":129,"context_line":"There is a proposal to convert some of the simpler scheduler filters to pure DB"},{"line_number":130,"context_line":"queries.  In that case there is less opportunity for useful error messages.  On"},{"line_number":131,"context_line":"the other hand, these are most likely to be simpler filters which might not"},{"line_number":132,"context_line":"have had the additional logging enabled in the first place, so I think that the"},{"line_number":133,"context_line":"proposal still holds value even if some filters would not get the extra"},{"line_number":134,"context_line":"logging."},{"line_number":135,"context_line":""},{"line_number":136,"context_line":""},{"line_number":137,"context_line":"Implementation"}],"source_content_type":"text/x-rst","patch_set":7,"id":"1a122d0e_a0fd2a70","line":134,"range":{"start_line":129,"start_character":0,"end_line":134,"end_character":8},"updated":"2016-05-04 15:55:30.000000000","message":"A link to this proposal could be helpful?\n\nAlso, IIRC all filters will be converted to DB queries at some point (including NUMA, CPU pinning etc.) once we get the nested resources for resource pools [1]\n\n[1] https://review.openstack.org/#/c/300176/","commit_id":"9b600b58da53198468622a5c4486c4be99faa918"},{"author":{"_account_id":15334,"name":"Stephen Finucane","display_name":"stephenfin","email":"stephenfin@redhat.com","username":"sfinucan"},"change_message_id":"9d5ae9741a3992fd75b4ee4fbdc8d1626e2872d2","unresolved":false,"context_lines":[{"line_number":126,"context_line":"Developer impact"},{"line_number":127,"context_line":"----------------"},{"line_number":128,"context_line":""},{"line_number":129,"context_line":"There is a proposal to convert some of the simpler scheduler filters to pure DB"},{"line_number":130,"context_line":"queries.  In that case there is less opportunity for useful error messages.  On"},{"line_number":131,"context_line":"the other hand, these are most likely to be simpler filters which might not"},{"line_number":132,"context_line":"have had the additional logging enabled in the first place, so I think that the"},{"line_number":133,"context_line":"proposal still holds value even if some filters would not get the extra"},{"line_number":134,"context_line":"logging."},{"line_number":135,"context_line":""},{"line_number":136,"context_line":""},{"line_number":137,"context_line":"Implementation"}],"source_content_type":"text/x-rst","patch_set":7,"id":"1a122d0e_5a2a3ee8","line":134,"range":{"start_line":129,"start_character":0,"end_line":134,"end_character":8},"in_reply_to":"1a122d0e_0646b64e","updated":"2016-05-04 16:41:10.000000000","message":"Thanks, and agreed","commit_id":"9b600b58da53198468622a5c4486c4be99faa918"},{"author":{"_account_id":8768,"name":"Chris Friesen","email":"chris.friesen@windriver.com","username":"cbf123"},"change_message_id":"07f5bb268893af8fa410a641bc37bff7d8fa9be3","unresolved":false,"context_lines":[{"line_number":126,"context_line":"Developer impact"},{"line_number":127,"context_line":"----------------"},{"line_number":128,"context_line":""},{"line_number":129,"context_line":"There is a proposal to convert some of the simpler scheduler filters to pure DB"},{"line_number":130,"context_line":"queries.  In that case there is less opportunity for useful error messages.  On"},{"line_number":131,"context_line":"the other hand, these are most likely to be simpler filters which might not"},{"line_number":132,"context_line":"have had the additional logging enabled in the first place, so I think that the"},{"line_number":133,"context_line":"proposal still holds value even if some filters would not get the extra"},{"line_number":134,"context_line":"logging."},{"line_number":135,"context_line":""},{"line_number":136,"context_line":""},{"line_number":137,"context_line":"Implementation"}],"source_content_type":"text/x-rst","patch_set":7,"id":"1a122d0e_73cc85bb","line":134,"range":{"start_line":129,"start_character":0,"end_line":134,"end_character":8},"in_reply_to":"1a122d0e_5a2a3ee8","updated":"2016-05-04 17:26:07.000000000","message":"Based on Jay\u0027s comments at https://review.openstack.org/#/c/300178/ it seems that he is not intending to replace all the filters:  \"The plan here is to replace simple filters, one at a time.\"","commit_id":"9b600b58da53198468622a5c4486c4be99faa918"},{"author":{"_account_id":8768,"name":"Chris Friesen","email":"chris.friesen@windriver.com","username":"cbf123"},"change_message_id":"0a31a4f56f7c1ae00a24346c9cc60b08923a7edd","unresolved":false,"context_lines":[{"line_number":126,"context_line":"Developer impact"},{"line_number":127,"context_line":"----------------"},{"line_number":128,"context_line":""},{"line_number":129,"context_line":"There is a proposal to convert some of the simpler scheduler filters to pure DB"},{"line_number":130,"context_line":"queries.  In that case there is less opportunity for useful error messages.  On"},{"line_number":131,"context_line":"the other hand, these are most likely to be simpler filters which might not"},{"line_number":132,"context_line":"have had the additional logging enabled in the first place, so I think that the"},{"line_number":133,"context_line":"proposal still holds value even if some filters would not get the extra"},{"line_number":134,"context_line":"logging."},{"line_number":135,"context_line":""},{"line_number":136,"context_line":""},{"line_number":137,"context_line":"Implementation"}],"source_content_type":"text/x-rst","patch_set":7,"id":"1a122d0e_0646b64e","line":134,"range":{"start_line":129,"start_character":0,"end_line":134,"end_character":8},"in_reply_to":"1a122d0e_a0fd2a70","updated":"2016-05-04 16:19:38.000000000","message":"I\u0027ll try and track down the DB query link, and will double-check about the more complicated filters being converted to DB queries.  (Though if that does happen I think it\u0027ll be a while, and we\u0027ll still want a way to get detailed logs.)","commit_id":"9b600b58da53198468622a5c4486c4be99faa918"},{"author":{"_account_id":1063,"name":"Ed Leafe","email":"ed@leafe.com","username":"ed-leafe"},"change_message_id":"3c1a033c84cd1ad25dce540f66a546b5eb826a65","unresolved":false,"context_lines":[{"line_number":75,"context_line":"suppression filter specified below."},{"line_number":76,"context_line":""},{"line_number":77,"context_line":"In order to further reduce the logging, it is proposed that a new"},{"line_number":78,"context_line":"\"scheduler_suppress_log_filters\" config option be added.  This will consist of"},{"line_number":79,"context_line":"a list of filter names (analogous to \"scheduler_default_filters\") for which we"},{"line_number":80,"context_line":"want to suppress logging.  In this way operators could avoid logging for cases"},{"line_number":81,"context_line":"which fail for trivial reasons like lack of CPU/RAM/disk.  I propose that this"}],"source_content_type":"text/x-rst","patch_set":8,"id":"1a122d0e_b2b0ce8c","line":78,"range":{"start_line":78,"start_character":1,"end_line":78,"end_character":31},"updated":"2016-05-04 21:32:58.000000000","message":"Bikeshedding, perhaps, but this name sounds like it should be a boolean: do we suppress log filters, or not? Maybe as trivial a change as \"scheduler_suppressed_log_filters\"?","commit_id":"2045057d344ccd77ed1989207103ddfcd7b85454"},{"author":{"_account_id":8768,"name":"Chris Friesen","email":"chris.friesen@windriver.com","username":"cbf123"},"change_message_id":"06d2919fc0e59a97f07c7b8bf3b89865e9af7301","unresolved":false,"context_lines":[{"line_number":75,"context_line":"suppression filter specified below."},{"line_number":76,"context_line":""},{"line_number":77,"context_line":"In order to further reduce the logging, it is proposed that a new"},{"line_number":78,"context_line":"\"scheduler_suppress_log_filters\" config option be added.  This will consist of"},{"line_number":79,"context_line":"a list of filter names (analogous to \"scheduler_default_filters\") for which we"},{"line_number":80,"context_line":"want to suppress logging.  In this way operators could avoid logging for cases"},{"line_number":81,"context_line":"which fail for trivial reasons like lack of CPU/RAM/disk.  I propose that this"}],"source_content_type":"text/x-rst","patch_set":8,"id":"1a122d0e_665e5714","line":78,"range":{"start_line":78,"start_character":1,"end_line":78,"end_character":31},"in_reply_to":"1a122d0e_b2b0ce8c","updated":"2016-05-05 15:23:49.000000000","message":"Sure, that makes sense.","commit_id":"2045057d344ccd77ed1989207103ddfcd7b85454"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"3315543f4925562e6d022e231941f8d5386c2dec","unresolved":false,"context_lines":[{"line_number":78,"context_line":"\"scheduler_suppress_log_filters\" config option be added.  This will consist of"},{"line_number":79,"context_line":"a list of filter names (analogous to \"scheduler_default_filters\") for which we"},{"line_number":80,"context_line":"want to suppress logging.  In this way operators could avoid logging for cases"},{"line_number":81,"context_line":"which fail for trivial reasons like lack of CPU/RAM/disk.  I propose that this"},{"line_number":82,"context_line":"option default to the empty list, so that deployers would initially get failure"},{"line_number":83,"context_line":"logs for all filters."},{"line_number":84,"context_line":""},{"line_number":85,"context_line":"Alternatives"},{"line_number":86,"context_line":"------------"}],"source_content_type":"text/x-rst","patch_set":8,"id":"1a122d0e_290c3041","line":83,"range":{"start_line":81,"start_character":59,"end_line":83,"end_character":21},"updated":"2016-05-05 15:48:05.000000000","message":"Just to clarify my understanding, if I have debug logging enabled and haven\u0027t changed this default, it\u0027s going to always dump these logs even on success? I thought there was concern about the performance impact of this?","commit_id":"2045057d344ccd77ed1989207103ddfcd7b85454"},{"author":{"_account_id":8768,"name":"Chris Friesen","email":"chris.friesen@windriver.com","username":"cbf123"},"change_message_id":"41ccefcb411744646f38e0fbaa684d80a4b7ef16","unresolved":false,"context_lines":[{"line_number":78,"context_line":"\"scheduler_suppress_log_filters\" config option be added.  This will consist of"},{"line_number":79,"context_line":"a list of filter names (analogous to \"scheduler_default_filters\") for which we"},{"line_number":80,"context_line":"want to suppress logging.  In this way operators could avoid logging for cases"},{"line_number":81,"context_line":"which fail for trivial reasons like lack of CPU/RAM/disk.  I propose that this"},{"line_number":82,"context_line":"option default to the empty list, so that deployers would initially get failure"},{"line_number":83,"context_line":"logs for all filters."},{"line_number":84,"context_line":""},{"line_number":85,"context_line":"Alternatives"},{"line_number":86,"context_line":"------------"}],"source_content_type":"text/x-rst","patch_set":8,"id":"1a122d0e_64512f30","line":83,"range":{"start_line":81,"start_character":59,"end_line":83,"end_character":21},"in_reply_to":"1a122d0e_290c3041","updated":"2016-05-05 16:08:34.000000000","message":"There were a number of operators that indicated that they wanted to have access to *all* the debug logs including the success path ones.\n\nUnder the current proposal if you have debug logging enabled and this feature lands, you will see no immediate change.  However, you\u0027ll be able to add scheduler filters to the \"scheduler_suppress_log_filters\" list to turn off logging for those filters.\n\nThe alternative would be to add a second flag (\"sched_debug_log_only_on_failure\" or something like that) controlling whether to log the filter debug logs always or only on failure.  This would allow debug logging in the scheduler but with the filter logs being emitted only on failure.  I\u0027m open to this option too...it would give a bit more flexibility.","commit_id":"2045057d344ccd77ed1989207103ddfcd7b85454"},{"author":{"_account_id":2271,"name":"Michael Still","email":"mikal@stillhq.com","username":"mikalstill"},"change_message_id":"51c32086434ab4019f9a43bf4aec7f9e49fade41","unresolved":false,"context_lines":[{"line_number":119,"context_line":"------------------"},{"line_number":120,"context_line":""},{"line_number":121,"context_line":"Minimal.  There will be some additional CPU time spent processing logs and some"},{"line_number":122,"context_line":"additional memory consumed during scheduling."},{"line_number":123,"context_line":""},{"line_number":124,"context_line":"Other deployer impact"},{"line_number":125,"context_line":"---------------------"}],"source_content_type":"text/x-rst","patch_set":8,"id":"1a122d0e_d55baa52","line":122,"updated":"2016-05-05 04:58:13.000000000","message":"Is that really true? I know that Rackspace had to turn off debug logging for some filters because of performance problems.\n\nThat said, I think we should still do this, but I do think that we should acknowledge it probably has a performance impact.","commit_id":"2045057d344ccd77ed1989207103ddfcd7b85454"},{"author":{"_account_id":8768,"name":"Chris Friesen","email":"chris.friesen@windriver.com","username":"cbf123"},"change_message_id":"06d2919fc0e59a97f07c7b8bf3b89865e9af7301","unresolved":false,"context_lines":[{"line_number":119,"context_line":"------------------"},{"line_number":120,"context_line":""},{"line_number":121,"context_line":"Minimal.  There will be some additional CPU time spent processing logs and some"},{"line_number":122,"context_line":"additional memory consumed during scheduling."},{"line_number":123,"context_line":""},{"line_number":124,"context_line":"Other deployer impact"},{"line_number":125,"context_line":"---------------------"}],"source_content_type":"text/x-rst","patch_set":8,"id":"1a122d0e_c612cb60","line":122,"in_reply_to":"1a122d0e_d55baa52","updated":"2016-05-05 15:23:49.000000000","message":"I must confess that I don\u0027t have hard data.  The \"minimal\" is under the assumption that the simple filters are added to the \"suppressed log filters\" list.  If we were to log for all filters then maybe the hit would be significant.  I can add a qualifier to the above statement.","commit_id":"2045057d344ccd77ed1989207103ddfcd7b85454"},{"author":{"_account_id":782,"name":"John Garbutt","email":"john@johngarbutt.com","username":"johngarbutt"},"change_message_id":"b7d1e4264c4960620c731e9efb6e7a8e053d3642","unresolved":false,"context_lines":[{"line_number":119,"context_line":"------------------"},{"line_number":120,"context_line":""},{"line_number":121,"context_line":"Minimal.  There will be some additional CPU time spent processing logs and some"},{"line_number":122,"context_line":"additional memory consumed during scheduling."},{"line_number":123,"context_line":""},{"line_number":124,"context_line":"Other deployer impact"},{"line_number":125,"context_line":"---------------------"}],"source_content_type":"text/x-rst","patch_set":8,"id":"1a122d0e_c83784fc","line":122,"in_reply_to":"1a122d0e_d55baa52","updated":"2016-05-05 11:50:06.000000000","message":"Just to be clear, I pushed those log fixes upstream, and they are merged.\n\nWe used to log in positive cases, thats bad for performance.\n\nIf your RAM filter gets rid of most hosts in the system, the follow on filters can log and make little impact.\n\nThe issue is logging for every host, for each filter, on every build request. So I think this is OK.","commit_id":"2045057d344ccd77ed1989207103ddfcd7b85454"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"3315543f4925562e6d022e231941f8d5386c2dec","unresolved":false,"context_lines":[{"line_number":131,"context_line":""},{"line_number":132,"context_line":"There is a proposal [1]_ to convert some of the simpler scheduler filters to"},{"line_number":133,"context_line":"pure DB queries.  In that case there is less opportunity for useful error"},{"line_number":134,"context_line":"messages.  On the other hand, these are most likely to be simpler filters which"},{"line_number":135,"context_line":"might not have had the additional logging enabled in the first place, so I"},{"line_number":136,"context_line":"think that the proposal still holds value even if some filters would not get"},{"line_number":137,"context_line":"the extra logging."},{"line_number":138,"context_line":""}],"source_content_type":"text/x-rst","patch_set":8,"id":"1a122d0e_e4f37f07","line":135,"range":{"start_line":134,"start_character":11,"end_line":135,"end_character":68},"updated":"2016-05-05 15:48:05.000000000","message":"This statement plus the above reply:\n\n\"The \"minimal\" is under the assumption that the simple filters are added to the \"suppressed log filters\" list.\"\n\nMakes me think that the default list of suppressed filters should be the simple filters. But maybe the default list depends on what your logging level is set to? I might be confusing the actual use cases/scenarios of how these would be set and what would happen when I have a failure vs success and debug logging enabled vs not, and what the suppression list is. It could be useful to case some of those out in the spec so we\u0027re all on the same page with how you expect to implement this.","commit_id":"2045057d344ccd77ed1989207103ddfcd7b85454"},{"author":{"_account_id":8688,"name":"Alexis Lee","email":"openstack@lxsli.co.uk","username":"lxsli"},"change_message_id":"2ed4e7319c4fac8eeddd5242bfd1e1bf05a02a1f","unresolved":false,"context_lines":[{"line_number":64,"context_line":"useful log message describing why the filter failed for that host."},{"line_number":65,"context_line":""},{"line_number":66,"context_line":"Second, when the log level is set higher than \"debug\", we will accumulate"},{"line_number":67,"context_line":"debug logs in memory (the current plan is to use a sched-filter-specific"},{"line_number":68,"context_line":"logging mechanism) and then emit them only if the overall scheduling"},{"line_number":69,"context_line":"operation fails (i.e. exception.NoValidHost is raised).  If the scheduling"},{"line_number":70,"context_line":"operation passes, the logs will be thrown out.  Since the logging would happen"}],"source_content_type":"text/x-rst","patch_set":9,"id":"1a122d0e_1f2242b9","line":67,"updated":"2016-05-05 16:19:46.000000000","message":"I have/had a spec to do something like this in oslo.log. This is a generally useful thing so if you code it, please either do so in oslo.log or at least have half an eye on making it reusable? Thank you!\n\nhttps://review.openstack.org/#/c/227766/","commit_id":"5ff3e9346958bc3d11ecb736d7c1eec262703a7b"},{"author":{"_account_id":8688,"name":"Alexis Lee","email":"openstack@lxsli.co.uk","username":"lxsli"},"change_message_id":"2ed4e7319c4fac8eeddd5242bfd1e1bf05a02a1f","unresolved":false,"context_lines":[{"line_number":71,"context_line":"only on failure, it is proposed that we log the detailed filter logs at the"},{"line_number":72,"context_line":"\"info\" level rather than the \"debug\" level.  (The actual NoValidHost error log"},{"line_number":73,"context_line":"will still be logged at the current logging level.) When the log level is set"},{"line_number":74,"context_line":"to \"debug\", we will emit logs for the success case as well, but subject to the"},{"line_number":75,"context_line":"suppression filter specified below."},{"line_number":76,"context_line":""},{"line_number":77,"context_line":"In order to further reduce the logging, it is proposed that a new"},{"line_number":78,"context_line":"\"scheduler_suppressed_log_filters\" config option be added.  This will consist"}],"source_content_type":"text/x-rst","patch_set":9,"id":"1a122d0e_9a03000d","line":75,"range":{"start_line":74,"start_character":20,"end_line":75,"end_character":34},"updated":"2016-05-05 16:19:46.000000000","message":"this seems unnecessary, if it scheduled then I don\u0027t care","commit_id":"5ff3e9346958bc3d11ecb736d7c1eec262703a7b"},{"author":{"_account_id":8688,"name":"Alexis Lee","email":"openstack@lxsli.co.uk","username":"lxsli"},"change_message_id":"2ed4e7319c4fac8eeddd5242bfd1e1bf05a02a1f","unresolved":false,"context_lines":[{"line_number":75,"context_line":"suppression filter specified below."},{"line_number":76,"context_line":""},{"line_number":77,"context_line":"In order to further reduce the logging, it is proposed that a new"},{"line_number":78,"context_line":"\"scheduler_suppressed_log_filters\" config option be added.  This will consist"},{"line_number":79,"context_line":"of a list of filter names (analogous to \"scheduler_default_filters\") for which"},{"line_number":80,"context_line":"we want to suppress logging.  In this way operators could avoid logging for"},{"line_number":81,"context_line":"cases which fail for trivial reasons like lack of CPU/RAM/disk.  I propose that"}],"source_content_type":"text/x-rst","patch_set":9,"id":"1a122d0e_3f5e1e45","line":78,"range":{"start_line":78,"start_character":1,"end_line":78,"end_character":33},"updated":"2016-05-05 16:19:46.000000000","message":"Will this suppress all failure logging or only the new, detailed logging for each filter?","commit_id":"5ff3e9346958bc3d11ecb736d7c1eec262703a7b"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"c6014f12a3eccfe71f05b9219eff1c740ef02fea","unresolved":false,"context_lines":[{"line_number":118,"context_line":"Performance Impact"},{"line_number":119,"context_line":"------------------"},{"line_number":120,"context_line":""},{"line_number":121,"context_line":"Expected to be low, assuming the simple filters (CPU/RAM/disk/etc.) have"},{"line_number":122,"context_line":"their logs suppressed.  There will be some additional CPU time spent"},{"line_number":123,"context_line":"processing logs and some additional memory consumed during scheduling."},{"line_number":124,"context_line":""},{"line_number":125,"context_line":"Other deployer impact"}],"source_content_type":"text/x-rst","patch_set":9,"id":"1a122d0e_84728b48","line":122,"range":{"start_line":121,"start_character":0,"end_line":122,"end_character":22},"updated":"2016-05-05 15:51:53.000000000","message":"It really sounds like these should be suppressed by default to avoid noise, and if you want them you enable them (remove from the suppression list). Although I guess it\u0027s a trade off because maybe by default I want debuggability over performance, and if I don\u0027t find these logs helpful then I consciously suppress them.","commit_id":"5ff3e9346958bc3d11ecb736d7c1eec262703a7b"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"ec14428e19d374d83d59fdcd13e2cd95219d01fb","unresolved":false,"context_lines":[{"line_number":61,"context_line":"As discussed at the Summit, the proposal is two-pronged."},{"line_number":62,"context_line":""},{"line_number":63,"context_line":"First, we will attempt to ensure that all scheduler filter failures have a"},{"line_number":64,"context_line":"useful log message describing why the filter failed for that host."},{"line_number":65,"context_line":""},{"line_number":66,"context_line":"Second, we propose adding a new \"sched_detailed_log_only_on_failure\" option,"},{"line_number":67,"context_line":"which will default to True.  When this option is set, we will accumulate"}],"source_content_type":"text/x-rst","patch_set":10,"id":"dab17558_4e646be3","line":64,"updated":"2016-05-09 14:39:45.000000000","message":"that sounds boiling the ocean, but okay.","commit_id":"11d34368ec4016d430e816e0b1203fc502567d30"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"ec14428e19d374d83d59fdcd13e2cd95219d01fb","unresolved":false,"context_lines":[{"line_number":149,"context_line":"think that the proposal still holds value even if some filters would not get"},{"line_number":150,"context_line":"the extra logging."},{"line_number":151,"context_line":""},{"line_number":152,"context_line":".. [1] https://blueprints.launchpad.net/nova/+spec/resource-providers-scheduler-db-filters"},{"line_number":153,"context_line":""},{"line_number":154,"context_line":""},{"line_number":155,"context_line":""}],"source_content_type":"text/x-rst","patch_set":10,"id":"dab17558_2e48075b","line":152,"updated":"2016-05-09 14:39:45.000000000","message":"Well, I wouldn\u0027t mention that yet unapproved spec which would confuse operators reading your prose, given it would create some unnecessary confusion that the proposal you mention is agreed, which is not the case atm.","commit_id":"11d34368ec4016d430e816e0b1203fc502567d30"},{"author":{"_account_id":8768,"name":"Chris Friesen","email":"chris.friesen@windriver.com","username":"cbf123"},"change_message_id":"666ed93cc47d854e2774e6a49e01271a7bccb150","unresolved":false,"context_lines":[{"line_number":149,"context_line":"think that the proposal still holds value even if some filters would not get"},{"line_number":150,"context_line":"the extra logging."},{"line_number":151,"context_line":""},{"line_number":152,"context_line":".. [1] https://blueprints.launchpad.net/nova/+spec/resource-providers-scheduler-db-filters"},{"line_number":153,"context_line":""},{"line_number":154,"context_line":""},{"line_number":155,"context_line":""}],"source_content_type":"text/x-rst","patch_set":10,"id":"dab17558_2ec64248","line":152,"in_reply_to":"dab17558_2e48075b","updated":"2016-05-09 16:17:36.000000000","message":"Okay, will remove.","commit_id":"11d34368ec4016d430e816e0b1203fc502567d30"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"792b870132c0c4ffdbc825a67f16780f972c584f","unresolved":false,"context_lines":[{"line_number":46,"context_line":"Or if the numa node doesn\u0027t have enough CPU/RAM to satisfy the requirements,"},{"line_number":47,"context_line":"we propose that the exact problem should be indicated in the log:"},{"line_number":48,"context_line":""},{"line_number":49,"context_line":"*\"%(host)s, %(node)s CPUs avail(%d) \u003c required(%d)\"*"},{"line_number":50,"context_line":""},{"line_number":51,"context_line":"Use Cases"},{"line_number":52,"context_line":"---------"}],"source_content_type":"text/x-rst","patch_set":11,"id":"dab17558_2bcac74f","line":49,"updated":"2016-05-13 14:16:51.000000000","message":"++","commit_id":"f9e3e35dc1f9ae3f7b0b01eecf54d706ef736b13"},{"author":{"_account_id":8688,"name":"Alexis Lee","email":"openstack@lxsli.co.uk","username":"lxsli"},"change_message_id":"697c66d9143e6bbe56cc0b47e7a7ded4ef50852b","unresolved":false,"context_lines":[{"line_number":61,"context_line":"As discussed at the Summit, the proposal is two-pronged."},{"line_number":62,"context_line":""},{"line_number":63,"context_line":"First, we will attempt to ensure that all scheduler filter failures have a"},{"line_number":64,"context_line":"useful log message describing why the filter failed for that host."},{"line_number":65,"context_line":""},{"line_number":66,"context_line":"Second, we propose adding a new \"sched_detailed_log_only_on_failure\" option,"},{"line_number":67,"context_line":"which will default to True.  When this option is set, we will accumulate"}],"source_content_type":"text/x-rst","patch_set":11,"id":"dab17558_a202436d","line":64,"updated":"2016-05-09 17:58:16.000000000","message":"could use some more detail here; there\u0027s much more text about the performance mitigations than this which I think is meant to be the main feature. EG which filters besides the NUMAFilter need work?","commit_id":"f9e3e35dc1f9ae3f7b0b01eecf54d706ef736b13"},{"author":{"_account_id":782,"name":"John Garbutt","email":"john@johngarbutt.com","username":"johngarbutt"},"change_message_id":"f0f22a6b49cfadf34cf2f14fc07f8d0568ddf249","unresolved":false,"context_lines":[{"line_number":61,"context_line":"As discussed at the Summit, the proposal is two-pronged."},{"line_number":62,"context_line":""},{"line_number":63,"context_line":"First, we will attempt to ensure that all scheduler filter failures have a"},{"line_number":64,"context_line":"useful log message describing why the filter failed for that host."},{"line_number":65,"context_line":""},{"line_number":66,"context_line":"Second, we propose adding a new \"sched_detailed_log_only_on_failure\" option,"},{"line_number":67,"context_line":"which will default to True.  When this option is set, we will accumulate"}],"source_content_type":"text/x-rst","patch_set":11,"id":"bab6814e_9e00be03","line":64,"in_reply_to":"dab17558_336ea795","updated":"2016-05-23 13:13:48.000000000","message":"I am OK with just the direction here. We establish the intent nicely.","commit_id":"f9e3e35dc1f9ae3f7b0b01eecf54d706ef736b13"},{"author":{"_account_id":8768,"name":"Chris Friesen","email":"chris.friesen@windriver.com","username":"cbf123"},"change_message_id":"6f630efc022e9d63bb06026ebbef013f961ee0e9","unresolved":false,"context_lines":[{"line_number":61,"context_line":"As discussed at the Summit, the proposal is two-pronged."},{"line_number":62,"context_line":""},{"line_number":63,"context_line":"First, we will attempt to ensure that all scheduler filter failures have a"},{"line_number":64,"context_line":"useful log message describing why the filter failed for that host."},{"line_number":65,"context_line":""},{"line_number":66,"context_line":"Second, we propose adding a new \"sched_detailed_log_only_on_failure\" option,"},{"line_number":67,"context_line":"which will default to True.  When this option is set, we will accumulate"}],"source_content_type":"text/x-rst","patch_set":11,"id":"dab17558_336ea795","line":64,"in_reply_to":"dab17558_3857a4cf","updated":"2016-05-09 19:13:27.000000000","message":"Why?  The changes to each filter are essentially standalone and independent from any other work, and in most cases they\u0027re trivial.  If reviewers don\u0027t think that changes to a given filter are warranted they can downvote (or ignore) the changes for that filter.  I don\u0027t think it makes sense to exhaustively itemize all planned changes here.\n\nTechnically these changes could be proposed without any blueprint or spec at all...see https://review.openstack.org/#/c/285322 for example.","commit_id":"f9e3e35dc1f9ae3f7b0b01eecf54d706ef736b13"},{"author":{"_account_id":8768,"name":"Chris Friesen","email":"chris.friesen@windriver.com","username":"cbf123"},"change_message_id":"58b548996c06b2687f991007bfbaecc5b763d22a","unresolved":false,"context_lines":[{"line_number":61,"context_line":"As discussed at the Summit, the proposal is two-pronged."},{"line_number":62,"context_line":""},{"line_number":63,"context_line":"First, we will attempt to ensure that all scheduler filter failures have a"},{"line_number":64,"context_line":"useful log message describing why the filter failed for that host."},{"line_number":65,"context_line":""},{"line_number":66,"context_line":"Second, we propose adding a new \"sched_detailed_log_only_on_failure\" option,"},{"line_number":67,"context_line":"which will default to True.  When this option is set, we will accumulate"}],"source_content_type":"text/x-rst","patch_set":11,"id":"dab17558_ddac8a37","line":64,"in_reply_to":"dab17558_a202436d","updated":"2016-05-09 18:34:17.000000000","message":"I don\u0027t think this needs more detail...there\u0027s nothing tricky about it.  We just need to go through the filters and add logs or detail where needed.  As some simple examples, there are a number of filters in affinity_filter.py which fail silently, AggregateMultiTenancyIsolation doesn\u0027t log the tenant id which failed or which were configured on the aggregate, ImagePropertiesFilter doesn\u0027t log the requested image properties, etc.","commit_id":"f9e3e35dc1f9ae3f7b0b01eecf54d706ef736b13"},{"author":{"_account_id":8688,"name":"Alexis Lee","email":"openstack@lxsli.co.uk","username":"lxsli"},"change_message_id":"0737b0b0ff30a1acfd2c902a6441393da7d629a5","unresolved":false,"context_lines":[{"line_number":61,"context_line":"As discussed at the Summit, the proposal is two-pronged."},{"line_number":62,"context_line":""},{"line_number":63,"context_line":"First, we will attempt to ensure that all scheduler filter failures have a"},{"line_number":64,"context_line":"useful log message describing why the filter failed for that host."},{"line_number":65,"context_line":""},{"line_number":66,"context_line":"Second, we propose adding a new \"sched_detailed_log_only_on_failure\" option,"},{"line_number":67,"context_line":"which will default to True.  When this option is set, we will accumulate"}],"source_content_type":"text/x-rst","patch_set":11,"id":"dab17558_3857a4cf","line":64,"in_reply_to":"dab17558_ddac8a37","updated":"2016-05-09 18:44:58.000000000","message":"My point is that an \"etc\" isn\u0027t good enough. Reviewers need to understand how extensive the changes will be and how much work will be involved.","commit_id":"f9e3e35dc1f9ae3f7b0b01eecf54d706ef736b13"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"792b870132c0c4ffdbc825a67f16780f972c584f","unresolved":false,"context_lines":[{"line_number":63,"context_line":"First, we will attempt to ensure that all scheduler filter failures have a"},{"line_number":64,"context_line":"useful log message describing why the filter failed for that host."},{"line_number":65,"context_line":""},{"line_number":66,"context_line":"Second, we propose adding a new \"sched_detailed_log_only_on_failure\" option,"},{"line_number":67,"context_line":"which will default to True.  When this option is set, we will accumulate"},{"line_number":68,"context_line":"debug logs in memory (the current plan is to use a sched-filter-specific"},{"line_number":69,"context_line":"logging mechanism, but we could likely use"}],"source_content_type":"text/x-rst","patch_set":11,"id":"dab17558_0bc5433c","line":66,"updated":"2016-05-13 14:16:51.000000000","message":"nit: Please name the config option log_failure_conditions and put it in a [scheduler] configuration section instead of prefixing it with sched_","commit_id":"f9e3e35dc1f9ae3f7b0b01eecf54d706ef736b13"},{"author":{"_account_id":782,"name":"John Garbutt","email":"john@johngarbutt.com","username":"johngarbutt"},"change_message_id":"f0f22a6b49cfadf34cf2f14fc07f8d0568ddf249","unresolved":false,"context_lines":[{"line_number":63,"context_line":"First, we will attempt to ensure that all scheduler filter failures have a"},{"line_number":64,"context_line":"useful log message describing why the filter failed for that host."},{"line_number":65,"context_line":""},{"line_number":66,"context_line":"Second, we propose adding a new \"sched_detailed_log_only_on_failure\" option,"},{"line_number":67,"context_line":"which will default to True.  When this option is set, we will accumulate"},{"line_number":68,"context_line":"debug logs in memory (the current plan is to use a sched-filter-specific"},{"line_number":69,"context_line":"logging mechanism, but we could likely use"}],"source_content_type":"text/x-rst","patch_set":11,"id":"bab6814e_51112751","line":66,"in_reply_to":"bab6814e_3618e3d2","updated":"2016-05-23 13:13:48.000000000","message":"+1 both comments.","commit_id":"f9e3e35dc1f9ae3f7b0b01eecf54d706ef736b13"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"25833c5faf1a889c53d45cebb2c73bd9c1e82c62","unresolved":false,"context_lines":[{"line_number":63,"context_line":"First, we will attempt to ensure that all scheduler filter failures have a"},{"line_number":64,"context_line":"useful log message describing why the filter failed for that host."},{"line_number":65,"context_line":""},{"line_number":66,"context_line":"Second, we propose adding a new \"sched_detailed_log_only_on_failure\" option,"},{"line_number":67,"context_line":"which will default to True.  When this option is set, we will accumulate"},{"line_number":68,"context_line":"debug logs in memory (the current plan is to use a sched-filter-specific"},{"line_number":69,"context_line":"logging mechanism, but we could likely use"}],"source_content_type":"text/x-rst","patch_set":11,"id":"bab6814e_3618e3d2","line":66,"in_reply_to":"dab17558_0bc5433c","updated":"2016-05-23 10:29:37.000000000","message":"Agreed, but that\u0027s something we can just nitpick during the implementation phase.","commit_id":"f9e3e35dc1f9ae3f7b0b01eecf54d706ef736b13"},{"author":{"_account_id":1063,"name":"Ed Leafe","email":"ed@leafe.com","username":"ed-leafe"},"change_message_id":"4b7ee17ea6c5a97a43411b72d9ae8649bdc46479","unresolved":false,"context_lines":[{"line_number":64,"context_line":"useful log message describing why the filter failed for that host."},{"line_number":65,"context_line":""},{"line_number":66,"context_line":"Second, we propose adding a new \"sched_detailed_log_only_on_failure\" option,"},{"line_number":67,"context_line":"which will default to True.  When this option is set, we will accumulate"},{"line_number":68,"context_line":"debug logs in memory (the current plan is to use a sched-filter-specific"},{"line_number":69,"context_line":"logging mechanism, but we could likely use"},{"line_number":70,"context_line":"https://review.openstack.org/#/c/227766/ or similar if it gets merged) and"}],"source_content_type":"text/x-rst","patch_set":11,"id":"dab17558_0b5ff541","line":67,"range":{"start_line":67,"start_character":6,"end_line":67,"end_character":26},"updated":"2016-05-13 18:56:46.000000000","message":"Instead of a boolean, why not have a \"detailed_log\" option in the [scheduler] section, with possible values of [\"ALWAYS\", \"FAILURE\", \"OFF\"]. This will allow adding these logs even on success, since the request to see details of scheduler choices even when a host is selected, as operators might want to know why Host A was chosen over Host B. Then unless it\u0027s set to \"OFF\", we accumulate debug logs in memory, and if it\u0027s \"FAILURE\", only emit them on NoValidHost.","commit_id":"f9e3e35dc1f9ae3f7b0b01eecf54d706ef736b13"},{"author":{"_account_id":782,"name":"John Garbutt","email":"john@johngarbutt.com","username":"johngarbutt"},"change_message_id":"f0f22a6b49cfadf34cf2f14fc07f8d0568ddf249","unresolved":false,"context_lines":[{"line_number":64,"context_line":"useful log message describing why the filter failed for that host."},{"line_number":65,"context_line":""},{"line_number":66,"context_line":"Second, we propose adding a new \"sched_detailed_log_only_on_failure\" option,"},{"line_number":67,"context_line":"which will default to True.  When this option is set, we will accumulate"},{"line_number":68,"context_line":"debug logs in memory (the current plan is to use a sched-filter-specific"},{"line_number":69,"context_line":"logging mechanism, but we could likely use"},{"line_number":70,"context_line":"https://review.openstack.org/#/c/227766/ or similar if it gets merged) and"}],"source_content_type":"text/x-rst","patch_set":11,"id":"bab6814e_d1a077f6","line":67,"range":{"start_line":67,"start_character":6,"end_line":67,"end_character":26},"in_reply_to":"bab6814e_164e7fc0","updated":"2016-05-23 13:13:48.000000000","message":"Lets focus on the case people are all super worried about, failures.","commit_id":"f9e3e35dc1f9ae3f7b0b01eecf54d706ef736b13"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"25833c5faf1a889c53d45cebb2c73bd9c1e82c62","unresolved":false,"context_lines":[{"line_number":64,"context_line":"useful log message describing why the filter failed for that host."},{"line_number":65,"context_line":""},{"line_number":66,"context_line":"Second, we propose adding a new \"sched_detailed_log_only_on_failure\" option,"},{"line_number":67,"context_line":"which will default to True.  When this option is set, we will accumulate"},{"line_number":68,"context_line":"debug logs in memory (the current plan is to use a sched-filter-specific"},{"line_number":69,"context_line":"logging mechanism, but we could likely use"},{"line_number":70,"context_line":"https://review.openstack.org/#/c/227766/ or similar if it gets merged) and"}],"source_content_type":"text/x-rst","patch_set":11,"id":"bab6814e_164e7fc0","line":67,"range":{"start_line":67,"start_character":6,"end_line":67,"end_character":26},"in_reply_to":"bab6814e_8b18c576","updated":"2016-05-23 10:29:37.000000000","message":"Agreed with Sean here, we already have log levels for that specific usecase, please don\u0027t mix both.","commit_id":"f9e3e35dc1f9ae3f7b0b01eecf54d706ef736b13"},{"author":{"_account_id":8768,"name":"Chris Friesen","email":"chris.friesen@windriver.com","username":"cbf123"},"change_message_id":"1aeb1646d33eb109fb105a881f6ee523807b4c86","unresolved":false,"context_lines":[{"line_number":64,"context_line":"useful log message describing why the filter failed for that host."},{"line_number":65,"context_line":""},{"line_number":66,"context_line":"Second, we propose adding a new \"sched_detailed_log_only_on_failure\" option,"},{"line_number":67,"context_line":"which will default to True.  When this option is set, we will accumulate"},{"line_number":68,"context_line":"debug logs in memory (the current plan is to use a sched-filter-specific"},{"line_number":69,"context_line":"logging mechanism, but we could likely use"},{"line_number":70,"context_line":"https://review.openstack.org/#/c/227766/ or similar if it gets merged) and"}],"source_content_type":"text/x-rst","patch_set":11,"id":"dab17558_593557d8","line":67,"range":{"start_line":67,"start_character":6,"end_line":67,"end_character":26},"in_reply_to":"dab17558_0b5ff541","updated":"2016-05-13 19:19:40.000000000","message":"Works for me.  Do we want the all-caps?","commit_id":"f9e3e35dc1f9ae3f7b0b01eecf54d706ef736b13"},{"author":{"_account_id":2750,"name":"Sean Dague","email":"sean@dague.net","username":"sdague"},"change_message_id":"35b37fa9fee64ff7c819051604e3dabb73a4c64a","unresolved":false,"context_lines":[{"line_number":64,"context_line":"useful log message describing why the filter failed for that host."},{"line_number":65,"context_line":""},{"line_number":66,"context_line":"Second, we propose adding a new \"sched_detailed_log_only_on_failure\" option,"},{"line_number":67,"context_line":"which will default to True.  When this option is set, we will accumulate"},{"line_number":68,"context_line":"debug logs in memory (the current plan is to use a sched-filter-specific"},{"line_number":69,"context_line":"logging mechanism, but we could likely use"},{"line_number":70,"context_line":"https://review.openstack.org/#/c/227766/ or similar if it gets merged) and"}],"source_content_type":"text/x-rst","patch_set":11,"id":"bab6814e_8b18c576","line":67,"range":{"start_line":67,"start_character":6,"end_line":67,"end_character":26},"in_reply_to":"dab17558_593557d8","updated":"2016-05-20 14:43:56.000000000","message":"Actually, I rather like the dedicated boolean, for the following reasons.\n\nLogging levels are on a different access. Whether we are logging at all at this axis is already controled.\n\nHaving a dedicated boolean means we don\u0027t end up in a non sensible state.","commit_id":"f9e3e35dc1f9ae3f7b0b01eecf54d706ef736b13"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"792b870132c0c4ffdbc825a67f16780f972c584f","unresolved":false,"context_lines":[{"line_number":69,"context_line":"logging mechanism, but we could likely use"},{"line_number":70,"context_line":"https://review.openstack.org/#/c/227766/ or similar if it gets merged) and"},{"line_number":71,"context_line":"then emit them only if the overall scheduling operation fails (i.e."},{"line_number":72,"context_line":"exception.NoValidHost is raised).  If the scheduling operation passes, the"},{"line_number":73,"context_line":"logs will be thrown out.  Since the logging would happen only on failure, it"},{"line_number":74,"context_line":"is proposed that we log the detailed filter logs at the \"info\" level."},{"line_number":75,"context_line":"(The actual NoValidHost error log will still be logged at the current logging"}],"source_content_type":"text/x-rst","patch_set":11,"id":"dab17558_6bc4cf3f","line":72,"updated":"2016-05-13 14:16:51.000000000","message":"@Yingxin: I would prefer a more readable and explicit debug log storage strategy than using recursion. Recursion tends to hide things in the call stack in non-obvious places, especially when dealing with nested/recursive exception handling, making the code hard to understand.","commit_id":"f9e3e35dc1f9ae3f7b0b01eecf54d706ef736b13"},{"author":{"_account_id":15648,"name":"Yingxin","email":"yingxin.cheng@intel.com","username":"Yingxin"},"change_message_id":"95b7762b83903ecf949448e2c6c3305a235f547a","unresolved":false,"context_lines":[{"line_number":68,"context_line":"debug logs in memory (the current plan is to use a sched-filter-specific"},{"line_number":69,"context_line":"logging mechanism, but we could likely use"},{"line_number":70,"context_line":"https://review.openstack.org/#/c/227766/ or similar if it gets merged) and"},{"line_number":71,"context_line":"then emit them only if the overall scheduling operation fails (i.e."},{"line_number":72,"context_line":"exception.NoValidHost is raised).  If the scheduling operation passes, the"},{"line_number":73,"context_line":"logs will be thrown out.  Since the logging would happen only on failure, it"},{"line_number":74,"context_line":"is proposed that we log the detailed filter logs at the \"info\" level."},{"line_number":75,"context_line":"(The actual NoValidHost error log will still be logged at the current logging"}],"source_content_type":"text/x-rst","patch_set":11,"id":"dab17558_2f80d88a","line":72,"range":{"start_line":71,"start_character":15,"end_line":72,"end_character":32},"updated":"2016-05-10 09:36:49.000000000","message":"Are you interested in this idea:\n\nLet filters call each other in the recursive way, instead of letting the scheduler call filters use iteration.\n\nThis solution is perfect for this problem because if one filter fails with no host left, it can throw an Exception to notify all the executed filters in the calling stack! And all those filters can print the stored debug messages if they catches this Exception. There\u0027s *no* need to introduce any scheduler specific logging mechanism, and I believe it is not a good design.","commit_id":"f9e3e35dc1f9ae3f7b0b01eecf54d706ef736b13"},{"author":{"_account_id":8768,"name":"Chris Friesen","email":"chris.friesen@windriver.com","username":"cbf123"},"change_message_id":"7236a344f1c2614da3d9588448011a57baf1f512","unresolved":false,"context_lines":[{"line_number":68,"context_line":"debug logs in memory (the current plan is to use a sched-filter-specific"},{"line_number":69,"context_line":"logging mechanism, but we could likely use"},{"line_number":70,"context_line":"https://review.openstack.org/#/c/227766/ or similar if it gets merged) and"},{"line_number":71,"context_line":"then emit them only if the overall scheduling operation fails (i.e."},{"line_number":72,"context_line":"exception.NoValidHost is raised).  If the scheduling operation passes, the"},{"line_number":73,"context_line":"logs will be thrown out.  Since the logging would happen only on failure, it"},{"line_number":74,"context_line":"is proposed that we log the detailed filter logs at the \"info\" level."},{"line_number":75,"context_line":"(The actual NoValidHost error log will still be logged at the current logging"}],"source_content_type":"text/x-rst","patch_set":11,"id":"dab17558_d769d0fa","line":72,"range":{"start_line":71,"start_character":15,"end_line":72,"end_character":32},"in_reply_to":"dab17558_2f80d88a","updated":"2016-05-10 16:01:54.000000000","message":"You\u0027d still need a way for the scheduler filters to store the debug log messages for all the compute nodes that failed, so that they can emit the logs later if the overall scheduler operation fails.\n\nAnd anyways, the NoValidHost exception is raised in select_destinations() if the number of hosts that passed all the filters is less than then number that was requested.  The filters themselves don\u0027t have the visibility of how many were requested.","commit_id":"f9e3e35dc1f9ae3f7b0b01eecf54d706ef736b13"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"25833c5faf1a889c53d45cebb2c73bd9c1e82c62","unresolved":false,"context_lines":[{"line_number":69,"context_line":"logging mechanism, but we could likely use"},{"line_number":70,"context_line":"https://review.openstack.org/#/c/227766/ or similar if it gets merged) and"},{"line_number":71,"context_line":"then emit them only if the overall scheduling operation fails (i.e."},{"line_number":72,"context_line":"exception.NoValidHost is raised).  If the scheduling operation passes, the"},{"line_number":73,"context_line":"logs will be thrown out.  Since the logging would happen only on failure, it"},{"line_number":74,"context_line":"is proposed that we log the detailed filter logs at the \"info\" level."},{"line_number":75,"context_line":"(The actual NoValidHost error log will still be logged at the current logging"}],"source_content_type":"text/x-rst","patch_set":11,"id":"bab6814e_f6f65b5b","line":72,"in_reply_to":"dab17558_6bc4cf3f","updated":"2016-05-23 10:29:37.000000000","message":"That\u0027s something we can discuss during the implementation phase, and I liked Chris not really providing a solution yet.","commit_id":"f9e3e35dc1f9ae3f7b0b01eecf54d706ef736b13"},{"author":{"_account_id":15648,"name":"Yingxin","email":"yingxin.cheng@intel.com","username":"Yingxin"},"change_message_id":"fbfd8e22066d2d45e4a119f3203ae530e9826a14","unresolved":false,"context_lines":[{"line_number":68,"context_line":"debug logs in memory (the current plan is to use a sched-filter-specific"},{"line_number":69,"context_line":"logging mechanism, but we could likely use"},{"line_number":70,"context_line":"https://review.openstack.org/#/c/227766/ or similar if it gets merged) and"},{"line_number":71,"context_line":"then emit them only if the overall scheduling operation fails (i.e."},{"line_number":72,"context_line":"exception.NoValidHost is raised).  If the scheduling operation passes, the"},{"line_number":73,"context_line":"logs will be thrown out.  Since the logging would happen only on failure, it"},{"line_number":74,"context_line":"is proposed that we log the detailed filter logs at the \"info\" level."},{"line_number":75,"context_line":"(The actual NoValidHost error log will still be logged at the current logging"}],"source_content_type":"text/x-rst","patch_set":11,"id":"dab17558_63af6f2a","line":72,"range":{"start_line":71,"start_character":15,"end_line":72,"end_character":32},"in_reply_to":"dab17558_d769d0fa","updated":"2016-05-11 01:48:10.000000000","message":"For 1. Where to store the debug log: you can choose to implement another logging mechanism or simply let filter themselves to store and manage the tracings.\n\nFor 2. Filters can have visibility if the requested number is passed to them as an argument. And the scheduler actually doesn\u0027t need to weigh the hosts if the hosts are less than requested number(Currently it does).\n\nAnd if NoValidHost() is raised by filters, this exception can even collect those reasons and pass them to conductor, which can collect those logs across retries.","commit_id":"f9e3e35dc1f9ae3f7b0b01eecf54d706ef736b13"},{"author":{"_account_id":782,"name":"John Garbutt","email":"john@johngarbutt.com","username":"johngarbutt"},"change_message_id":"f0f22a6b49cfadf34cf2f14fc07f8d0568ddf249","unresolved":false,"context_lines":[{"line_number":81,"context_line":"of a list of filter names (analogous to \"scheduler_default_filters\") for which"},{"line_number":82,"context_line":"we want to suppress debug logging.  In this way operators could avoid logging"},{"line_number":83,"context_line":"for cases which fail for trivial reasons like lack of CPU/RAM/disk.  We propose"},{"line_number":84,"context_line":"that this option default to [\u0027CoreFilter\u0027, \u0027RamFilter\u0027, \u0027DiskFilter\u0027,"},{"line_number":85,"context_line":"\u0027ComputeFilter\u0027].  This option would apply regardless of which way the"},{"line_number":86,"context_line":"\"log only on failure\" config option is set, for several reasons:"},{"line_number":87,"context_line":" * When logging only on NoValidHost we would still need to do work to store"},{"line_number":88,"context_line":"   the logs for the success path, so this would give a way to reduce the"}],"source_content_type":"text/x-rst","patch_set":11,"id":"bab6814e_d1bfd7cd","line":85,"range":{"start_line":84,"start_character":28,"end_line":85,"end_character":16},"updated":"2016-05-23 13:13:48.000000000","message":"Honestly, I am tempted to say, we just don\u0027t add failure logging for those filters, and just have the single boolean.","commit_id":"f9e3e35dc1f9ae3f7b0b01eecf54d706ef736b13"},{"author":{"_account_id":782,"name":"John Garbutt","email":"john@johngarbutt.com","username":"johngarbutt"},"change_message_id":"f0f22a6b49cfadf34cf2f14fc07f8d0568ddf249","unresolved":false,"context_lines":[{"line_number":83,"context_line":"for cases which fail for trivial reasons like lack of CPU/RAM/disk.  We propose"},{"line_number":84,"context_line":"that this option default to [\u0027CoreFilter\u0027, \u0027RamFilter\u0027, \u0027DiskFilter\u0027,"},{"line_number":85,"context_line":"\u0027ComputeFilter\u0027].  This option would apply regardless of which way the"},{"line_number":86,"context_line":"\"log only on failure\" config option is set, for several reasons:"},{"line_number":87,"context_line":" * When logging only on NoValidHost we would still need to do work to store"},{"line_number":88,"context_line":"   the logs for the success path, so this would give a way to reduce the"},{"line_number":89,"context_line":"   overhead and also reduce the amount of logging on a scheduler failure."}],"source_content_type":"text/x-rst","patch_set":11,"id":"bab6814e_11ccbfb9","line":86,"updated":"2016-05-23 13:13:48.000000000","message":"Nit: I think we need an empty line here to get your indented formatting.","commit_id":"f9e3e35dc1f9ae3f7b0b01eecf54d706ef736b13"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"25833c5faf1a889c53d45cebb2c73bd9c1e82c62","unresolved":false,"context_lines":[{"line_number":160,"context_line":"Work Items"},{"line_number":161,"context_line":"----------"},{"line_number":162,"context_line":""},{"line_number":163,"context_line":"* Add the new config option."},{"line_number":164,"context_line":"* Add the core scheduler code to log the error messages (if there are any) if"},{"line_number":165,"context_line":"  the scheduler is unable to locate a suitable compute node."},{"line_number":166,"context_line":"* For each filter, ensure that all failure modes are covered by an error"}],"source_content_type":"text/x-rst","patch_set":11,"id":"bab6814e_f4049415","line":163,"range":{"start_line":163,"start_character":21,"end_line":163,"end_character":27},"updated":"2016-05-23 10:29:37.000000000","message":"femtonit: technically, there is an \u0027s\u0027 to \u0027option\u0027 given you provided two of them above ;-)","commit_id":"f9e3e35dc1f9ae3f7b0b01eecf54d706ef736b13"}]}