)]}'
{"specs/queens/approved/allocation-candidates-limit.rst":[{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"9f8442333264b0d3e3c427b9114cf7692e095239","unresolved":false,"context_lines":[{"line_number":63,"context_line":"  support for expressing pagination or ordering is being considered. The value"},{"line_number":64,"context_line":"  means the first N candidates. If unset, there is no limit."},{"line_number":65,"context_line":""},{"line_number":66,"context_line":"* Add a `limiter` column to the `resource_providers` table the value of which"},{"line_number":67,"context_line":"  will be used to order the limited results. This will initially be populated"},{"line_number":68,"context_line":"  with a random integer in a large range. Adjust the queries used to get the"},{"line_number":69,"context_line":"  resource providers involved in allocation candidates to to limit by the limit"}],"source_content_type":"text/x-rst","patch_set":1,"id":"7f515b1d_c8630b1a","line":66,"range":{"start_line":66,"start_character":9,"end_line":66,"end_character":16},"updated":"2017-09-25 15:35:05.000000000","message":"How about \"weight\" or \"sort_weight\" instead? What the value of the column represents is the weight given to the provider in determining sort.","commit_id":"eb2ff0a94f449253f2a339f1aae2586c2fba4173"},{"author":{"_account_id":1063,"name":"Ed Leafe","email":"ed@leafe.com","username":"ed-leafe"},"change_message_id":"cf740f6d542ccf31736f060be00fd67754758397","unresolved":false,"context_lines":[{"line_number":63,"context_line":"  support for expressing pagination or ordering is being considered. The value"},{"line_number":64,"context_line":"  means the first N candidates. If unset, there is no limit."},{"line_number":65,"context_line":""},{"line_number":66,"context_line":"* Add a `limiter` column to the `resource_providers` table the value of which"},{"line_number":67,"context_line":"  will be used to order the limited results. This will initially be populated"},{"line_number":68,"context_line":"  with a random integer in a large range. Adjust the queries used to get the"},{"line_number":69,"context_line":"  resource providers involved in allocation candidates to to limit by the limit"}],"source_content_type":"text/x-rst","patch_set":1,"id":"7f515b1d_e931260d","line":66,"range":{"start_line":66,"start_character":9,"end_line":66,"end_character":16},"in_reply_to":"7f515b1d_58db6aec","updated":"2017-09-25 18:25:55.000000000","message":"As much as I love bikeshedding, \u0027weight\u0027 seems fine.","commit_id":"eb2ff0a94f449253f2a339f1aae2586c2fba4173"},{"author":{"_account_id":11564,"name":"Chris Dent","email":"cdent@anticdent.org","username":"chdent"},"change_message_id":"cfd2df533e728ed7c1f3c97fd772eee1c8a93acb","unresolved":false,"context_lines":[{"line_number":63,"context_line":"  support for expressing pagination or ordering is being considered. The value"},{"line_number":64,"context_line":"  means the first N candidates. If unset, there is no limit."},{"line_number":65,"context_line":""},{"line_number":66,"context_line":"* Add a `limiter` column to the `resource_providers` table the value of which"},{"line_number":67,"context_line":"  will be used to order the limited results. This will initially be populated"},{"line_number":68,"context_line":"  with a random integer in a large range. Adjust the queries used to get the"},{"line_number":69,"context_line":"  resource providers involved in allocation candidates to to limit by the limit"}],"source_content_type":"text/x-rst","patch_set":1,"id":"7f515b1d_58db6aec","line":66,"range":{"start_line":66,"start_character":9,"end_line":66,"end_character":16},"in_reply_to":"7f515b1d_c8630b1a","updated":"2017-09-25 16:57:20.000000000","message":"weight works for me","commit_id":"eb2ff0a94f449253f2a339f1aae2586c2fba4173"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"426b5bd9a3a5b753f75bf63e0ede8be944772a7b","unresolved":false,"context_lines":[{"line_number":63,"context_line":"  support for expressing pagination or ordering is being considered. The value"},{"line_number":64,"context_line":"  means the first N candidates. If unset, there is no limit."},{"line_number":65,"context_line":""},{"line_number":66,"context_line":"* Add a `limiter` column to the `resource_providers` table the value of which"},{"line_number":67,"context_line":"  will be used to order the limited results. This will initially be populated"},{"line_number":68,"context_line":"  with a random integer in a large range. Adjust the queries used to get the"},{"line_number":69,"context_line":"  resource providers involved in allocation candidates to to limit by the limit"}],"source_content_type":"text/x-rst","patch_set":1,"id":"7f515b1d_521bbe71","line":66,"range":{"start_line":66,"start_character":9,"end_line":66,"end_character":16},"in_reply_to":"7f515b1d_e931260d","updated":"2017-09-29 19:26:39.000000000","message":"+1 for weight","commit_id":"eb2ff0a94f449253f2a339f1aae2586c2fba4173"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"8f9aad5303a99d28ded0c36f95167dc374ef10cc","unresolved":false,"context_lines":[{"line_number":68,"context_line":"  with a random integer in a large range. Adjust the queries used to get the"},{"line_number":69,"context_line":"  resource providers involved in allocation candidates to to limit by the limit"},{"line_number":70,"context_line":"  and order by that column.  When a resource provider is created, set the"},{"line_number":71,"context_line":"  column. If the system prefers to bias for pack, we are done."},{"line_number":72,"context_line":""},{"line_number":73,"context_line":"* For systems that prefer a bias for spread, add a periodic job which updates"},{"line_number":74,"context_line":"  the random numbers in the added column. The choice to do so will be"}],"source_content_type":"text/x-rst","patch_set":1,"id":"7f515b1d_82a164ac","line":71,"updated":"2017-09-25 21:18:09.000000000","message":"If the randomizer is on the resource provider, how does this work when we need results across multiple RPs (including/especially nested/shared)?  Obviously the results still need to be \"sticky\" (by which I mean e.g. one allocation_candidate can\u0027t give me a VCPU from hostA and a VF from hostB just because their respective RPs\u0027 `weight` fields sort together).  We could:\n\n- Pick the *first* RP in each given allocation_candidate and use its weight to sort the overall list.\n- Always sort by the *lowest* RP weight in each given allocation_candidate.\n- Do some kind of fancy summation of the RP weights in each allocation_candidate (has the possibility of collisions, but shrug).\n- Other, crazier things.\n\nOr maybe I\u0027m totally misunderstanding where in the process this sorting/limiting is happening, and this is a non-problem :)","commit_id":"eb2ff0a94f449253f2a339f1aae2586c2fba4173"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"20a94035520d971f28dea54c0f17e5ea81cf1a8f","unresolved":false,"context_lines":[{"line_number":68,"context_line":"  with a random integer in a large range. Adjust the queries used to get the"},{"line_number":69,"context_line":"  resource providers involved in allocation candidates to to limit by the limit"},{"line_number":70,"context_line":"  and order by that column.  When a resource provider is created, set the"},{"line_number":71,"context_line":"  column. If the system prefers to bias for pack, we are done."},{"line_number":72,"context_line":""},{"line_number":73,"context_line":"* For systems that prefer a bias for spread, add a periodic job which updates"},{"line_number":74,"context_line":"  the random numbers in the added column. The choice to do so will be"}],"source_content_type":"text/x-rst","patch_set":1,"id":"7f515b1d_442a1c7f","line":71,"in_reply_to":"7f515b1d_82a164ac","updated":"2017-09-27 13:53:44.000000000","message":"I hope this sorting and limiting happens between allocation candidates. In a single allocation candidate placement should always return allocation request against every resource provider.","commit_id":"eb2ff0a94f449253f2a339f1aae2586c2fba4173"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"8f9aad5303a99d28ded0c36f95167dc374ef10cc","unresolved":false,"context_lines":[{"line_number":69,"context_line":"  resource providers involved in allocation candidates to to limit by the limit"},{"line_number":70,"context_line":"  and order by that column.  When a resource provider is created, set the"},{"line_number":71,"context_line":"  column. If the system prefers to bias for pack, we are done."},{"line_number":72,"context_line":""},{"line_number":73,"context_line":"* For systems that prefer a bias for spread, add a periodic job which updates"},{"line_number":74,"context_line":"  the random numbers in the added column. The choice to do so will be"},{"line_number":75,"context_line":"  controlled by a configuration option, ``randomize_candidates``. When true the"}],"source_content_type":"text/x-rst","patch_set":1,"id":"7f515b1d_82060486","line":72,"updated":"2017-09-25 21:18:09.000000000","message":"Another related issue when multiple RPs are in play: there\u0027s no easy way to map the number of RP records to the number of allocation_candidates that result from munging \u0027em all together.  It could easily swing either way: 1,000 RPs could map to 10,000 allocation_candidates or... zero.","commit_id":"eb2ff0a94f449253f2a339f1aae2586c2fba4173"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"426b5bd9a3a5b753f75bf63e0ede8be944772a7b","unresolved":false,"context_lines":[{"line_number":69,"context_line":"  resource providers involved in allocation candidates to to limit by the limit"},{"line_number":70,"context_line":"  and order by that column.  When a resource provider is created, set the"},{"line_number":71,"context_line":"  column. If the system prefers to bias for pack, we are done."},{"line_number":72,"context_line":""},{"line_number":73,"context_line":"* For systems that prefer a bias for spread, add a periodic job which updates"},{"line_number":74,"context_line":"  the random numbers in the added column. The choice to do so will be"},{"line_number":75,"context_line":"  controlled by a configuration option, ``randomize_candidates``. When true the"}],"source_content_type":"text/x-rst","patch_set":1,"id":"7f515b1d_d27c4e73","line":72,"in_reply_to":"7f515b1d_186a47ed","updated":"2017-09-29 19:26:39.000000000","message":"the order by and limit are applying to different things.\nthe limit will be on the outer most query only.\n\nthe joins and sub queries use to filter the set of candidates would not use the limit.","commit_id":"eb2ff0a94f449253f2a339f1aae2586c2fba4173"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"d04f21802cb98908d3720f207a8b28f461292a37","unresolved":false,"context_lines":[{"line_number":69,"context_line":"  resource providers involved in allocation candidates to to limit by the limit"},{"line_number":70,"context_line":"  and order by that column.  When a resource provider is created, set the"},{"line_number":71,"context_line":"  column. If the system prefers to bias for pack, we are done."},{"line_number":72,"context_line":""},{"line_number":73,"context_line":"* For systems that prefer a bias for spread, add a periodic job which updates"},{"line_number":74,"context_line":"  the random numbers in the added column. The choice to do so will be"},{"line_number":75,"context_line":"  controlled by a configuration option, ``randomize_candidates``. When true the"}],"source_content_type":"text/x-rst","patch_set":1,"id":"7f515b1d_186a47ed","line":72,"in_reply_to":"7f515b1d_24f040fe","updated":"2017-09-27 14:56:21.000000000","message":"Yeah, I don\u0027t think we\u0027re suggesting in any way to limit the size of an *individual* allocation_candidate.  Realistically, there\u0027s no way a single allocation_candidate can contain resources from thousands (or even dozens) of providers, so that\u0027s not an issue.\n\nNo, the idea is to limit the number of allocation_candidates.  But this doc suggests doing that by randomizing and limiting the number of resource providers picked up by an internal query.  And my point is that \"limit me to N allocation_candidates\" does *not* translate internally to \"get me N resource providers\".","commit_id":"eb2ff0a94f449253f2a339f1aae2586c2fba4173"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"20a94035520d971f28dea54c0f17e5ea81cf1a8f","unresolved":false,"context_lines":[{"line_number":69,"context_line":"  resource providers involved in allocation candidates to to limit by the limit"},{"line_number":70,"context_line":"  and order by that column.  When a resource provider is created, set the"},{"line_number":71,"context_line":"  column. If the system prefers to bias for pack, we are done."},{"line_number":72,"context_line":""},{"line_number":73,"context_line":"* For systems that prefer a bias for spread, add a periodic job which updates"},{"line_number":74,"context_line":"  the random numbers in the added column. The choice to do so will be"},{"line_number":75,"context_line":"  controlled by a configuration option, ``randomize_candidates``. When true the"}],"source_content_type":"text/x-rst","patch_set":1,"id":"7f515b1d_24f040fe","line":72,"in_reply_to":"7f515b1d_82060486","updated":"2017-09-27 13:53:44.000000000","message":"We should not limit the number of RPs returned in a single allocation candidate. As an allocation candidate is only valid if it contains allocation request against every RP that is needed to fulfill the original allocation_candidate request.\n\nIf the placement user can ask for so many different resources that a single allocation candidate needs to contain 1000 allocation request then we have to consider returning just a single allocation_candidate uuid and add an API that supports requesting the details of such allocation candidate with pagination support.","commit_id":"eb2ff0a94f449253f2a339f1aae2586c2fba4173"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"9f8442333264b0d3e3c427b9114cf7692e095239","unresolved":false,"context_lines":[{"line_number":72,"context_line":""},{"line_number":73,"context_line":"* For systems that prefer a bias for spread, add a periodic job which updates"},{"line_number":74,"context_line":"  the random numbers in the added column. The choice to do so will be"},{"line_number":75,"context_line":"  controlled by a configuration option, ``randomize_candidates``. When true the"},{"line_number":76,"context_line":"  periodic job will run. This update must take care to not update the"},{"line_number":77,"context_line":"  ``updated_at`` column in the table."},{"line_number":78,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"7f515b1d_c81aeb7b","line":75,"range":{"start_line":75,"start_character":42,"end_line":75,"end_character":62},"updated":"2017-09-25 15:35:05.000000000","message":"I would prefer the CONF option be called sort_weight_reset_interval_seconds and have a value (default of 0) that indicates the number of seconds between the action taken to reset the weights for providers.","commit_id":"eb2ff0a94f449253f2a339f1aae2586c2fba4173"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"d5b4fb0cdcba068546a526d6055994e484a6ad7f","unresolved":false,"context_lines":[{"line_number":72,"context_line":""},{"line_number":73,"context_line":"* For systems that prefer a bias for spread, add a periodic job which updates"},{"line_number":74,"context_line":"  the random numbers in the added column. The choice to do so will be"},{"line_number":75,"context_line":"  controlled by a configuration option, ``randomize_candidates``. When true the"},{"line_number":76,"context_line":"  periodic job will run. This update must take care to not update the"},{"line_number":77,"context_line":"  ``updated_at`` column in the table."},{"line_number":78,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"7f515b1d_cbf0a26a","line":75,"range":{"start_line":75,"start_character":42,"end_line":75,"end_character":62},"in_reply_to":"7f515b1d_18fe1237","updated":"2017-09-25 17:09:45.000000000","message":"yup, saw that, thanks.","commit_id":"eb2ff0a94f449253f2a339f1aae2586c2fba4173"},{"author":{"_account_id":11564,"name":"Chris Dent","email":"cdent@anticdent.org","username":"chdent"},"change_message_id":"cfd2df533e728ed7c1f3c97fd772eee1c8a93acb","unresolved":false,"context_lines":[{"line_number":72,"context_line":""},{"line_number":73,"context_line":"* For systems that prefer a bias for spread, add a periodic job which updates"},{"line_number":74,"context_line":"  the random numbers in the added column. The choice to do so will be"},{"line_number":75,"context_line":"  controlled by a configuration option, ``randomize_candidates``. When true the"},{"line_number":76,"context_line":"  periodic job will run. This update must take care to not update the"},{"line_number":77,"context_line":"  ``updated_at`` column in the table."},{"line_number":78,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"7f515b1d_18fe1237","line":75,"range":{"start_line":75,"start_character":42,"end_line":75,"end_character":62},"in_reply_to":"7f515b1d_c81aeb7b","updated":"2017-09-25 16:57:20.000000000","message":"that\u0027s sort of in the alternatives below\n\nI rushed this version out as a way of capturing what was said at the ptg and then figured it would be resolve to something proper as we discussed it.","commit_id":"eb2ff0a94f449253f2a339f1aae2586c2fba4173"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"9f8442333264b0d3e3c427b9114cf7692e095239","unresolved":false,"context_lines":[{"line_number":73,"context_line":"* For systems that prefer a bias for spread, add a periodic job which updates"},{"line_number":74,"context_line":"  the random numbers in the added column. The choice to do so will be"},{"line_number":75,"context_line":"  controlled by a configuration option, ``randomize_candidates``. When true the"},{"line_number":76,"context_line":"  periodic job will run. This update must take care to not update the"},{"line_number":77,"context_line":"  ``updated_at`` column in the table."},{"line_number":78,"context_line":""},{"line_number":79,"context_line":"At this time, the placement service has no capacity to run a periodic job. It"},{"line_number":80,"context_line":"solely handles web requests. In a WSGI application one common and relatively"}],"source_content_type":"text/x-rst","patch_set":1,"id":"7f515b1d_a898cf1a","line":77,"range":{"start_line":76,"start_character":25,"end_line":77,"end_character":36},"updated":"2017-09-25 15:35:05.000000000","message":"Why? We don\u0027t use the updated_at column for anything at all...","commit_id":"eb2ff0a94f449253f2a339f1aae2586c2fba4173"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"07210230d04ae89e84f90bf00d04cd8529fa38b7","unresolved":false,"context_lines":[{"line_number":73,"context_line":"* For systems that prefer a bias for spread, add a periodic job which updates"},{"line_number":74,"context_line":"  the random numbers in the added column. The choice to do so will be"},{"line_number":75,"context_line":"  controlled by a configuration option, ``randomize_candidates``. When true the"},{"line_number":76,"context_line":"  periodic job will run. This update must take care to not update the"},{"line_number":77,"context_line":"  ``updated_at`` column in the table."},{"line_number":78,"context_line":""},{"line_number":79,"context_line":"At this time, the placement service has no capacity to run a periodic job. It"},{"line_number":80,"context_line":"solely handles web requests. In a WSGI application one common and relatively"}],"source_content_type":"text/x-rst","patch_set":1,"id":"7f515b1d_c08114c5","line":77,"range":{"start_line":76,"start_character":25,"end_line":77,"end_character":36},"in_reply_to":"7f515b1d_123ae697","updated":"2017-09-29 21:05:54.000000000","message":"Nope, we\u0027re not even populating those columns... it\u0027s a bug.","commit_id":"eb2ff0a94f449253f2a339f1aae2586c2fba4173"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"d5b4fb0cdcba068546a526d6055994e484a6ad7f","unresolved":false,"context_lines":[{"line_number":73,"context_line":"* For systems that prefer a bias for spread, add a periodic job which updates"},{"line_number":74,"context_line":"  the random numbers in the added column. The choice to do so will be"},{"line_number":75,"context_line":"  controlled by a configuration option, ``randomize_candidates``. When true the"},{"line_number":76,"context_line":"  periodic job will run. This update must take care to not update the"},{"line_number":77,"context_line":"  ``updated_at`` column in the table."},{"line_number":78,"context_line":""},{"line_number":79,"context_line":"At this time, the placement service has no capacity to run a periodic job. It"},{"line_number":80,"context_line":"solely handles web requests. In a WSGI application one common and relatively"}],"source_content_type":"text/x-rst","patch_set":1,"id":"7f515b1d_abf32673","line":77,"range":{"start_line":76,"start_character":25,"end_line":77,"end_character":36},"in_reply_to":"7f515b1d_18acd232","updated":"2017-09-25 17:09:45.000000000","message":"Aha. Yeah, the cache headers stuff indeed makes sense.\n\nThen again, changing the weight column value means the provider has been updated (just not its generation). Would it be that terrible to update the updated_at column at the same time? Meh, prolly wouldn\u0027t make a huge difference but I can see why you might not want to since the weight column is internal information to the placement service.","commit_id":"eb2ff0a94f449253f2a339f1aae2586c2fba4173"},{"author":{"_account_id":11564,"name":"Chris Dent","email":"cdent@anticdent.org","username":"chdent"},"change_message_id":"cfd2df533e728ed7c1f3c97fd772eee1c8a93acb","unresolved":false,"context_lines":[{"line_number":73,"context_line":"* For systems that prefer a bias for spread, add a periodic job which updates"},{"line_number":74,"context_line":"  the random numbers in the added column. The choice to do so will be"},{"line_number":75,"context_line":"  controlled by a configuration option, ``randomize_candidates``. When true the"},{"line_number":76,"context_line":"  periodic job will run. This update must take care to not update the"},{"line_number":77,"context_line":"  ``updated_at`` column in the table."},{"line_number":78,"context_line":""},{"line_number":79,"context_line":"At this time, the placement service has no capacity to run a periodic job. It"},{"line_number":80,"context_line":"solely handles web requests. In a WSGI application one common and relatively"}],"source_content_type":"text/x-rst","patch_set":1,"id":"7f515b1d_18acd232","line":77,"range":{"start_line":76,"start_character":25,"end_line":77,"end_character":36},"in_reply_to":"7f515b1d_a898cf1a","updated":"2017-09-25 16:57:20.000000000","message":"I had this in mind: https://review.openstack.org/#/c/496853/\n\nThat makes use of updated_at\n\nAnd besides, if the field is there, it ought to be meaningful?","commit_id":"eb2ff0a94f449253f2a339f1aae2586c2fba4173"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"426b5bd9a3a5b753f75bf63e0ede8be944772a7b","unresolved":false,"context_lines":[{"line_number":73,"context_line":"* For systems that prefer a bias for spread, add a periodic job which updates"},{"line_number":74,"context_line":"  the random numbers in the added column. The choice to do so will be"},{"line_number":75,"context_line":"  controlled by a configuration option, ``randomize_candidates``. When true the"},{"line_number":76,"context_line":"  periodic job will run. This update must take care to not update the"},{"line_number":77,"context_line":"  ``updated_at`` column in the table."},{"line_number":78,"context_line":""},{"line_number":79,"context_line":"At this time, the placement service has no capacity to run a periodic job. It"},{"line_number":80,"context_line":"solely handles web requests. In a WSGI application one common and relatively"}],"source_content_type":"text/x-rst","patch_set":1,"id":"7f515b1d_123ae697","line":77,"range":{"start_line":76,"start_character":25,"end_line":77,"end_character":36},"in_reply_to":"7f515b1d_abf32673","updated":"2017-09-29 19:26:39.000000000","message":"are we using a db trigger to set the updated at column today?\nif so we will have to replace the exsiting trigger on upgrade whic should be fine but just taught i would ask.","commit_id":"eb2ff0a94f449253f2a339f1aae2586c2fba4173"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"426b5bd9a3a5b753f75bf63e0ede8be944772a7b","unresolved":false,"context_lines":[{"line_number":80,"context_line":"solely handles web requests. In a WSGI application one common and relatively"},{"line_number":81,"context_line":"simple way to do this is to run a task in the thread or process handling a"},{"line_number":82,"context_line":"request and response, after the response has been sent to the client::"},{"line_number":83,"context_line":""},{"line_number":84,"context_line":"    try:"},{"line_number":85,"context_line":"        start_response(\u0027200 OK\u0027, [])"},{"line_number":86,"context_line":"        return [\u0027my\u0027, \u0027stuff\u0027]"},{"line_number":87,"context_line":"    finally:"},{"line_number":88,"context_line":"        if time_p():"},{"line_number":89,"context_line":"            periodic_job()"},{"line_number":90,"context_line":""},{"line_number":91,"context_line":"This can only work if the WSGI application returns content as a list of"},{"line_number":92,"context_line":"strings. The placement service follows this pattern, and as a JSON API will"}],"source_content_type":"text/x-rst","patch_set":1,"id":"7f515b1d_b229d237","line":89,"range":{"start_line":83,"start_character":0,"end_line":89,"end_character":26},"updated":"2017-09-29 19:26:39.000000000","message":"this will not result in the periodic_job running after the return. when retrun is called the finally block will be executed first.","commit_id":"eb2ff0a94f449253f2a339f1aae2586c2fba4173"},{"author":{"_account_id":11564,"name":"Chris Dent","email":"cdent@anticdent.org","username":"chdent"},"change_message_id":"54c559acd3eaf22f5cc422b45933a9632f01f180","unresolved":false,"context_lines":[{"line_number":80,"context_line":"solely handles web requests. In a WSGI application one common and relatively"},{"line_number":81,"context_line":"simple way to do this is to run a task in the thread or process handling a"},{"line_number":82,"context_line":"request and response, after the response has been sent to the client::"},{"line_number":83,"context_line":""},{"line_number":84,"context_line":"    try:"},{"line_number":85,"context_line":"        start_response(\u0027200 OK\u0027, [])"},{"line_number":86,"context_line":"        return [\u0027my\u0027, \u0027stuff\u0027]"},{"line_number":87,"context_line":"    finally:"},{"line_number":88,"context_line":"        if time_p():"},{"line_number":89,"context_line":"            periodic_job()"},{"line_number":90,"context_line":""},{"line_number":91,"context_line":"This can only work if the WSGI application returns content as a list of"},{"line_number":92,"context_line":"strings. The placement service follows this pattern, and as a JSON API will"}],"source_content_type":"text/x-rst","patch_set":1,"id":"7f515b1d_220102b0","line":89,"range":{"start_line":83,"start_character":0,"end_line":89,"end_character":26},"in_reply_to":"7f515b1d_b229d237","updated":"2017-10-01 12:40:14.000000000","message":"I left out a bit, which is encapsulating the whole thing in a genertor-like thing, see: http://modwsgi.readthedocs.io/en/develop/user-guides/registering-cleanup-code.html#cleanup-at-end-of-request","commit_id":"eb2ff0a94f449253f2a339f1aae2586c2fba4173"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"9f8442333264b0d3e3c427b9114cf7692e095239","unresolved":false,"context_lines":[{"line_number":96,"context_line":"should run will probably check to see if a fixed amount of time has passed"},{"line_number":97,"context_line":"since the current process updated the random column. In a distributed"},{"line_number":98,"context_line":"environment this could lead to multiple processes attempting to update the"},{"line_number":99,"context_line":"column at the same time."},{"line_number":100,"context_line":""},{"line_number":101,"context_line":"Alternatives"},{"line_number":102,"context_line":"------------"}],"source_content_type":"text/x-rst","patch_set":1,"id":"7f515b1d_03f104ce","line":99,"updated":"2017-09-25 15:35:05.000000000","message":"The simplest solution would be to have a simple bash script run in a cron job that would update batches of resource provider records.\n\nThe entire script could be done in just a few lines of code:\n\n #! /bin/bash\n $PLACEMENT_DB_USER\u003d${PLACEMENT_DB_USER:admin}\n $PLACEMENT_DB_PASS\u003d${PLACEMENT_DB_PASS:supersecret}\n $PLACEMENT_DB_NAME\u003d${PLACEMENT_DB_NAME:placement}\n\n num_providers\u003d`mysql --skip-column-names --user\u003d$PLACEMENT_DB_USER --password\u003d$PLACEMENT_DB_PASS --database\u003d$PLACEMENT_DB_NAME -e \"SELECT MAX(id) FROM resource_providers\"`\n cur_id\u003d1\n while true; do\n     last_id\u003d$(( $cur_id + 100 ))\n     mysql --skip-column-names --user\u003d$PLACEMENT_DB_USER --password\u003d$PLACEMENT_DB_PASS --database\u003d$PLACEMENT_DB_NAME -e \"UPDATE resource_providers SET sort_weight \u003d RAND() WHERE id BETWEEN $cur_id AND $last_id\"\n     cur_id\u003d$(( $last_id + 1 ))\n     if [[ $cur_id \u003c $num_providers ]]; then\n          break\n     fi\n done","commit_id":"eb2ff0a94f449253f2a339f1aae2586c2fba4173"},{"author":{"_account_id":11564,"name":"Chris Dent","email":"cdent@anticdent.org","username":"chdent"},"change_message_id":"cfd2df533e728ed7c1f3c97fd772eee1c8a93acb","unresolved":false,"context_lines":[{"line_number":96,"context_line":"should run will probably check to see if a fixed amount of time has passed"},{"line_number":97,"context_line":"since the current process updated the random column. In a distributed"},{"line_number":98,"context_line":"environment this could lead to multiple processes attempting to update the"},{"line_number":99,"context_line":"column at the same time."},{"line_number":100,"context_line":""},{"line_number":101,"context_line":"Alternatives"},{"line_number":102,"context_line":"------------"}],"source_content_type":"text/x-rst","patch_set":1,"id":"7f515b1d_8bdd0a44","line":99,"in_reply_to":"7f515b1d_03f104ce","updated":"2017-09-25 16:57:20.000000000","message":"Yeah, as you noticed below that\u0027s in the alternatives. The reason I wrote it with some kind of job is because that\u0027s what the room seemed to want when we were at the ptg: some kind of way of doing periodic jobs.\n\nI guess we just wait for a few more bits of input and then see which way to go.\n\nI\u0027m somewhat inclined to have some kind of built in periodic because I have some sort of antagonism for pack, thus want to make it easy for people to spread, but it is fraught lots of niggly details that a cron job doesn\u0027t have to concern itself with.","commit_id":"eb2ff0a94f449253f2a339f1aae2586c2fba4173"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"426b5bd9a3a5b753f75bf63e0ede8be944772a7b","unresolved":false,"context_lines":[{"line_number":96,"context_line":"should run will probably check to see if a fixed amount of time has passed"},{"line_number":97,"context_line":"since the current process updated the random column. In a distributed"},{"line_number":98,"context_line":"environment this could lead to multiple processes attempting to update the"},{"line_number":99,"context_line":"column at the same time."},{"line_number":100,"context_line":""},{"line_number":101,"context_line":"Alternatives"},{"line_number":102,"context_line":"------------"}],"source_content_type":"text/x-rst","patch_set":1,"id":"7f515b1d_32984273","line":99,"in_reply_to":"7f515b1d_2badf630","updated":"2017-09-29 19:26:39.000000000","message":"i think something external makes sense too.\n\ndan did raise a concern that would lead to some\ncreating a pre-scheduler scheduler but if they do i\ndon\u0027t think we are beholden to support them. as far as placement shoudl care its just a random index int.","commit_id":"eb2ff0a94f449253f2a339f1aae2586c2fba4173"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"07210230d04ae89e84f90bf00d04cd8529fa38b7","unresolved":false,"context_lines":[{"line_number":96,"context_line":"should run will probably check to see if a fixed amount of time has passed"},{"line_number":97,"context_line":"since the current process updated the random column. In a distributed"},{"line_number":98,"context_line":"environment this could lead to multiple processes attempting to update the"},{"line_number":99,"context_line":"column at the same time."},{"line_number":100,"context_line":""},{"line_number":101,"context_line":"Alternatives"},{"line_number":102,"context_line":"------------"}],"source_content_type":"text/x-rst","patch_set":1,"id":"7f515b1d_606a601a","line":99,"in_reply_to":"7f515b1d_32984273","updated":"2017-09-29 21:05:54.000000000","message":"exactly, yes.","commit_id":"eb2ff0a94f449253f2a339f1aae2586c2fba4173"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"d5b4fb0cdcba068546a526d6055994e484a6ad7f","unresolved":false,"context_lines":[{"line_number":96,"context_line":"should run will probably check to see if a fixed amount of time has passed"},{"line_number":97,"context_line":"since the current process updated the random column. In a distributed"},{"line_number":98,"context_line":"environment this could lead to multiple processes attempting to update the"},{"line_number":99,"context_line":"column at the same time."},{"line_number":100,"context_line":""},{"line_number":101,"context_line":"Alternatives"},{"line_number":102,"context_line":"------------"}],"source_content_type":"text/x-rst","patch_set":1,"id":"7f515b1d_2badf630","line":99,"in_reply_to":"7f515b1d_8bdd0a44","updated":"2017-09-25 17:09:45.000000000","message":"Yeah, instead of hacking a periodic into the WSGI service, I\u0027d just as much prefer a separate placement-periodics (or similar) job that would only do periodic stuff.\n\nOr just use cron... ;)","commit_id":"eb2ff0a94f449253f2a339f1aae2586c2fba4173"},{"author":{"_account_id":9172,"name":"Radoslav Gerganov","email":"rgerganov@vmware.com","username":"rgerganov"},"change_message_id":"33ba539ed742af2a384c6d1d8b78d26c47b654a9","unresolved":false,"context_lines":[{"line_number":98,"context_line":"environment this could lead to multiple processes attempting to update the"},{"line_number":99,"context_line":"column at the same time."},{"line_number":100,"context_line":""},{"line_number":101,"context_line":"Alternatives"},{"line_number":102,"context_line":"------------"},{"line_number":103,"context_line":""},{"line_number":104,"context_line":"* We could add limit to ``GET /resource_providers`` too."}],"source_content_type":"text/x-rst","patch_set":1,"id":"7f515b1d_358e5e2e","line":101,"updated":"2017-09-26 08:16:07.000000000","message":"How about keeping the existing DB model and using Reservoir sampling (https://en.wikipedia.org/wiki/Reservoir_sampling)? If there is an effective way to iterate over the result set then we simply have to choose a random sample with k elements where k is the specified limit. This can be done in O(n) time.","commit_id":"eb2ff0a94f449253f2a339f1aae2586c2fba4173"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"a72078fb4dc3e8ba60f301179cc9e9be2d4467b8","unresolved":false,"context_lines":[{"line_number":98,"context_line":"environment this could lead to multiple processes attempting to update the"},{"line_number":99,"context_line":"column at the same time."},{"line_number":100,"context_line":""},{"line_number":101,"context_line":"Alternatives"},{"line_number":102,"context_line":"------------"},{"line_number":103,"context_line":""},{"line_number":104,"context_line":"* We could add limit to ``GET /resource_providers`` too."}],"source_content_type":"text/x-rst","patch_set":1,"id":"7f515b1d_98dbd77a","line":101,"in_reply_to":"7f515b1d_2cbeecd4","updated":"2017-09-27 14:37:50.000000000","message":"Just to clarify: the ideal would be to do the sampling in the database itself (before the results even get to the server-side python process) via a stored procedure.  (I don\u0027t know anything about our database engine, but I assume it\u0027s not a python process, and is smart enough to grow/shrink its memory footprint.)","commit_id":"eb2ff0a94f449253f2a339f1aae2586c2fba4173"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"a34fa1dfeaac7578aadfbd67ffcf01b33300c1d5","unresolved":false,"context_lines":[{"line_number":98,"context_line":"environment this could lead to multiple processes attempting to update the"},{"line_number":99,"context_line":"column at the same time."},{"line_number":100,"context_line":""},{"line_number":101,"context_line":"Alternatives"},{"line_number":102,"context_line":"------------"},{"line_number":103,"context_line":""},{"line_number":104,"context_line":"* We could add limit to ``GET /resource_providers`` too."}],"source_content_type":"text/x-rst","patch_set":1,"id":"7f515b1d_969cdf5d","line":101,"in_reply_to":"7f515b1d_358e5e2e","updated":"2017-09-26 22:23:03.000000000","message":"If this can be made to work, I like it a lot better than the proposed randomized-column thing.\n- It doesn\u0027t require database changes\n- It doesn\u0027t require a periodic task or cron job\n- We can run the algorithm over the allocation_candidates result list rather than trying to figure out how it applies to multiple RPs (see comments L71 \u0026 L72)\n\nIt does require that the comprehensive list of allocation_candidates be calculated.  But that can be contained within the database.  And I kinda suspect we were going to have to do that anyway even with the original proposal.","commit_id":"eb2ff0a94f449253f2a339f1aae2586c2fba4173"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"887dd8cc101da38c6286b0310609eee3744176af","unresolved":false,"context_lines":[{"line_number":98,"context_line":"environment this could lead to multiple processes attempting to update the"},{"line_number":99,"context_line":"column at the same time."},{"line_number":100,"context_line":""},{"line_number":101,"context_line":"Alternatives"},{"line_number":102,"context_line":"------------"},{"line_number":103,"context_line":""},{"line_number":104,"context_line":"* We could add limit to ``GET /resource_providers`` too."}],"source_content_type":"text/x-rst","patch_set":1,"id":"7f515b1d_40908495","line":101,"in_reply_to":"7f515b1d_358e5e2e","updated":"2017-09-29 20:25:58.000000000","message":"We want the \"sampling\" to be done in better than O(n). O(n) represents a poorly-scaling solution in this case, since it represents a full table scan of every record in the resource providers table and would be the case if we did ORDER BY RAND() after winnowing the number of resource providers that meet the requested resource amounts.\n\nWhat we\u0027re proposing with the weight column is offloading the randomization to some external process that would run periodically to shuffle the order of returned hosts that have identical matching conditions for a given request for resources.\n\nDoing \"ORDER BY weight LIMIT 10\" is an O(1) scale, versus \"ORDER BY RAND()\" which is an O(n) scale.","commit_id":"eb2ff0a94f449253f2a339f1aae2586c2fba4173"},{"author":{"_account_id":11564,"name":"Chris Dent","email":"cdent@anticdent.org","username":"chdent"},"change_message_id":"54c559acd3eaf22f5cc422b45933a9632f01f180","unresolved":false,"context_lines":[{"line_number":98,"context_line":"environment this could lead to multiple processes attempting to update the"},{"line_number":99,"context_line":"column at the same time."},{"line_number":100,"context_line":""},{"line_number":101,"context_line":"Alternatives"},{"line_number":102,"context_line":"------------"},{"line_number":103,"context_line":""},{"line_number":104,"context_line":"* We could add limit to ``GET /resource_providers`` too."}],"source_content_type":"text/x-rst","patch_set":1,"id":"7f515b1d_42f63ed9","line":101,"in_reply_to":"7f515b1d_40908495","updated":"2017-10-01 12:40:14.000000000","message":"\"We want the \"sampling\" to be done in better than O(n).\"\n\nDo we though? That\u0027s something that\u0027s come about since we started talking about solutions. The original goal was to send less data over the wire and one way to do that is to generate less data, but another way to do it is to generate the same amount of data on the server side, but send less, by sampling it.\n\nSince there is no single sql query which is generating the result set (once we add in traits, and shared and nesting) and we are composing lots of objects that traverse the data quite a bit, I\u0027m not entirely clear how any of these ideas will actually apply in the database queries when we get down to the concrete application. Sure, conceptually: order by weight, but _where_?\n\nSo maybe it easier to create the results then truncate? I\u0027m not saying we _should_ do that, but it does sound simpler and that might be good.","commit_id":"eb2ff0a94f449253f2a339f1aae2586c2fba4173"},{"author":{"_account_id":11564,"name":"Chris Dent","email":"cdent@anticdent.org","username":"chdent"},"change_message_id":"530ca5a0f6210fb4204b559ab5692c572141de8e","unresolved":false,"context_lines":[{"line_number":98,"context_line":"environment this could lead to multiple processes attempting to update the"},{"line_number":99,"context_line":"column at the same time."},{"line_number":100,"context_line":""},{"line_number":101,"context_line":"Alternatives"},{"line_number":102,"context_line":"------------"},{"line_number":103,"context_line":""},{"line_number":104,"context_line":"* We could add limit to ``GET /resource_providers`` too."}],"source_content_type":"text/x-rst","patch_set":1,"id":"7f515b1d_2cbeecd4","line":101,"in_reply_to":"7f515b1d_969cdf5d","updated":"2017-09-27 11:50:53.000000000","message":"When initially discussing this I had understood us to be concerned with the result size on both the client and server, but if we are only concerned with the size of the data being transferred to the client, but can have a working set that is large on the server, then this idea could be made to work.\n\nThe cost is that once a python process gets large it will stay large. Some WSGI servers provide \"kill this process (after this request is done) if it\u0027s RSS has gone over X\", so that\u0027s potentially not a problem.\n\nFiguring out where to inject this, or any other kind of limiting is tricky because the tuple which makes up the allocation_candidates response is composed in a pretty complicated way.\n\nI think working out those details is going to be easier in code than in spec so we probably don\u0027t need to nail that down here. We do, however, need to nail down what goals we are trying to satisfy. If we are okay with the server-side being big, but just want to make the over-the-wire small, then I would think doing a sampling is a reasonable way to go.","commit_id":"eb2ff0a94f449253f2a339f1aae2586c2fba4173"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"426b5bd9a3a5b753f75bf63e0ede8be944772a7b","unresolved":false,"context_lines":[{"line_number":98,"context_line":"environment this could lead to multiple processes attempting to update the"},{"line_number":99,"context_line":"column at the same time."},{"line_number":100,"context_line":""},{"line_number":101,"context_line":"Alternatives"},{"line_number":102,"context_line":"------------"},{"line_number":103,"context_line":""},{"line_number":104,"context_line":"* We could add limit to ``GET /resource_providers`` too."}],"source_content_type":"text/x-rst","patch_set":1,"id":"7f515b1d_928b76c1","line":101,"in_reply_to":"7f515b1d_98dbd77a","updated":"2017-09-29 19:26:39.000000000","message":"yes we want the sampling to be done in sql not in python which was the original motivation for the new column.\n\nthis is rather on topic \nhttps://www.periscopedata.com/blog/how-to-sample-rows-in-sql-273x-faster.html\n\ntakeing the second last example\n    select * from users\n\n    where id in (\n\n      select round(random() * 21e6)::integer as id\n\n      from generate_series(1, 110)\n\n      group by id -- Discard duplicates\n\n    )\n\n    limit 100\n\nif we were to take our current querys and add a row and max run to the result set and converted it in to an anonymous table we chould do someting like the following\n\nselect tlb.rp_id from\n(\n     select row_number() as row, count(1) as max_row ,rp.id as rp_id\n    FROM resource_providers AS rp JOIN inventories ... -- rest of current query goes here\n) as tlb,\nwhere tlb.row in (\n      select round(random() * tlb.max_row)::integer as id\n\n      from generate_series(1, @my_limit)\n\n      group by row -- Discard duplicates\n)\nlimit @my_limit\n\n\nthat should more or less generate the a randomly sampled subset for the original query however it will inques sql memory usesage in return for not requireing any table walk or new columns at the cost of computing the sampleing for each query.","commit_id":"eb2ff0a94f449253f2a339f1aae2586c2fba4173"},{"author":{"_account_id":1063,"name":"Ed Leafe","email":"ed@leafe.com","username":"ed-leafe"},"change_message_id":"cf740f6d542ccf31736f060be00fd67754758397","unresolved":false,"context_lines":[{"line_number":101,"context_line":"Alternatives"},{"line_number":102,"context_line":"------------"},{"line_number":103,"context_line":""},{"line_number":104,"context_line":"* We could add limit to ``GET /resource_providers`` too."},{"line_number":105,"context_line":""},{"line_number":106,"context_line":"* We could set a relatively low default value for limit."},{"line_number":107,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"7f515b1d_a42233a3","line":104,"updated":"2017-09-25 18:25:55.000000000","message":"That\u0027s not really an alternative for this spec is addressing.","commit_id":"eb2ff0a94f449253f2a339f1aae2586c2fba4173"},{"author":{"_account_id":11564,"name":"Chris Dent","email":"cdent@anticdent.org","username":"chdent"},"change_message_id":"c9520e4ec5228ee1d6a459f543797166f20ff1f1","unresolved":false,"context_lines":[{"line_number":101,"context_line":"Alternatives"},{"line_number":102,"context_line":"------------"},{"line_number":103,"context_line":""},{"line_number":104,"context_line":"* We could add limit to ``GET /resource_providers`` too."},{"line_number":105,"context_line":""},{"line_number":106,"context_line":"* We could set a relatively low default value for limit."},{"line_number":107,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"7f515b1d_c48d2f32","line":104,"in_reply_to":"7f515b1d_a42233a3","updated":"2017-09-25 18:44:14.000000000","message":"It\u0027s a statement of \"we can expand the scope to cover this other thing that is essentially the same\"\n\nespecially given as it is not much more work","commit_id":"eb2ff0a94f449253f2a339f1aae2586c2fba4173"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"426b5bd9a3a5b753f75bf63e0ede8be944772a7b","unresolved":false,"context_lines":[{"line_number":101,"context_line":"Alternatives"},{"line_number":102,"context_line":"------------"},{"line_number":103,"context_line":""},{"line_number":104,"context_line":"* We could add limit to ``GET /resource_providers`` too."},{"line_number":105,"context_line":""},{"line_number":106,"context_line":"* We could set a relatively low default value for limit."},{"line_number":107,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"7f515b1d_d53f9884","line":104,"in_reply_to":"7f515b1d_c48d2f32","updated":"2017-09-29 19:26:39.000000000","message":"yes but in that case would you want a random sampleing or just pagination.","commit_id":"eb2ff0a94f449253f2a339f1aae2586c2fba4173"},{"author":{"_account_id":1063,"name":"Ed Leafe","email":"ed@leafe.com","username":"ed-leafe"},"change_message_id":"cf740f6d542ccf31736f060be00fd67754758397","unresolved":false,"context_lines":[{"line_number":109,"context_line":"  worst case scenario for which there exist other workarounds such as making"},{"line_number":110,"context_line":"  flavors and other boot request impacting structures have sufficient detail to"},{"line_number":111,"context_line":"  always limit the size of the response. We can also say that 10,000 results"},{"line_number":112,"context_line":"  really isn\u0027t that big of a deal."},{"line_number":113,"context_line":""},{"line_number":114,"context_line":"* We can sort by an existing field (such as resource provider uuid) when doing"},{"line_number":115,"context_line":"  a limit, effectively setting placement bias to pack."}],"source_content_type":"text/x-rst","patch_set":1,"id":"7f515b1d_646afbb3","line":112,"updated":"2017-09-25 18:25:55.000000000","message":"That\u0027s 10,000 results *after* applying any resource filtering. Doesn\u0027t seem like a likely scenario.","commit_id":"eb2ff0a94f449253f2a339f1aae2586c2fba4173"},{"author":{"_account_id":11564,"name":"Chris Dent","email":"cdent@anticdent.org","username":"chdent"},"change_message_id":"c9520e4ec5228ee1d6a459f543797166f20ff1f1","unresolved":false,"context_lines":[{"line_number":109,"context_line":"  worst case scenario for which there exist other workarounds such as making"},{"line_number":110,"context_line":"  flavors and other boot request impacting structures have sufficient detail to"},{"line_number":111,"context_line":"  always limit the size of the response. We can also say that 10,000 results"},{"line_number":112,"context_line":"  really isn\u0027t that big of a deal."},{"line_number":113,"context_line":""},{"line_number":114,"context_line":"* We can sort by an existing field (such as resource provider uuid) when doing"},{"line_number":115,"context_line":"  a limit, effectively setting placement bias to pack."}],"source_content_type":"text/x-rst","patch_set":1,"id":"7f515b1d_a499136f","line":112,"in_reply_to":"7f515b1d_646afbb3","updated":"2017-09-25 18:44:14.000000000","message":"empty cloud with 10,000 computes, and I request one compute node with 1VCPU, 1GB RAM and no disk. 10,000 results back.\n\nOr cloud with 100,000 computes, 90% used, still 10,000 results back.","commit_id":"eb2ff0a94f449253f2a339f1aae2586c2fba4173"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"8f9aad5303a99d28ded0c36f95167dc374ef10cc","unresolved":false,"context_lines":[{"line_number":109,"context_line":"  worst case scenario for which there exist other workarounds such as making"},{"line_number":110,"context_line":"  flavors and other boot request impacting structures have sufficient detail to"},{"line_number":111,"context_line":"  always limit the size of the response. We can also say that 10,000 results"},{"line_number":112,"context_line":"  really isn\u0027t that big of a deal."},{"line_number":113,"context_line":""},{"line_number":114,"context_line":"* We can sort by an existing field (such as resource provider uuid) when doing"},{"line_number":115,"context_line":"  a limit, effectively setting placement bias to pack."}],"source_content_type":"text/x-rst","patch_set":1,"id":"7f515b1d_1fedd18a","line":112,"in_reply_to":"7f515b1d_a499136f","updated":"2017-09-25 21:18:09.000000000","message":"The problem compounds with nested RPs where we can return multiple permutations within the same compute host.  E.g. if I have 16 NUMA nodes per host, it\u0027s easy to envision a request where I only need 625 empty hosts to get to 10,000 results.  Dot-product that with SR-IOV VFs, vGPUs, etc. and...","commit_id":"eb2ff0a94f449253f2a339f1aae2586c2fba4173"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"9f8442333264b0d3e3c427b9114cf7692e095239","unresolved":false,"context_lines":[{"line_number":130,"context_line":"  different levels of support that WSGI servers have for threading and signal"},{"line_number":131,"context_line":"  handling."},{"line_number":132,"context_line":""},{"line_number":133,"context_line":"* We can document that people who want spread handling should run a cron job to"},{"line_number":134,"context_line":"  update the column."},{"line_number":135,"context_line":""},{"line_number":136,"context_line":"* We could adjust the update task so it can be run either once (ever), once per"},{"line_number":137,"context_line":"  process, or on the interval described by configuration."}],"source_content_type":"text/x-rst","patch_set":1,"id":"7f515b1d_4395bc3f","line":134,"range":{"start_line":133,"start_character":2,"end_line":134,"end_character":20},"updated":"2017-09-25 15:35:05.000000000","message":"bingo.","commit_id":"eb2ff0a94f449253f2a339f1aae2586c2fba4173"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"20a94035520d971f28dea54c0f17e5ea81cf1a8f","unresolved":false,"context_lines":[{"line_number":132,"context_line":""},{"line_number":133,"context_line":"* We can document that people who want spread handling should run a cron job to"},{"line_number":134,"context_line":"  update the column."},{"line_number":135,"context_line":""},{"line_number":136,"context_line":"* We could adjust the update task so it can be run either once (ever), once per"},{"line_number":137,"context_line":"  process, or on the interval described by configuration."},{"line_number":138,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"7f515b1d_4a010505","line":135,"updated":"2017-09-27 13:53:44.000000000","message":"As another alternative mentioned in the ptg the placement service could update the weight column of the resource provider to a new random integer when an allocation happens against the resource provider. This way we could avoid the need for a periodic update. This would not change every provider\u0027s weight just those that has changed allocations. But I think it is enough to avoid packing in general.","commit_id":"eb2ff0a94f449253f2a339f1aae2586c2fba4173"},{"author":{"_account_id":1063,"name":"Ed Leafe","email":"ed@leafe.com","username":"ed-leafe"},"change_message_id":"cf740f6d542ccf31736f060be00fd67754758397","unresolved":false,"context_lines":[{"line_number":135,"context_line":""},{"line_number":136,"context_line":"* We could adjust the update task so it can be run either once (ever), once per"},{"line_number":137,"context_line":"  process, or on the interval described by configuration."},{"line_number":138,"context_line":""},{"line_number":139,"context_line":"For a first pass, the relatively simple model proposed above will work and can"},{"line_number":140,"context_line":"be incrementally improved or adjusted if there are issues."},{"line_number":141,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"7f515b1d_c463efb0","line":138,"updated":"2017-09-25 18:25:55.000000000","message":"One more alternative: *don\u0027t* add any columns or cron jobs. Just change the query to:\n\n  SELECT fld1, fld2, ..., fldN, RAND() as weight\n  FROM resource_providers\n  WHERE blah\u003dduh\n  ORDER BY weight\n  LIMIT 10000","commit_id":"eb2ff0a94f449253f2a339f1aae2586c2fba4173"},{"author":{"_account_id":11564,"name":"Chris Dent","email":"cdent@anticdent.org","username":"chdent"},"change_message_id":"c9520e4ec5228ee1d6a459f543797166f20ff1f1","unresolved":false,"context_lines":[{"line_number":135,"context_line":""},{"line_number":136,"context_line":"* We could adjust the update task so it can be run either once (ever), once per"},{"line_number":137,"context_line":"  process, or on the interval described by configuration."},{"line_number":138,"context_line":""},{"line_number":139,"context_line":"For a first pass, the relatively simple model proposed above will work and can"},{"line_number":140,"context_line":"be incrementally improved or adjusted if there are issues."},{"line_number":141,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"7f515b1d_e406eb86","line":138,"in_reply_to":"7f515b1d_c463efb0","updated":"2017-09-25 18:44:14.000000000","message":"It looks like this presents the same problem as the scenario on line 40 above? An order by on a calculated column causes  a temporary and filesort:\n\nexplain select name, rand() as weight from resource_providers order by weight limit 10000;\n\n[...]  Using index; Using temporary; Using filesort \n\nFrom the ptg, we wanted to avoid that.","commit_id":"eb2ff0a94f449253f2a339f1aae2586c2fba4173"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"426b5bd9a3a5b753f75bf63e0ede8be944772a7b","unresolved":false,"context_lines":[{"line_number":135,"context_line":""},{"line_number":136,"context_line":"* We could adjust the update task so it can be run either once (ever), once per"},{"line_number":137,"context_line":"  process, or on the interval described by configuration."},{"line_number":138,"context_line":""},{"line_number":139,"context_line":"For a first pass, the relatively simple model proposed above will work and can"},{"line_number":140,"context_line":"be incrementally improved or adjusted if there are issues."},{"line_number":141,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"7f515b1d_95e920e7","line":138,"in_reply_to":"7f515b1d_e406eb86","updated":"2017-09-29 19:26:39.000000000","message":"checkout my example in the comment in line 100\nnot sure if the anonomous table i am generating will result i someting similar. but it should be better then rand.","commit_id":"eb2ff0a94f449253f2a339f1aae2586c2fba4173"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"20a94035520d971f28dea54c0f17e5ea81cf1a8f","unresolved":false,"context_lines":[{"line_number":176,"context_line":""},{"line_number":177,"context_line":"The purpose of this change is to enhance performance by limiting the size of"},{"line_number":178,"context_line":"generated result sets. However in an environment where there are a very large"},{"line_number":179,"context_line":"number of resource providers the update job could generate a lot of work."},{"line_number":180,"context_line":""},{"line_number":181,"context_line":"Other deployer impact"},{"line_number":182,"context_line":"---------------------"}],"source_content_type":"text/x-rst","patch_set":1,"id":"7f515b1d_aa72b979","line":179,"updated":"2017-09-27 13:53:44.000000000","message":"The work to update everything at once can be avoided in the alternative I described in L135","commit_id":"eb2ff0a94f449253f2a339f1aae2586c2fba4173"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"ee5958775a990d5659ac6ebaa4f929ca9f4410cb","unresolved":false,"context_lines":[{"line_number":33,"context_line":"---------"},{"line_number":34,"context_line":""},{"line_number":35,"context_line":"As a client of the placement service I would like to optionally be able to"},{"line_number":36,"context_line":"request a limited number of allocation candidates in a performant fashion."},{"line_number":37,"context_line":""},{"line_number":38,"context_line":"Proposed change"},{"line_number":39,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":2,"id":"7f515b1d_4e43fc42","line":36,"range":{"start_line":36,"start_character":49,"end_line":36,"end_character":73},"updated":"2017-10-02 17:01:25.000000000","message":"remove this, it\u0027s not necessary for the use case.","commit_id":"30d0c5cff8bfe8457be154579897ef46046408db"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"2ee7cf2612fa0bc5365734456912e28ad4074c1c","unresolved":false,"context_lines":[{"line_number":33,"context_line":"---------"},{"line_number":34,"context_line":""},{"line_number":35,"context_line":"As a client of the placement service I would like to optionally be able to"},{"line_number":36,"context_line":"request a limited number of allocation candidates in a performant fashion."},{"line_number":37,"context_line":""},{"line_number":38,"context_line":"Proposed change"},{"line_number":39,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":2,"id":"7f515b1d_f36bb081","line":36,"range":{"start_line":36,"start_character":49,"end_line":36,"end_character":73},"in_reply_to":"7f515b1d_10c9a257","updated":"2017-10-02 19:18:00.000000000","message":"yes looks like the oxford dictionary has recently added\nthe adjective form but in general the noun form is the only recognized form.\nhttps://en.oxforddictionaries.com/definition/performant\n\n\n\n\ni taught there was previous guidance form the docs team\nregarding the use of performant however it does not appear to be listed in \nhttps://docs.openstack.org/doc-contrib-guide/writing-style/word-choice.html\n\nor \n\nhttps://docs.openstack.org/doc-contrib-guide/writing-style/general-writing-guidelines.html\n\nso I guess it was just in my own bias, that said it is not recognised in either form by Merriam-Webster’s Collegiate Dictionary as the word is only recognised as part of the uk branch of English.\n\nanyway not to rat hole but i agree that this section can be removed as the usecase does not need to contain performance requirement in this case.","commit_id":"30d0c5cff8bfe8457be154579897ef46046408db"},{"author":{"_account_id":11564,"name":"Chris Dent","email":"cdent@anticdent.org","username":"chdent"},"change_message_id":"34cc19be90d86927fb647be424d004dfb57e856b","unresolved":false,"context_lines":[{"line_number":33,"context_line":"---------"},{"line_number":34,"context_line":""},{"line_number":35,"context_line":"As a client of the placement service I would like to optionally be able to"},{"line_number":36,"context_line":"request a limited number of allocation candidates in a performant fashion."},{"line_number":37,"context_line":""},{"line_number":38,"context_line":"Proposed change"},{"line_number":39,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":2,"id":"7f515b1d_41261f6a","line":36,"range":{"start_line":36,"start_character":49,"end_line":36,"end_character":73},"in_reply_to":"7f515b1d_4e43fc42","updated":"2017-10-03 10:39:36.000000000","message":"Done","commit_id":"30d0c5cff8bfe8457be154579897ef46046408db"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"180eda5bab704af92d7de44861c6cde2a1b7a8f2","unresolved":false,"context_lines":[{"line_number":33,"context_line":"---------"},{"line_number":34,"context_line":""},{"line_number":35,"context_line":"As a client of the placement service I would like to optionally be able to"},{"line_number":36,"context_line":"request a limited number of allocation candidates in a performant fashion."},{"line_number":37,"context_line":""},{"line_number":38,"context_line":"Proposed change"},{"line_number":39,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":2,"id":"7f515b1d_ee6628bb","line":36,"range":{"start_line":36,"start_character":49,"end_line":36,"end_character":73},"in_reply_to":"7f515b1d_4e43fc42","updated":"2017-10-02 17:23:36.000000000","message":"well you did want to limit this to an O(1) algorithm but that can me covered in the proposed changes i guess.\n\nthat said performant is not a word even though we all know what it means.","commit_id":"30d0c5cff8bfe8457be154579897ef46046408db"},{"author":{"_account_id":1063,"name":"Ed Leafe","email":"ed@leafe.com","username":"ed-leafe"},"change_message_id":"233d3f001fa757c7300f341f2a094a1ff8cca660","unresolved":false,"context_lines":[{"line_number":33,"context_line":"---------"},{"line_number":34,"context_line":""},{"line_number":35,"context_line":"As a client of the placement service I would like to optionally be able to"},{"line_number":36,"context_line":"request a limited number of allocation candidates in a performant fashion."},{"line_number":37,"context_line":""},{"line_number":38,"context_line":"Proposed change"},{"line_number":39,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":2,"id":"7f515b1d_10c9a257","line":36,"range":{"start_line":36,"start_character":49,"end_line":36,"end_character":73},"in_reply_to":"7f515b1d_ee6628bb","updated":"2017-10-02 18:05:30.000000000","message":"Well, it is a word, but not the way we use it.\n\nhttp://www.dictionary.com/browse/performant","commit_id":"30d0c5cff8bfe8457be154579897ef46046408db"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"9f2339202df66339ff8ab91ddf43a7d6a03f9445","unresolved":false,"context_lines":[{"line_number":53,"context_line":"  `allocation_requests` of the size defined by the ``limit`` and provide"},{"line_number":54,"context_line":"  `provider_summaries` of only those providers mentioned in the requests."},{"line_number":55,"context_line":""},{"line_number":56,"context_line":"* That slice is either the first `N` items or a random sampling depending on"},{"line_number":57,"context_line":"  the value of a configuration setting (name to be determined) which allows a"},{"line_number":58,"context_line":"  deployment to choose pack or spread behavior in the results."},{"line_number":59,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"7f515b1d_ce300ca2","line":56,"range":{"start_line":56,"start_character":28,"end_line":56,"end_character":36},"updated":"2017-10-02 17:12:19.000000000","message":"without an ORDER BY, \"first N\" is meaningless...","commit_id":"30d0c5cff8bfe8457be154579897ef46046408db"},{"author":{"_account_id":11564,"name":"Chris Dent","email":"cdent@anticdent.org","username":"chdent"},"change_message_id":"505b74516375e899a93f1a63056b0441c79b2718","unresolved":false,"context_lines":[{"line_number":53,"context_line":"  `allocation_requests` of the size defined by the ``limit`` and provide"},{"line_number":54,"context_line":"  `provider_summaries` of only those providers mentioned in the requests."},{"line_number":55,"context_line":""},{"line_number":56,"context_line":"* That slice is either the first `N` items or a random sampling depending on"},{"line_number":57,"context_line":"  the value of a configuration setting (name to be determined) which allows a"},{"line_number":58,"context_line":"  deployment to choose pack or spread behavior in the results."},{"line_number":59,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"7f515b1d_ee8bc8bc","line":56,"range":{"start_line":56,"start_character":28,"end_line":56,"end_character":36},"in_reply_to":"7f515b1d_ce300ca2","updated":"2017-10-02 17:39:22.000000000","message":"So here\u0027s where we get back in to the rub that has rubbed us raw before: We\u0027ve asserted that there is an ordering to the database that is the \"natural\" order, and that\u0027s why we proposed the \"weight\" column so we could have a \"real\" thing to randomize on.\n\nThe idea proposed here is that we either preserve that natural order, by taking the first N (getting something akin to pack and pretty much the same order we currently present), or random.sample(), based on the config.","commit_id":"30d0c5cff8bfe8457be154579897ef46046408db"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"2ee7cf2612fa0bc5365734456912e28ad4074c1c","unresolved":false,"context_lines":[{"line_number":53,"context_line":"  `allocation_requests` of the size defined by the ``limit`` and provide"},{"line_number":54,"context_line":"  `provider_summaries` of only those providers mentioned in the requests."},{"line_number":55,"context_line":""},{"line_number":56,"context_line":"* That slice is either the first `N` items or a random sampling depending on"},{"line_number":57,"context_line":"  the value of a configuration setting (name to be determined) which allows a"},{"line_number":58,"context_line":"  deployment to choose pack or spread behavior in the results."},{"line_number":59,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"7f515b1d_3315c840","line":56,"range":{"start_line":56,"start_character":28,"end_line":56,"end_character":36},"in_reply_to":"7f515b1d_ee8bc8bc","updated":"2017-10-02 19:18:00.000000000","message":"well the nautal order that the db returns the results in is not stable and will change based on 1 the db used and 2 the whim of the query optimizer and the execution plan it generates.\n\non a busy cloud natural order will in some case be random and in others bias to packing hosts.","commit_id":"30d0c5cff8bfe8457be154579897ef46046408db"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"180eda5bab704af92d7de44861c6cde2a1b7a8f2","unresolved":false,"context_lines":[{"line_number":53,"context_line":"  `allocation_requests` of the size defined by the ``limit`` and provide"},{"line_number":54,"context_line":"  `provider_summaries` of only those providers mentioned in the requests."},{"line_number":55,"context_line":""},{"line_number":56,"context_line":"* That slice is either the first `N` items or a random sampling depending on"},{"line_number":57,"context_line":"  the value of a configuration setting (name to be determined) which allows a"},{"line_number":58,"context_line":"  deployment to choose pack or spread behavior in the results."},{"line_number":59,"context_line":""},{"line_number":60,"context_line":"This solution will mean a smaller dataset being tranformed into JSON and"},{"line_number":61,"context_line":"transmitted over the wire to the candidate, and a smaller number of objects"}],"source_content_type":"text/x-rst","patch_set":2,"id":"7f515b1d_6e9a3862","line":58,"range":{"start_line":56,"start_character":1,"end_line":58,"end_character":62},"updated":"2017-10-02 17:23:36.000000000","message":"well this is not quite correct. it will always be be the first N. what changes is how that list is generated natural vs random ordering correct? i assume you are not intending to do the filtering in python server side correct?","commit_id":"30d0c5cff8bfe8457be154579897ef46046408db"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"2ee7cf2612fa0bc5365734456912e28ad4074c1c","unresolved":false,"context_lines":[{"line_number":53,"context_line":"  `allocation_requests` of the size defined by the ``limit`` and provide"},{"line_number":54,"context_line":"  `provider_summaries` of only those providers mentioned in the requests."},{"line_number":55,"context_line":""},{"line_number":56,"context_line":"* That slice is either the first `N` items or a random sampling depending on"},{"line_number":57,"context_line":"  the value of a configuration setting (name to be determined) which allows a"},{"line_number":58,"context_line":"  deployment to choose pack or spread behavior in the results."},{"line_number":59,"context_line":""},{"line_number":60,"context_line":"This solution will mean a smaller dataset being tranformed into JSON and"},{"line_number":61,"context_line":"transmitted over the wire to the candidate, and a smaller number of objects"}],"source_content_type":"text/x-rst","patch_set":2,"id":"7f515b1d_13f64cd8","line":58,"range":{"start_line":56,"start_character":1,"end_line":58,"end_character":62},"in_reply_to":"7f515b1d_2e7440ce","updated":"2017-10-02 19:18:00.000000000","message":"ok just read the summary comment. \ni think doing it in python would be an ok start and i do not and the actually implementation should not leak through the api in any case but i dont want to block future change that could move this to the db level i we need it for performance reasons in the future.","commit_id":"30d0c5cff8bfe8457be154579897ef46046408db"},{"author":{"_account_id":11564,"name":"Chris Dent","email":"cdent@anticdent.org","username":"chdent"},"change_message_id":"505b74516375e899a93f1a63056b0441c79b2718","unresolved":false,"context_lines":[{"line_number":53,"context_line":"  `allocation_requests` of the size defined by the ``limit`` and provide"},{"line_number":54,"context_line":"  `provider_summaries` of only those providers mentioned in the requests."},{"line_number":55,"context_line":""},{"line_number":56,"context_line":"* That slice is either the first `N` items or a random sampling depending on"},{"line_number":57,"context_line":"  the value of a configuration setting (name to be determined) which allows a"},{"line_number":58,"context_line":"  deployment to choose pack or spread behavior in the results."},{"line_number":59,"context_line":""},{"line_number":60,"context_line":"This solution will mean a smaller dataset being tranformed into JSON and"},{"line_number":61,"context_line":"transmitted over the wire to the candidate, and a smaller number of objects"}],"source_content_type":"text/x-rst","patch_set":2,"id":"7f515b1d_2e7440ce","line":58,"range":{"start_line":56,"start_character":1,"end_line":58,"end_character":62},"in_reply_to":"7f515b1d_6e9a3862","updated":"2017-10-02 17:39:22.000000000","message":"No, I am intending to do the filtering in python. More detail on that in the summary comment.","commit_id":"30d0c5cff8bfe8457be154579897ef46046408db"},{"author":{"_account_id":1063,"name":"Ed Leafe","email":"ed@leafe.com","username":"ed-leafe"},"change_message_id":"4480ba9dca90617a1bfe3f226db9f07d6d4b54fa","unresolved":false,"context_lines":[{"line_number":104,"context_line":""},{"line_number":105,"context_line":"The purpose of this change is to enhance global performance by limiting the"},{"line_number":106,"context_line":"size of result sets sent over the wire. However in an environment where there"},{"line_number":107,"context_line":"are a very large number of resource providers the database query will still"},{"line_number":108,"context_line":"have a large result set."},{"line_number":109,"context_line":""},{"line_number":110,"context_line":"Other deployer impact"},{"line_number":111,"context_line":"---------------------"}],"source_content_type":"text/x-rst","patch_set":2,"id":"7f515b1d_8ef1f403","line":108,"range":{"start_line":107,"start_character":46,"end_line":108,"end_character":24},"updated":"2017-10-02 17:19:57.000000000","message":"I\u0027d like to get an idea as to just how problematic this is before anyone starts complaining about this. A query with 10K results should not be a problem for a modern database system.","commit_id":"30d0c5cff8bfe8457be154579897ef46046408db"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"2ee7cf2612fa0bc5365734456912e28ad4074c1c","unresolved":false,"context_lines":[{"line_number":104,"context_line":""},{"line_number":105,"context_line":"The purpose of this change is to enhance global performance by limiting the"},{"line_number":106,"context_line":"size of result sets sent over the wire. However in an environment where there"},{"line_number":107,"context_line":"are a very large number of resource providers the database query will still"},{"line_number":108,"context_line":"have a large result set."},{"line_number":109,"context_line":""},{"line_number":110,"context_line":"Other deployer impact"},{"line_number":111,"context_line":"---------------------"}],"source_content_type":"text/x-rst","patch_set":2,"id":"7f515b1d_d3a554a0","line":108,"range":{"start_line":107,"start_character":46,"end_line":108,"end_character":24},"in_reply_to":"7f515b1d_8ef1f403","updated":"2017-10-02 19:18:00.000000000","message":"that is not the motivation behind this change.\nthe issue is the memory and bandwidth implications of serialisation, transmitting and deserialising 10K json object into python objects. with nested resource providers that would become a multi megabyte response rather quickly. hence the desire to filter server side. preferably in the db for speed but in python if we must.","commit_id":"30d0c5cff8bfe8457be154579897ef46046408db"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"ee5958775a990d5659ac6ebaa4f929ca9f4410cb","unresolved":false,"context_lines":[{"line_number":115,"context_line":"Developer impact"},{"line_number":116,"context_line":"----------------"},{"line_number":117,"context_line":""},{"line_number":118,"context_line":"The new configuration item will need to be changed for those deployments that"},{"line_number":119,"context_line":"do not want the default."},{"line_number":120,"context_line":""},{"line_number":121,"context_line":"Implementation"},{"line_number":122,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":2,"id":"7f515b1d_8e7c5472","line":119,"range":{"start_line":118,"start_character":0,"end_line":119,"end_character":24},"updated":"2017-10-02 17:01:25.000000000","message":"What is the proposed name of this new configuration option? Also, why the need for a new configuration option? What value does this give operators?","commit_id":"30d0c5cff8bfe8457be154579897ef46046408db"},{"author":{"_account_id":11564,"name":"Chris Dent","email":"cdent@anticdent.org","username":"chdent"},"change_message_id":"b9096c59cc901136f22fde098e804c76d9ee8e78","unresolved":false,"context_lines":[{"line_number":115,"context_line":"Developer impact"},{"line_number":116,"context_line":"----------------"},{"line_number":117,"context_line":""},{"line_number":118,"context_line":"The new configuration item will need to be changed for those deployments that"},{"line_number":119,"context_line":"do not want the default."},{"line_number":120,"context_line":""},{"line_number":121,"context_line":"Implementation"},{"line_number":122,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":2,"id":"7f515b1d_aed7104b","line":119,"range":{"start_line":118,"start_character":0,"end_line":119,"end_character":24},"in_reply_to":"7f515b1d_8e7c5472","updated":"2017-10-02 17:04:45.000000000","message":"see line 57 (it\u0027s tbd, because I have no idea what to name it)","commit_id":"30d0c5cff8bfe8457be154579897ef46046408db"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"2ee7cf2612fa0bc5365734456912e28ad4074c1c","unresolved":false,"context_lines":[{"line_number":115,"context_line":"Developer impact"},{"line_number":116,"context_line":"----------------"},{"line_number":117,"context_line":""},{"line_number":118,"context_line":"The new configuration item will need to be changed for those deployments that"},{"line_number":119,"context_line":"do not want the default."},{"line_number":120,"context_line":""},{"line_number":121,"context_line":"Implementation"},{"line_number":122,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":2,"id":"7f515b1d_73e580db","line":119,"range":{"start_line":118,"start_character":0,"end_line":119,"end_character":24},"in_reply_to":"7f515b1d_ae0b908b","updated":"2017-10-02 19:18:00.000000000","message":"well that is locking into semantics into the config.\nnatual db order does not guarntee packing so we would have to actually sort the hosts by some utilisation criteria  of we had a pack or spread option.\n\nso really we would want to use {natural,random} as the enum values however that might requires us to introduce periodic task if the performance of the python version i not sufficient.\n\nafter all when this is introduced the intention is that all calls from the scheduler/conductor will use limit so while performance is not the main goal it has to preform well enough\nto not impact scheduling measurably.","commit_id":"30d0c5cff8bfe8457be154579897ef46046408db"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"abb3b3968a0a1b86da18970ffcd8e95e07c7eacf","unresolved":false,"context_lines":[{"line_number":115,"context_line":"Developer impact"},{"line_number":116,"context_line":"----------------"},{"line_number":117,"context_line":""},{"line_number":118,"context_line":"The new configuration item will need to be changed for those deployments that"},{"line_number":119,"context_line":"do not want the default."},{"line_number":120,"context_line":""},{"line_number":121,"context_line":"Implementation"},{"line_number":122,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":2,"id":"7f515b1d_ae0b908b","line":119,"range":{"start_line":118,"start_character":0,"end_line":119,"end_character":24},"in_reply_to":"7f515b1d_aed7104b","updated":"2017-10-02 17:48:07.000000000","message":"[placement]get_allocation_candidates_limit_distribution_strategy \u003d {pack|spread}","commit_id":"30d0c5cff8bfe8457be154579897ef46046408db"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"ee5958775a990d5659ac6ebaa4f929ca9f4410cb","unresolved":false,"context_lines":[{"line_number":119,"context_line":"do not want the default."},{"line_number":120,"context_line":""},{"line_number":121,"context_line":"Implementation"},{"line_number":122,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":123,"context_line":""},{"line_number":124,"context_line":"Assignee(s)"},{"line_number":125,"context_line":"-----------"}],"source_content_type":"text/x-rst","patch_set":2,"id":"7f515b1d_ee8aa861","line":122,"updated":"2017-10-02 17:01:25.000000000","message":"It\u0027s the implementation that\u0027s the actual meat of this proposal and the thing that\u0027s interesting and affects the performance and scale of the operation... I don\u0027t understand why there is no detail here on the implementation.","commit_id":"30d0c5cff8bfe8457be154579897ef46046408db"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"abb3b3968a0a1b86da18970ffcd8e95e07c7eacf","unresolved":false,"context_lines":[{"line_number":119,"context_line":"do not want the default."},{"line_number":120,"context_line":""},{"line_number":121,"context_line":"Implementation"},{"line_number":122,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":123,"context_line":""},{"line_number":124,"context_line":"Assignee(s)"},{"line_number":125,"context_line":"-----------"}],"source_content_type":"text/x-rst","patch_set":2,"id":"7f515b1d_50583a2f","line":122,"in_reply_to":"7f515b1d_0e76243f","updated":"2017-10-02 17:48:07.000000000","message":"That last thing would be:\n\n if CONF.placement.get_allocation_candidates_limit_distribution_strategy is SPREAD:\n     return random.sample(db_results, limit)\n elif CONF.placement.get_allocation_candidates_limit_distribution_strategy is PACK:\n     return db_results[:limit]\n else:\n     raise ValueError(\"Invalid get_allocation_candidates_limit_distribution_strategy \u0027%s\u0027\" % CONF.placement.get_allocation_candidates_limit_distribution_strategy)","commit_id":"30d0c5cff8bfe8457be154579897ef46046408db"},{"author":{"_account_id":1063,"name":"Ed Leafe","email":"ed@leafe.com","username":"ed-leafe"},"change_message_id":"4480ba9dca90617a1bfe3f226db9f07d6d4b54fa","unresolved":false,"context_lines":[{"line_number":119,"context_line":"do not want the default."},{"line_number":120,"context_line":""},{"line_number":121,"context_line":"Implementation"},{"line_number":122,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":123,"context_line":""},{"line_number":124,"context_line":"Assignee(s)"},{"line_number":125,"context_line":"-----------"}],"source_content_type":"text/x-rst","patch_set":2,"id":"7f515b1d_0e76243f","line":122,"in_reply_to":"7f515b1d_4ec89ca4","updated":"2017-10-02 17:19:57.000000000","message":"Implementation:\n\n  db_results \u003d run_the_query(filters_and_stuff)\n  import random\n  return random.sample(db_results, limit)","commit_id":"30d0c5cff8bfe8457be154579897ef46046408db"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"180eda5bab704af92d7de44861c6cde2a1b7a8f2","unresolved":false,"context_lines":[{"line_number":119,"context_line":"do not want the default."},{"line_number":120,"context_line":""},{"line_number":121,"context_line":"Implementation"},{"line_number":122,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":123,"context_line":""},{"line_number":124,"context_line":"Assignee(s)"},{"line_number":125,"context_line":"-----------"}],"source_content_type":"text/x-rst","patch_set":2,"id":"7f515b1d_2e7b2028","line":122,"in_reply_to":"7f515b1d_4ec89ca4","updated":"2017-10-02 17:23:36.000000000","message":"well we use this section to like to patches if there is poc code but yes we normally put the detail in the proposed change.","commit_id":"30d0c5cff8bfe8457be154579897ef46046408db"},{"author":{"_account_id":1063,"name":"Ed Leafe","email":"ed@leafe.com","username":"ed-leafe"},"change_message_id":"233d3f001fa757c7300f341f2a094a1ff8cca660","unresolved":false,"context_lines":[{"line_number":119,"context_line":"do not want the default."},{"line_number":120,"context_line":""},{"line_number":121,"context_line":"Implementation"},{"line_number":122,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":123,"context_line":""},{"line_number":124,"context_line":"Assignee(s)"},{"line_number":125,"context_line":"-----------"}],"source_content_type":"text/x-rst","patch_set":2,"id":"7f515b1d_f0d8461e","line":122,"in_reply_to":"7f515b1d_50583a2f","updated":"2017-10-02 18:05:30.000000000","message":"Well, you\u0027d never do the PACK in practice; you\u0027d just LIMIT the db query","commit_id":"30d0c5cff8bfe8457be154579897ef46046408db"},{"author":{"_account_id":11564,"name":"Chris Dent","email":"cdent@anticdent.org","username":"chdent"},"change_message_id":"b9096c59cc901136f22fde098e804c76d9ee8e78","unresolved":false,"context_lines":[{"line_number":119,"context_line":"do not want the default."},{"line_number":120,"context_line":""},{"line_number":121,"context_line":"Implementation"},{"line_number":122,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":123,"context_line":""},{"line_number":124,"context_line":"Assignee(s)"},{"line_number":125,"context_line":"-----------"}],"source_content_type":"text/x-rst","patch_set":2,"id":"7f515b1d_4ec89ca4","line":122,"in_reply_to":"7f515b1d_ee8aa861","updated":"2017-10-02 17:04:45.000000000","message":"We never put much of anything in this section, see : http://specs.openstack.org/openstack/nova-specs/specs/queens/approved/migration-allocations.html#implementation\n\nInstead we put it in the proposed change, which is where it is in this case. Line 52 is the real guts.","commit_id":"30d0c5cff8bfe8457be154579897ef46046408db"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"2ee7cf2612fa0bc5365734456912e28ad4074c1c","unresolved":false,"context_lines":[{"line_number":119,"context_line":"do not want the default."},{"line_number":120,"context_line":""},{"line_number":121,"context_line":"Implementation"},{"line_number":122,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":123,"context_line":""},{"line_number":124,"context_line":"Assignee(s)"},{"line_number":125,"context_line":"-----------"}],"source_content_type":"text/x-rst","patch_set":2,"id":"7f515b1d_d31434eb","line":122,"in_reply_to":"7f515b1d_f0d8461e","updated":"2017-10-02 19:18:00.000000000","message":"well with this proposal we are not adjusting the db query at all so no you would do db_results[:limit] as eric suggested.","commit_id":"30d0c5cff8bfe8457be154579897ef46046408db"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"4d19434bb60764f580d1fb9ce36ca713c6bc227e","unresolved":false,"context_lines":[{"line_number":152,"context_line":"Testing"},{"line_number":153,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":154,"context_line":""},{"line_number":155,"context_line":"N/A"},{"line_number":156,"context_line":""},{"line_number":157,"context_line":"Documentation Impact"},{"line_number":158,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":2,"id":"7f515b1d_0be968e4","line":155,"updated":"2017-10-03 08:46:20.000000000","message":"To be able to see if we have to improve the performance further we need some test env where we can create a lot of resource providers and then query allocation_candidates.","commit_id":"30d0c5cff8bfe8457be154579897ef46046408db"},{"author":{"_account_id":11564,"name":"Chris Dent","email":"cdent@anticdent.org","username":"chdent"},"change_message_id":"34cc19be90d86927fb647be424d004dfb57e856b","unresolved":false,"context_lines":[{"line_number":152,"context_line":"Testing"},{"line_number":153,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":154,"context_line":""},{"line_number":155,"context_line":"N/A"},{"line_number":156,"context_line":""},{"line_number":157,"context_line":"Documentation Impact"},{"line_number":158,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":2,"id":"7f515b1d_01d26728","line":155,"in_reply_to":"7f515b1d_0be968e4","updated":"2017-10-03 10:39:36.000000000","message":"I\u0027m not sure on how much of this we really need to do: we currently do nothing and accept it as okay. Now we\u0027re going to do less; presumably that will also be okay? I\u0027ll add a note here though.","commit_id":"30d0c5cff8bfe8457be154579897ef46046408db"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"5643c3efda66348798b41915b789c02e18070da1","unresolved":false,"context_lines":[{"line_number":163,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":164,"context_line":""},{"line_number":165,"context_line":"We may wish to provide some facilities for examining performance before and"},{"line_number":166,"context_line":"after this change, and after this change and before any further adjustments."},{"line_number":167,"context_line":""},{"line_number":168,"context_line":"Documentation Impact"},{"line_number":169,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":3,"id":"7f515b1d_a68f2674","line":166,"range":{"start_line":166,"start_character":19,"end_line":166,"end_character":76},"updated":"2017-10-03 14:44:05.000000000","message":"It think this is what I was after with my previous comment.","commit_id":"f0d60fa4c036d45bbadc712a09ff644f95fea81e"},{"author":{"_account_id":1063,"name":"Ed Leafe","email":"ed@leafe.com","username":"ed-leafe"},"change_message_id":"9082014bc1ad7e657f410df223603a6a028c6beb","unresolved":false,"context_lines":[{"line_number":56,"context_line":"* That slice is either the first `N` items or a random sampling depending on"},{"line_number":57,"context_line":"  the value of a boolean configuration setting"},{"line_number":58,"context_line":"  (``randomize_allocation_candidates``). This allows a deployment to choose"},{"line_number":59,"context_line":"  pack or spread behavior in the results. The default, ``False``, maintains"},{"line_number":60,"context_line":"  existing behavior."},{"line_number":61,"context_line":""},{"line_number":62,"context_line":"This solution will mean a smaller dataset being transformed into JSON and"},{"line_number":63,"context_line":"transmitted over the wire to the candidate, and a smaller number of objects"}],"source_content_type":"text/x-rst","patch_set":4,"id":"7f515b1d_465aa783","line":60,"range":{"start_line":59,"start_character":42,"end_line":60,"end_character":20},"updated":"2017-10-03 21:34:42.000000000","message":"nit: existing behavior is not to limit at all, so this is technically incorrect.","commit_id":"6a3e023d55fcbc5657f83e2611e3f3729e2e7ea9"},{"author":{"_account_id":11564,"name":"Chris Dent","email":"cdent@anticdent.org","username":"chdent"},"change_message_id":"b7f27ec7501f125a7667ef1a5e1b36e509cdc5d3","unresolved":false,"context_lines":[{"line_number":56,"context_line":"* That slice is either the first `N` items or a random sampling depending on"},{"line_number":57,"context_line":"  the value of a boolean configuration setting"},{"line_number":58,"context_line":"  (``randomize_allocation_candidates``). This allows a deployment to choose"},{"line_number":59,"context_line":"  pack or spread behavior in the results. The default, ``False``, maintains"},{"line_number":60,"context_line":"  existing behavior."},{"line_number":61,"context_line":""},{"line_number":62,"context_line":"This solution will mean a smaller dataset being transformed into JSON and"},{"line_number":63,"context_line":"transmitted over the wire to the candidate, and a smaller number of objects"}],"source_content_type":"text/x-rst","patch_set":4,"id":"7f515b1d_bcb3ae4d","line":60,"range":{"start_line":59,"start_character":42,"end_line":60,"end_character":20},"in_reply_to":"7f515b1d_465aa783","updated":"2017-10-03 22:31:22.000000000","message":"I meant the existing ordering behavior, which is \"natural\" which leads to pack. I can clarify if required.","commit_id":"6a3e023d55fcbc5657f83e2611e3f3729e2e7ea9"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"0c0fe8972baed119dc646e1877c3f8702c38c2c1","unresolved":false,"context_lines":[{"line_number":56,"context_line":"* That slice is either the first `N` items or a random sampling depending on"},{"line_number":57,"context_line":"  the value of a boolean configuration setting"},{"line_number":58,"context_line":"  (``randomize_allocation_candidates``). This allows a deployment to choose"},{"line_number":59,"context_line":"  pack or spread behavior in the results. The default, ``False``, maintains"},{"line_number":60,"context_line":"  existing behavior."},{"line_number":61,"context_line":""},{"line_number":62,"context_line":"This solution will mean a smaller dataset being transformed into JSON and"},{"line_number":63,"context_line":"transmitted over the wire to the candidate, and a smaller number of objects"}],"source_content_type":"text/x-rst","patch_set":4,"id":"5f4e5783_6dc74066","line":60,"range":{"start_line":59,"start_character":42,"end_line":60,"end_character":20},"in_reply_to":"7f515b1d_6a5d56e8","updated":"2017-10-13 20:38:13.000000000","message":"For two subsequent requests for the same thing, the order will be the same. Not guaranteed, but it will. In the few-flavors lots-of-empty-computes case, that will mean an undesirable packing.\n\nSo, I agree with ed\u0027s wording nit, although I think it\u0027s not quite as critical.","commit_id":"6a3e023d55fcbc5657f83e2611e3f3729e2e7ea9"},{"author":{"_account_id":11604,"name":"sean mooney","email":"smooney@redhat.com","username":"sean-k-mooney"},"change_message_id":"e240b64053fb1bd5930a15da491a65b976126fc8","unresolved":false,"context_lines":[{"line_number":56,"context_line":"* That slice is either the first `N` items or a random sampling depending on"},{"line_number":57,"context_line":"  the value of a boolean configuration setting"},{"line_number":58,"context_line":"  (``randomize_allocation_candidates``). This allows a deployment to choose"},{"line_number":59,"context_line":"  pack or spread behavior in the results. The default, ``False``, maintains"},{"line_number":60,"context_line":"  existing behavior."},{"line_number":61,"context_line":""},{"line_number":62,"context_line":"This solution will mean a smaller dataset being transformed into JSON and"},{"line_number":63,"context_line":"transmitted over the wire to the candidate, and a smaller number of objects"}],"source_content_type":"text/x-rst","patch_set":4,"id":"7f515b1d_6a5d56e8","line":60,"range":{"start_line":59,"start_character":42,"end_line":60,"end_character":20},"in_reply_to":"7f515b1d_bcb3ae4d","updated":"2017-10-04 12:03:36.000000000","message":"natural order will not always lead to packing so presenting it as such is not correct.\n\nnatural order of a query in a db that does not contain an order by is generally not stable and therfor may not bias towords packing. it would be betere to remove \"This allows a deployment to choose pack or spread behavior in the results.\"\n\nrandomize_allocation_candidates does exactly what it says. it will generate a random sampeling i true or use natural ordering if false. lets not layer other semantic on top.","commit_id":"6a3e023d55fcbc5657f83e2611e3f3729e2e7ea9"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"0c0fe8972baed119dc646e1877c3f8702c38c2c1","unresolved":false,"context_lines":[{"line_number":62,"context_line":"This solution will mean a smaller dataset being transformed into JSON and"},{"line_number":63,"context_line":"transmitted over the wire to the candidate, and a smaller number of objects"},{"line_number":64,"context_line":"being created server-side, but still a large number of rows being returned from"},{"line_number":65,"context_line":"database queries."},{"line_number":66,"context_line":""},{"line_number":67,"context_line":"Alternatives"},{"line_number":68,"context_line":"------------"}],"source_content_type":"text/x-rst","patch_set":4,"id":"5f4e5783_2d8ca834","line":65,"updated":"2017-10-13 20:38:13.000000000","message":"Why is this? If we limit the sql query we should get back the limit.","commit_id":"6a3e023d55fcbc5657f83e2611e3f3729e2e7ea9"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"eac03e1fdf61e7ce6bf79f791a107e03f92886d0","unresolved":false,"context_lines":[{"line_number":62,"context_line":"This solution will mean a smaller dataset being transformed into JSON and"},{"line_number":63,"context_line":"transmitted over the wire to the candidate, and a smaller number of objects"},{"line_number":64,"context_line":"being created server-side, but still a large number of rows being returned from"},{"line_number":65,"context_line":"database queries."},{"line_number":66,"context_line":""},{"line_number":67,"context_line":"Alternatives"},{"line_number":68,"context_line":"------------"}],"source_content_type":"text/x-rst","patch_set":4,"id":"5f4e5783_a823e655","line":65,"in_reply_to":"5f4e5783_28013606","updated":"2017-10-13 21:09:06.000000000","message":"I thought we were doing a randomization in SQL, but I clearly lost track of that crazy conversation at PTG.\n\nBut, ack on the random sampling in python for now and hope that we can do a little better for memory footprint in the future somehow.","commit_id":"6a3e023d55fcbc5657f83e2611e3f3729e2e7ea9"},{"author":{"_account_id":11564,"name":"Chris Dent","email":"cdent@anticdent.org","username":"chdent"},"change_message_id":"6e20ec973864b0c952ed19ecb99bd2fa9d119e00","unresolved":false,"context_lines":[{"line_number":62,"context_line":"This solution will mean a smaller dataset being transformed into JSON and"},{"line_number":63,"context_line":"transmitted over the wire to the candidate, and a smaller number of objects"},{"line_number":64,"context_line":"being created server-side, but still a large number of rows being returned from"},{"line_number":65,"context_line":"database queries."},{"line_number":66,"context_line":""},{"line_number":67,"context_line":"Alternatives"},{"line_number":68,"context_line":"------------"}],"source_content_type":"text/x-rst","patch_set":4,"id":"5f4e5783_28013606","line":65,"in_reply_to":"5f4e5783_2d8ca834","updated":"2017-10-13 20:54:05.000000000","message":"The proposed solution above is to limit the results _after_ the sql query has been run, because:\n\na) the creation of a set of allocation requests is the result of a few different sql queries and then the composition of some objects\n\nb) we can\u0027t do limit in sql without something to order on (see all the previous discussion on the previous reviews) so what\u0027s proposed instead is get all the results and take a random sample","commit_id":"6a3e023d55fcbc5657f83e2611e3f3729e2e7ea9"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"2049e94e8f62e7d6c57f22e106432e5862c419cb","unresolved":false,"context_lines":[{"line_number":62,"context_line":"This solution will mean a smaller dataset being transformed into JSON and"},{"line_number":63,"context_line":"transmitted over the wire to the candidate, and a smaller number of objects"},{"line_number":64,"context_line":"being created server-side, but still a large number of rows being returned from"},{"line_number":65,"context_line":"database queries."},{"line_number":66,"context_line":""},{"line_number":67,"context_line":"Alternatives"},{"line_number":68,"context_line":"------------"}],"source_content_type":"text/x-rst","patch_set":4,"id":"5f4e5783_638f5175","line":65,"in_reply_to":"5f4e5783_a823e655","updated":"2017-10-13 22:00:24.000000000","message":"Agree with Dan that I thought we could just limit in SQL, since pulling 10K+ records out of the database, shuffling and filtering them in python, and then limiting the results, helps with what goes over the wire and the nova-scheduler ultimately pulls from the database for compute nodes, but it doesn\u0027t help with the initial SQL queries done in Placement, which are also going to affect scheduling performance at large scale.\n\nI was also wondering how the limit is going to be handled with things like shared resource providers in aggregate relationships and nested resource providers, I\u0027m less clear on that stuff, i.e. if compute node RP1 is going to be returned which provides VCPU and MEMORY_MB but is in a sharing relationship with a shared storage provider providing DISK_GB, and we\u0027re at the limit, would we possibly cut at the compute node RP and not return the shared storage RP? That could be less than good.\n\nAnd for nested resource providers, how are those accounted for in said limit? Is an entire tree counted as 1 or is each node in the tree counted as 1?","commit_id":"6a3e023d55fcbc5657f83e2611e3f3729e2e7ea9"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"2049e94e8f62e7d6c57f22e106432e5862c419cb","unresolved":false,"context_lines":[{"line_number":70,"context_line":"There are many alternatives, most of which involve invasive changes in the"},{"line_number":71,"context_line":"database and establishing periodic jobs in the placement service."},{"line_number":72,"context_line":""},{"line_number":73,"context_line":"For a first pass, the relatively simple model proposed above will work and can"},{"line_number":74,"context_line":"be incrementally improved or adjusted if there are issues."},{"line_number":75,"context_line":""},{"line_number":76,"context_line":"Data model impact"},{"line_number":77,"context_line":"-----------------"}],"source_content_type":"text/x-rst","patch_set":4,"id":"5f4e5783_23eb9913","line":74,"range":{"start_line":73,"start_character":0,"end_line":74,"end_character":58},"updated":"2017-10-13 22:00:24.000000000","message":"Yeah, agree, stem the bleeding for large deployments now and improve on this later.","commit_id":"6a3e023d55fcbc5657f83e2611e3f3729e2e7ea9"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"2049e94e8f62e7d6c57f22e106432e5862c419cb","unresolved":false,"context_lines":[{"line_number":120,"context_line":"Developer impact"},{"line_number":121,"context_line":"----------------"},{"line_number":122,"context_line":""},{"line_number":123,"context_line":"The new configuration item will need to be changed for those deployments that"},{"line_number":124,"context_line":"do not want the default."},{"line_number":125,"context_line":""},{"line_number":126,"context_line":"Implementation"},{"line_number":127,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":4,"id":"5f4e5783_63e4f13c","line":124,"range":{"start_line":123,"start_character":0,"end_line":124,"end_character":24},"updated":"2017-10-13 22:00:24.000000000","message":"Did you mean to put this in the deployer impact section?","commit_id":"6a3e023d55fcbc5657f83e2611e3f3729e2e7ea9"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"0c0fe8972baed119dc646e1877c3f8702c38c2c1","unresolved":false,"context_lines":[{"line_number":152,"context_line":"* Update functional tests to confirm the limit and randomization functionality"},{"line_number":153,"context_line":"  of ``get_by_filters``."},{"line_number":154,"context_line":"* Update gabbi tests for ``GET /allocation_candidates`` to exercise the"},{"line_number":155,"context_line":"  ``limit`` functionality."},{"line_number":156,"context_line":""},{"line_number":157,"context_line":"Dependencies"},{"line_number":158,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":4,"id":"5f4e5783_2dba68c0","line":155,"updated":"2017-10-13 20:38:13.000000000","message":"Where in here is the randomizing?","commit_id":"6a3e023d55fcbc5657f83e2611e3f3729e2e7ea9"},{"author":{"_account_id":11564,"name":"Chris Dent","email":"cdent@anticdent.org","username":"chdent"},"change_message_id":"6e20ec973864b0c952ed19ecb99bd2fa9d119e00","unresolved":false,"context_lines":[{"line_number":152,"context_line":"* Update functional tests to confirm the limit and randomization functionality"},{"line_number":153,"context_line":"  of ``get_by_filters``."},{"line_number":154,"context_line":"* Update gabbi tests for ``GET /allocation_candidates`` to exercise the"},{"line_number":155,"context_line":"  ``limit`` functionality."},{"line_number":156,"context_line":""},{"line_number":157,"context_line":"Dependencies"},{"line_number":158,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":4,"id":"5f4e5783_a83d26c5","line":155,"in_reply_to":"5f4e5783_2dba68c0","updated":"2017-10-13 20:54:05.000000000","message":"An oversight, but if you\u0027ll accept \"processing\" to mean \"doing all the parts explained in the proposed change\" we\u0027re set","commit_id":"6a3e023d55fcbc5657f83e2611e3f3729e2e7ea9"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"eac03e1fdf61e7ce6bf79f791a107e03f92886d0","unresolved":false,"context_lines":[{"line_number":152,"context_line":"* Update functional tests to confirm the limit and randomization functionality"},{"line_number":153,"context_line":"  of ``get_by_filters``."},{"line_number":154,"context_line":"* Update gabbi tests for ``GET /allocation_candidates`` to exercise the"},{"line_number":155,"context_line":"  ``limit`` functionality."},{"line_number":156,"context_line":""},{"line_number":157,"context_line":"Dependencies"},{"line_number":158,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":4,"id":"5f4e5783_68196e20","line":155,"in_reply_to":"5f4e5783_a83d26c5","updated":"2017-10-13 21:09:06.000000000","message":"Ack.","commit_id":"6a3e023d55fcbc5657f83e2611e3f3729e2e7ea9"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"2049e94e8f62e7d6c57f22e106432e5862c419cb","unresolved":false,"context_lines":[{"line_number":162,"context_line":"Testing"},{"line_number":163,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":164,"context_line":""},{"line_number":165,"context_line":"We may wish to provide some facilities for examining performance before and"},{"line_number":166,"context_line":"after this change, and after this change and before any further adjustments."},{"line_number":167,"context_line":""},{"line_number":168,"context_line":"Documentation Impact"},{"line_number":169,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":4,"id":"5f4e5783_033e5d8b","line":166,"range":{"start_line":165,"start_character":0,"end_line":166,"end_character":76},"updated":"2017-10-13 22:00:24.000000000","message":"Glad you mentioned this, since we\u0027re trying to solve a performance issue we should get some benchmarks before and after the change is applied. Not sure how easy that is with single-node devstack though, you\u0027d have to fake out the computes to get any reasonably large number of them, otherwise we\u0027re left comparing the relative time difference in a query to a single compute node resource provider which is not sexy, not sexy at all.","commit_id":"6a3e023d55fcbc5657f83e2611e3f3729e2e7ea9"}]}