)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"d16ce181bceae350c8b6e1629c4445bf300f78f6","unresolved":false,"context_lines":[{"line_number":10,"context_line":"so maybe we can discuss. Some ideas:"},{"line_number":11,"context_line":""},{"line_number":12,"context_line":"- Just a boolean, where we batch at a fixed size if enabled"},{"line_number":13,"context_line":"- A batch size, which we use if set, else no batching"},{"line_number":14,"context_line":"- A batch size, determined as a fraction of the total limit plus"},{"line_number":15,"context_line":"  some percentage, calculated by the count of cells. So, 10 cells"},{"line_number":16,"context_line":"  and limit of 1000 means we batch at (1000/10)+10% or something."}],"source_content_type":"text/x-gerrit-commit-message","patch_set":2,"id":"3f79a3b5_85f5f0ef","line":13,"updated":"2018-08-20 14:36:29.000000000","message":"Very similar to the first option above, but IntOpt rather than BoolOpt - I\u0027d go with this over the BoolOpt approach because then at least operators who configure [api]/max_limit to some other value can control the batch size as well (if they wanted to).","commit_id":"923dc600e57eee8f598c58230e78a8ba953a2d24"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"d16ce181bceae350c8b6e1629c4445bf300f78f6","unresolved":false,"context_lines":[{"line_number":11,"context_line":""},{"line_number":12,"context_line":"- Just a boolean, where we batch at a fixed size if enabled"},{"line_number":13,"context_line":"- A batch size, which we use if set, else no batching"},{"line_number":14,"context_line":"- A batch size, determined as a fraction of the total limit plus"},{"line_number":15,"context_line":"  some percentage, calculated by the count of cells. So, 10 cells"},{"line_number":16,"context_line":"  and limit of 1000 means we batch at (1000/10)+10% or something."},{"line_number":17,"context_line":"  This requires getting a list of cells cached at some point so"},{"line_number":18,"context_line":"  we know how large."},{"line_number":19,"context_line":"- A divisor (plus percentage?) of the total limit to use for the"},{"line_number":20,"context_line":"  batch size, instead of \"number of cells\"."},{"line_number":21,"context_line":""}],"source_content_type":"text/x-gerrit-commit-message","patch_set":2,"id":"3f79a3b5_25697cbd","line":18,"range":{"start_line":14,"start_character":0,"end_line":18,"end_character":20},"updated":"2018-08-20 14:36:29.000000000","message":"FWIW, this is what I envisioned as a default behavior anyway. We could add multiple options, with a default, called [api]/cells_batch_strategy or something.\n\nConsidering your options listed in here, it sounds like we could do something like:\n\n[api]\ncells_batch_strategy \u003d fixed # would have to pull the value from another option\n\ncells_batch_strategy \u003d none # no batching, just use the limit which defaults to [api]/max_limit (1000)\n\ncells_batch_strategy \u003d distributed # what you\u0027ve described in this bullet, (limit/num_of_cells)+10%\n\ncells_batch_strategy \u003d divisor # would have to pull the value from another option\n\nOn the other hand, having written this, having multiple strategies might be over-doing it and confusing, unless we have known cases where one strategy is clearly better than others, e.g. what difference is there in strategy performance when I have 2 cells (default for nova) vs 10 or 70 cells?\n\nAlso, if we backport this, we would need to consider if we default to existing behavior for a backport and then change the default on master only, or backport with the suggested default for performance (fix) reasons.","commit_id":"923dc600e57eee8f598c58230e78a8ba953a2d24"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"17b6847f04492bb505708a066b4850a5df075b92","unresolved":false,"context_lines":[{"line_number":16,"context_line":"up processing up to 10,000 instance records despite only"},{"line_number":17,"context_line":"returning 1000 to the user (because of the limit)."},{"line_number":18,"context_line":""},{"line_number":19,"context_line":"This builds on the previous patch to support batching generically"},{"line_number":20,"context_line":"in the multi_cell_list by providing a couple of strategies by"},{"line_number":21,"context_line":"which the batch size per cell can be determined. These should"},{"line_number":22,"context_line":"provide a lot of gain in the short term, and we can extend them"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":8,"id":"3f79a3b5_157f83e0","line":19,"range":{"start_line":19,"start_character":12,"end_line":19,"end_character":33},"updated":"2018-08-22 21:00:23.000000000","message":"nit: This builds on \u003chash or change id\u003e because while it means something in a gerrit review series, it won\u0027t when looking at git history or github later.","commit_id":"058d8c2efc1cc2c3c46d289a57a1e788e8373113"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"975597ed0025aabbd53c52bc315b236f9a3033ee","unresolved":false,"context_lines":[{"line_number":16,"context_line":"up processing up to 10,000 instance records despite only"},{"line_number":17,"context_line":"returning 1000 to the user (because of the limit)."},{"line_number":18,"context_line":""},{"line_number":19,"context_line":"This builds on the previous patch to support batching generically"},{"line_number":20,"context_line":"in the multi_cell_list by providing a couple of strategies by"},{"line_number":21,"context_line":"which the batch size per cell can be determined. These should"},{"line_number":22,"context_line":"provide a lot of gain in the short term, and we can extend them"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":8,"id":"3f79a3b5_a548c5b2","line":19,"range":{"start_line":19,"start_character":12,"end_line":19,"end_character":33},"in_reply_to":"3f79a3b5_157f83e0","updated":"2018-08-23 17:00:14.000000000","message":"The problem with this is that the hash will likely change before we get these merged (and change ids are long). I\u0027ll reword this to sidestep the problem.","commit_id":"058d8c2efc1cc2c3c46d289a57a1e788e8373113"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"7338decab8bf1ab8cd5d273e3e27d3253701c744","unresolved":false,"context_lines":[{"line_number":16,"context_line":"up processing up to 10,000 instance records despite only"},{"line_number":17,"context_line":"returning 1000 to the user (because of the limit)."},{"line_number":18,"context_line":""},{"line_number":19,"context_line":"This builds on the previous patch to support batching generically"},{"line_number":20,"context_line":"in the multi_cell_list by providing a couple of strategies by"},{"line_number":21,"context_line":"which the batch size per cell can be determined. These should"},{"line_number":22,"context_line":"provide a lot of gain in the short term, and we can extend them"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":10,"id":"3f79a3b5_cc8077c2","line":19,"range":{"start_line":19,"start_character":5,"end_line":19,"end_character":33},"updated":"2018-08-24 14:38:05.000000000","message":"Use a change ID to refer to the thing you mean. You said you\u0027d reword this, which is fine, but a change ID is also easy.","commit_id":"d90f4008fe0c963de9c8e9b06bb4c80c5974ce89"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"ff5cd003546d070d95d59cde0b018cd509f5c5ed","unresolved":false,"context_lines":[{"line_number":16,"context_line":"up processing up to 10,000 instance records despite only"},{"line_number":17,"context_line":"returning 1000 to the user (because of the limit)."},{"line_number":18,"context_line":""},{"line_number":19,"context_line":"This builds on the previous patch to support batching generically"},{"line_number":20,"context_line":"in the multi_cell_list by providing a couple of strategies by"},{"line_number":21,"context_line":"which the batch size per cell can be determined. These should"},{"line_number":22,"context_line":"provide a lot of gain in the short term, and we can extend them"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":10,"id":"3f79a3b5_2758192f","line":19,"range":{"start_line":19,"start_character":5,"end_line":19,"end_character":33},"in_reply_to":"3f79a3b5_2f7ebde3","updated":"2018-08-24 20:40:19.000000000","message":"Thanks. I find it valuable long after this merges if I\u0027m ever debugging something and coming across this in git history I like to have the reference rather than have to hunt for it.","commit_id":"d90f4008fe0c963de9c8e9b06bb4c80c5974ce89"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"401dafb9c79843d2b253fe8f8a276a240c3b9b40","unresolved":false,"context_lines":[{"line_number":16,"context_line":"up processing up to 10,000 instance records despite only"},{"line_number":17,"context_line":"returning 1000 to the user (because of the limit)."},{"line_number":18,"context_line":""},{"line_number":19,"context_line":"This builds on the previous patch to support batching generically"},{"line_number":20,"context_line":"in the multi_cell_list by providing a couple of strategies by"},{"line_number":21,"context_line":"which the batch size per cell can be determined. These should"},{"line_number":22,"context_line":"provide a lot of gain in the short term, and we can extend them"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":10,"id":"3f79a3b5_2f7ebde3","line":19,"range":{"start_line":19,"start_character":5,"end_line":19,"end_character":33},"in_reply_to":"3f79a3b5_cc8077c2","updated":"2018-08-24 15:43:08.000000000","message":"Yep, and I forgot. I think the change id is ugly and not that important, but clearly it is so I will do it anyway.","commit_id":"d90f4008fe0c963de9c8e9b06bb4c80c5974ce89"}],"nova/compute/instance_list.py":[{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"17b6847f04492bb505708a066b4850a5df075b92","unresolved":false,"context_lines":[{"line_number":98,"context_line":"        ctx, filters, limit, marker, columns_to_join\u003dcolumns_to_join)"},{"line_number":99,"context_line":""},{"line_number":100,"context_line":""},{"line_number":101,"context_line":"def get_instance_list_cells_batch_size(limit, cells):"},{"line_number":102,"context_line":"    \"\"\"Calculate the proper batch size for a list request."},{"line_number":103,"context_line":""},{"line_number":104,"context_line":"    This will consider config, request limit, and cells being queried and"}],"source_content_type":"text/x-python","patch_set":8,"id":"3f79a3b5_7556f75e","line":101,"updated":"2018-08-22 21:00:23.000000000","message":"I could see pulling this out at some point so we can use it in GET /os-migrations which iterates all cells and has a limit param starting in 2.59. It is, however, an admin-only API by default and therefore will be used much less than listing instances by normal users (although normal users likely aren\u0027t going to have more than 10 instances to list either).","commit_id":"058d8c2efc1cc2c3c46d289a57a1e788e8373113"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"975597ed0025aabbd53c52bc315b236f9a3033ee","unresolved":false,"context_lines":[{"line_number":98,"context_line":"        ctx, filters, limit, marker, columns_to_join\u003dcolumns_to_join)"},{"line_number":99,"context_line":""},{"line_number":100,"context_line":""},{"line_number":101,"context_line":"def get_instance_list_cells_batch_size(limit, cells):"},{"line_number":102,"context_line":"    \"\"\"Calculate the proper batch size for a list request."},{"line_number":103,"context_line":""},{"line_number":104,"context_line":"    This will consider config, request limit, and cells being queried and"}],"source_content_type":"text/x-python","patch_set":8,"id":"3f79a3b5_e5855dae","line":101,"in_reply_to":"3f79a3b5_7556f75e","updated":"2018-08-23 17:00:14.000000000","message":"Yeah, I figured that we need these to behave differently and thus scoped all this specifically to instance list. Since users can hit the instance list interface, we need things like the lower limit on batch size, and more tuning for the potential weight of an instance over something like a migration.","commit_id":"058d8c2efc1cc2c3c46d289a57a1e788e8373113"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"17b6847f04492bb505708a066b4850a5df075b92","unresolved":false,"context_lines":[{"line_number":109,"context_line":"    :returns: An integer batch size"},{"line_number":110,"context_line":"    \"\"\""},{"line_number":111,"context_line":"    strategy \u003d CONF.api.instance_list_cells_batch_strategy"},{"line_number":112,"context_line":"    fixed_size \u003d CONF.api.instance_list_cells_batch_fixed_size"},{"line_number":113,"context_line":"    limit \u003d limit or CONF.api.max_limit"},{"line_number":114,"context_line":""},{"line_number":115,"context_line":"    if len(cells) \u003c\u003d 1:"}],"source_content_type":"text/x-python","patch_set":8,"id":"3f79a3b5_75db57ec","line":112,"updated":"2018-08-22 21:00:23.000000000","message":"This could just move inside the conditional below.","commit_id":"058d8c2efc1cc2c3c46d289a57a1e788e8373113"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"17b6847f04492bb505708a066b4850a5df075b92","unresolved":false,"context_lines":[{"line_number":113,"context_line":"    limit \u003d limit or CONF.api.max_limit"},{"line_number":114,"context_line":""},{"line_number":115,"context_line":"    if len(cells) \u003c\u003d 1:"},{"line_number":116,"context_line":"        # If we\u0027re limited to one (or no) cell for whatever reason, do"},{"line_number":117,"context_line":"        # not do any batching and just pull the desired limit from the"},{"line_number":118,"context_line":"        # single cell in one shot."},{"line_number":119,"context_line":"        return limit"}],"source_content_type":"text/x-python","patch_set":8,"id":"3f79a3b5_7504b74a","line":116,"range":{"start_line":116,"start_character":35,"end_line":116,"end_character":40},"updated":"2018-08-22 21:00:23.000000000","message":"As in instance_list_per_project_cells\u003dTrue and the requesting project doesn\u0027t have any instance mappings in any cells, right?","commit_id":"058d8c2efc1cc2c3c46d289a57a1e788e8373113"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"17b6847f04492bb505708a066b4850a5df075b92","unresolved":false,"context_lines":[{"line_number":113,"context_line":"    limit \u003d limit or CONF.api.max_limit"},{"line_number":114,"context_line":""},{"line_number":115,"context_line":"    if len(cells) \u003c\u003d 1:"},{"line_number":116,"context_line":"        # If we\u0027re limited to one (or no) cell for whatever reason, do"},{"line_number":117,"context_line":"        # not do any batching and just pull the desired limit from the"},{"line_number":118,"context_line":"        # single cell in one shot."},{"line_number":119,"context_line":"        return limit"}],"source_content_type":"text/x-python","patch_set":8,"id":"3f79a3b5_f5be8763","line":116,"range":{"start_line":116,"start_character":47,"end_line":116,"end_character":66},"updated":"2018-08-22 21:00:23.000000000","message":"Could be due to instance_list_per_project_cells\u003dTrue, or that cells aren\u0027t setup yet?","commit_id":"058d8c2efc1cc2c3c46d289a57a1e788e8373113"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"975597ed0025aabbd53c52bc315b236f9a3033ee","unresolved":false,"context_lines":[{"line_number":113,"context_line":"    limit \u003d limit or CONF.api.max_limit"},{"line_number":114,"context_line":""},{"line_number":115,"context_line":"    if len(cells) \u003c\u003d 1:"},{"line_number":116,"context_line":"        # If we\u0027re limited to one (or no) cell for whatever reason, do"},{"line_number":117,"context_line":"        # not do any batching and just pull the desired limit from the"},{"line_number":118,"context_line":"        # single cell in one shot."},{"line_number":119,"context_line":"        return limit"}],"source_content_type":"text/x-python","patch_set":8,"id":"3f79a3b5_e521ddb5","line":116,"range":{"start_line":116,"start_character":47,"end_line":116,"end_character":66},"in_reply_to":"3f79a3b5_f5be8763","updated":"2018-08-23 17:00:14.000000000","message":"Yeah, I wanted this logic to be solid regardless of future filtering we might do. If we used the instance_list interface internally and only wanted to look at one cell or something, I would want it to have sane behavior. The cases where it makes no sense to batch are constant regardless of how we get there today or in the future.","commit_id":"058d8c2efc1cc2c3c46d289a57a1e788e8373113"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"17b6847f04492bb505708a066b4850a5df075b92","unresolved":false,"context_lines":[{"line_number":122,"context_line":"        # Fixed strategy, always a static batch size"},{"line_number":123,"context_line":"        batch_size \u003d fixed_size"},{"line_number":124,"context_line":"    elif strategy \u003d\u003d \u0027distributed\u0027:"},{"line_number":125,"context_line":"        # Distributed strategy, 10% more than even parititioning"},{"line_number":126,"context_line":"        batch_size \u003d int((limit / len(cells)) * 1.10)"},{"line_number":127,"context_line":""},{"line_number":128,"context_line":"    # We never query a larger batch than the total requested, and never"}],"source_content_type":"text/x-python","patch_set":8,"id":"3f79a3b5_35a87f1e","line":125,"range":{"start_line":125,"start_character":51,"end_line":125,"end_character":64},"updated":"2018-08-22 21:00:23.000000000","message":"partitioning","commit_id":"058d8c2efc1cc2c3c46d289a57a1e788e8373113"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"17b6847f04492bb505708a066b4850a5df075b92","unresolved":false,"context_lines":[{"line_number":126,"context_line":"        batch_size \u003d int((limit / len(cells)) * 1.10)"},{"line_number":127,"context_line":""},{"line_number":128,"context_line":"    # We never query a larger batch than the total requested, and never"},{"line_number":129,"context_line":"    # smaller than the lower limit of 100."},{"line_number":130,"context_line":"    return max(min(batch_size, limit), 100)"},{"line_number":131,"context_line":""},{"line_number":132,"context_line":""}],"source_content_type":"text/x-python","patch_set":8,"id":"3f79a3b5_35563f2c","line":129,"range":{"start_line":129,"start_character":23,"end_line":129,"end_character":42},"updated":"2018-08-22 21:00:23.000000000","message":"How did the lower limit value come about?\n\nAh I see the config option help now:\n\n\"Any strategy will yield a batch size of at least 100 records,\nto avoid a user causing many tiny database queries in their request.\"","commit_id":"058d8c2efc1cc2c3c46d289a57a1e788e8373113"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"975597ed0025aabbd53c52bc315b236f9a3033ee","unresolved":false,"context_lines":[{"line_number":126,"context_line":"        batch_size \u003d int((limit / len(cells)) * 1.10)"},{"line_number":127,"context_line":""},{"line_number":128,"context_line":"    # We never query a larger batch than the total requested, and never"},{"line_number":129,"context_line":"    # smaller than the lower limit of 100."},{"line_number":130,"context_line":"    return max(min(batch_size, limit), 100)"},{"line_number":131,"context_line":""},{"line_number":132,"context_line":""}],"source_content_type":"text/x-python","patch_set":8,"id":"3f79a3b5_a815fc3e","line":129,"range":{"start_line":129,"start_character":23,"end_line":129,"end_character":42},"in_reply_to":"3f79a3b5_35563f2c","updated":"2018-08-23 17:00:14.000000000","message":"Yeah just to prevent a user from being able to cause tons of tiny DB requests. If I am on CERN and I do a list of size 50, I would get a limit\u003d1 (ish) query to each cell. Now, this just means we do larger queries for no reason I guess, but it seems like the right thing to do.\n\nDo you disagree?","commit_id":"058d8c2efc1cc2c3c46d289a57a1e788e8373113"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"7338decab8bf1ab8cd5d273e3e27d3253701c744","unresolved":false,"context_lines":[{"line_number":126,"context_line":"        batch_size \u003d int((limit / len(cells)) * 1.10)"},{"line_number":127,"context_line":""},{"line_number":128,"context_line":"    # We never query a larger batch than the total requested, and never"},{"line_number":129,"context_line":"    # smaller than the lower limit of 100."},{"line_number":130,"context_line":"    return max(min(batch_size, limit), 100)"},{"line_number":131,"context_line":""},{"line_number":132,"context_line":""}],"source_content_type":"text/x-python","patch_set":8,"id":"3f79a3b5_8c0fdf35","line":129,"range":{"start_line":129,"start_character":23,"end_line":129,"end_character":42},"in_reply_to":"3f79a3b5_a815fc3e","updated":"2018-08-24 14:38:05.000000000","message":"No it makes sense when you use the CERN case. For example, with CERN if I have instances in 5 cells and using the default limit, the batch size would be 220. If I\u0027m limiting to 50 per page as in your example, then I\u0027m down to a batch size of 11 which is small. So yeah 100 seems OK. I guess if people want this to be customizable later we could add a config option.","commit_id":"058d8c2efc1cc2c3c46d289a57a1e788e8373113"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"7338decab8bf1ab8cd5d273e3e27d3253701c744","unresolved":false,"context_lines":[{"line_number":109,"context_line":"    :returns: An integer batch size"},{"line_number":110,"context_line":"    \"\"\""},{"line_number":111,"context_line":"    strategy \u003d CONF.api.instance_list_cells_batch_strategy"},{"line_number":112,"context_line":"    fixed_size \u003d CONF.api.instance_list_cells_batch_fixed_size"},{"line_number":113,"context_line":"    limit \u003d limit or CONF.api.max_limit"},{"line_number":114,"context_line":""},{"line_number":115,"context_line":"    if len(cells) \u003c\u003d 1:"}],"source_content_type":"text/x-python","patch_set":10,"id":"3f79a3b5_8c6eff2c","line":112,"updated":"2018-08-24 14:38:05.000000000","message":"From PS8: This could just move inside the conditional below.","commit_id":"d90f4008fe0c963de9c8e9b06bb4c80c5974ce89"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"401dafb9c79843d2b253fe8f8a276a240c3b9b40","unresolved":false,"context_lines":[{"line_number":109,"context_line":"    :returns: An integer batch size"},{"line_number":110,"context_line":"    \"\"\""},{"line_number":111,"context_line":"    strategy \u003d CONF.api.instance_list_cells_batch_strategy"},{"line_number":112,"context_line":"    fixed_size \u003d CONF.api.instance_list_cells_batch_fixed_size"},{"line_number":113,"context_line":"    limit \u003d limit or CONF.api.max_limit"},{"line_number":114,"context_line":""},{"line_number":115,"context_line":"    if len(cells) \u003c\u003d 1:"}],"source_content_type":"text/x-python","patch_set":10,"id":"3f79a3b5_cf25c95d","line":112,"in_reply_to":"3f79a3b5_8c6eff2c","updated":"2018-08-24 15:43:08.000000000","message":"I didn\u0027t move it because I don\u0027t like it down there, but...yes sir.","commit_id":"d90f4008fe0c963de9c8e9b06bb4c80c5974ce89"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"7338decab8bf1ab8cd5d273e3e27d3253701c744","unresolved":false,"context_lines":[{"line_number":123,"context_line":"        batch_size \u003d fixed_size"},{"line_number":124,"context_line":"    elif strategy \u003d\u003d \u0027distributed\u0027:"},{"line_number":125,"context_line":"        # Distributed strategy, 10% more than even partitioning"},{"line_number":126,"context_line":"        batch_size \u003d int((limit / len(cells)) * 1.10)"},{"line_number":127,"context_line":""},{"line_number":128,"context_line":"    # We never query a larger batch than the total requested, and never"},{"line_number":129,"context_line":"    # smaller than the lower limit of 100."}],"source_content_type":"text/x-python","patch_set":10,"id":"3f79a3b5_0cc0afe0","line":126,"range":{"start_line":126,"start_character":8,"end_line":126,"end_character":53},"updated":"2018-08-24 14:38:05.000000000","message":"Per your earlier example:\n\n\u003e If I am on CERN and I do a list of size 50, I would get a limit\u003d1 (ish) query to each cell.\n\nWe\u0027d end up with 0 here if they weren\u0027t restricting the list to cells that the project has instances in, which means we should really use math.ceil on this to round up to 1, but it doesn\u0027t matter because we have the floor of 100 below.","commit_id":"d90f4008fe0c963de9c8e9b06bb4c80c5974ce89"}],"nova/conf/api.py":[{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"17b6847f04492bb505708a066b4850a5df075b92","unresolved":false,"context_lines":[{"line_number":270,"context_line":"        choices\u003d(\"fixed\", \"distributed\"),"},{"line_number":271,"context_line":"        default\u003d\"distributed\","},{"line_number":272,"context_line":"        help\u003d\"\"\""},{"line_number":273,"context_line":"This controls the method by which the API to query cell databases in"},{"line_number":274,"context_line":"smaller batches during large instance list operations. If batching is"},{"line_number":275,"context_line":"performed, a large instance list operation will request some fraction"},{"line_number":276,"context_line":"of the overall API limit from each cell database initially, and will"}],"source_content_type":"text/x-python","patch_set":8,"id":"3f79a3b5_15378301","line":273,"range":{"start_line":273,"start_character":42,"end_line":273,"end_character":50},"updated":"2018-08-22 21:00:23.000000000","message":"queries?","commit_id":"058d8c2efc1cc2c3c46d289a57a1e788e8373113"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"17b6847f04492bb505708a066b4850a5df075b92","unresolved":false,"context_lines":[{"line_number":278,"context_line":"from each cell as necessary. Larger batches mean less chattiness"},{"line_number":279,"context_line":"between the API and the database, but potentially more wasted effort"},{"line_number":280,"context_line":"processing the results from the database which will not be returned to"},{"line_number":281,"context_line":"the user. Any strategy will yield a batch size of at least 100 records,"},{"line_number":282,"context_line":"to avoid a user causing many tiny database queries in their request."},{"line_number":283,"context_line":""},{"line_number":284,"context_line":"\u0027\u0027distributed\u0027\u0027 (the default) will attempt to divide the limit"}],"source_content_type":"text/x-python","patch_set":8,"id":"3f79a3b5_f51a077b","line":281,"range":{"start_line":281,"start_character":59,"end_line":281,"end_character":62},"updated":"2018-08-22 21:00:23.000000000","message":"Arguably we could limit this by default to instances quota which defaults to 10, but any tenant with more than that would suffer the \u0027many tiny db queries\u0027 fate.","commit_id":"058d8c2efc1cc2c3c46d289a57a1e788e8373113"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"17b6847f04492bb505708a066b4850a5df075b92","unresolved":false,"context_lines":[{"line_number":281,"context_line":"the user. Any strategy will yield a batch size of at least 100 records,"},{"line_number":282,"context_line":"to avoid a user causing many tiny database queries in their request."},{"line_number":283,"context_line":""},{"line_number":284,"context_line":"\u0027\u0027distributed\u0027\u0027 (the default) will attempt to divide the limit"},{"line_number":285,"context_line":"requested by the user by the number of cells in the system. This"},{"line_number":286,"context_line":"requires counting the cells in the system initially, which will not be"},{"line_number":287,"context_line":"refreshed until service restart or SIGHUP. The actual batch size will"}],"source_content_type":"text/x-python","patch_set":8,"id":"3f79a3b5_7013655d","line":284,"range":{"start_line":284,"start_character":0,"end_line":284,"end_character":15},"updated":"2018-08-22 21:00:23.000000000","message":"This renders a little weird:\n\nhttp://logs.openstack.org/31/593131/8/check/build-openstack-sphinx-docs/9702d21/html/configuration/config.html#api.instance_list_cells_batch_strategy\n\nUse double backtick or something?\n\n``distributed`` - that\u0027s more normal for our rst docs.","commit_id":"058d8c2efc1cc2c3c46d289a57a1e788e8373113"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"17b6847f04492bb505708a066b4850a5df075b92","unresolved":false,"context_lines":[{"line_number":291,"context_line":"defined by ``instance_list_cells_batch_fixed_size``. If the limit is"},{"line_number":292,"context_line":"smaller than the batch size, the limit will be used instead. If you do"},{"line_number":293,"context_line":"not wish batching to be used at all, setting the fixed size equal to"},{"line_number":294,"context_line":"max_limit will cause only one request per cell database to be issued."},{"line_number":295,"context_line":""},{"line_number":296,"context_line":"Possible values:"},{"line_number":297,"context_line":""}],"source_content_type":"text/x-python","patch_set":8,"id":"3f79a3b5_703cc5c7","line":294,"range":{"start_line":294,"start_character":0,"end_line":294,"end_character":9},"updated":"2018-08-22 21:00:23.000000000","message":"equal to the ``max_limit`` value","commit_id":"058d8c2efc1cc2c3c46d289a57a1e788e8373113"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"17b6847f04492bb505708a066b4850a5df075b92","unresolved":false,"context_lines":[{"line_number":314,"context_line":""},{"line_number":315,"context_line":"Related options:"},{"line_number":316,"context_line":""},{"line_number":317,"context_line":"* instance_list_cells_batch_strategy"},{"line_number":318,"context_line":"\"\"\"),"},{"line_number":319,"context_line":"]"},{"line_number":320,"context_line":""}],"source_content_type":"text/x-python","patch_set":8,"id":"3f79a3b5_d0583932","line":317,"updated":"2018-08-22 21:00:23.000000000","message":"Add max_limit too given the above:\n\n\"If you do\nnot wish batching to be used at all, setting the fixed size equal to\nmax_limit will cause only one request per cell database to be issued.\"","commit_id":"058d8c2efc1cc2c3c46d289a57a1e788e8373113"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"7338decab8bf1ab8cd5d273e3e27d3253701c744","unresolved":false,"context_lines":[{"line_number":304,"context_line":"* instance_list_cells_batch_fixed_size"},{"line_number":305,"context_line":"* max_limit"},{"line_number":306,"context_line":"\"\"\"),"},{"line_number":307,"context_line":"    cfg.IntOpt(\"instance_list_cells_batch_fixed_size\","},{"line_number":308,"context_line":"        default\u003d100,"},{"line_number":309,"context_line":"        help\u003d\"\"\""},{"line_number":310,"context_line":"This controls the batch size of instances requested from each cell"}],"source_content_type":"text/x-python","patch_set":10,"id":"3f79a3b5_ef70c5a6","line":307,"updated":"2018-08-24 14:38:05.000000000","message":"So, given the floor is currently 100, we should probably put min\u003d100 on this option so it\u0027s reinforced. That, or instance_list_cells_batch_fixed_size should override that hard-coded 100 floor it seems. Like, if I want my batches to always be 50 for whatever reason I might be surprised to see that ignored in the code since I\u0027m telling nova exactly what I want the batch sizes to be.","commit_id":"d90f4008fe0c963de9c8e9b06bb4c80c5974ce89"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"ff5cd003546d070d95d59cde0b018cd509f5c5ed","unresolved":false,"context_lines":[{"line_number":304,"context_line":"* instance_list_cells_batch_fixed_size"},{"line_number":305,"context_line":"* max_limit"},{"line_number":306,"context_line":"\"\"\"),"},{"line_number":307,"context_line":"    cfg.IntOpt(\"instance_list_cells_batch_fixed_size\","},{"line_number":308,"context_line":"        default\u003d100,"},{"line_number":309,"context_line":"        help\u003d\"\"\""},{"line_number":310,"context_line":"This controls the batch size of instances requested from each cell"}],"source_content_type":"text/x-python","patch_set":10,"id":"3f79a3b5_677b718b","line":307,"in_reply_to":"3f79a3b5_0f8c2134","updated":"2018-08-24 20:40:19.000000000","message":"Agree, was just stating let\u0027s either (a) be explicit or (b) allow the override. I\u0027m good with (a).","commit_id":"d90f4008fe0c963de9c8e9b06bb4c80c5974ce89"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"401dafb9c79843d2b253fe8f8a276a240c3b9b40","unresolved":false,"context_lines":[{"line_number":304,"context_line":"* instance_list_cells_batch_fixed_size"},{"line_number":305,"context_line":"* max_limit"},{"line_number":306,"context_line":"\"\"\"),"},{"line_number":307,"context_line":"    cfg.IntOpt(\"instance_list_cells_batch_fixed_size\","},{"line_number":308,"context_line":"        default\u003d100,"},{"line_number":309,"context_line":"        help\u003d\"\"\""},{"line_number":310,"context_line":"This controls the batch size of instances requested from each cell"}],"source_content_type":"text/x-python","patch_set":10,"id":"3f79a3b5_0f8c2134","line":307,"in_reply_to":"3f79a3b5_ef70c5a6","updated":"2018-08-24 15:43:08.000000000","message":"Given the numbers from Huawei, I don\u0027t think 100 is too large for anyone, and yeah I agree putting the min here should help make sure nobody is surprised.","commit_id":"d90f4008fe0c963de9c8e9b06bb4c80c5974ce89"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"b24c6c5d3938eafd3072419b141592009da824e3","unresolved":false,"context_lines":[{"line_number":312,"context_line":"database if ``instance_list_cells_batch_strategy``` is set to ``fixed``."},{"line_number":313,"context_line":"This integral value will define the limit issued to each cell every time"},{"line_number":314,"context_line":"a batch of instances is requested, regardless of the number of cells in"},{"line_number":315,"context_line":"the system or any other factors. Per the general logic called out in"},{"line_number":316,"context_line":"in the documentation for ``instance_list_cells_batch_strategy``, the"},{"line_number":317,"context_line":"minimum value for this is 100 records per batch."},{"line_number":318,"context_line":""},{"line_number":319,"context_line":"Related options:"}],"source_content_type":"text/x-python","patch_set":13,"id":"3f79a3b5_c7c7e595","line":316,"range":{"start_line":315,"start_character":66,"end_line":316,"end_character":2},"updated":"2018-08-24 20:46:21.000000000","message":"in","commit_id":"36e85e70fbce91c756d37cd9c5ed2320dc66d6f8"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"c76d044f5913f12e5ba87d28cfbf75c632a3b2e3","unresolved":false,"context_lines":[{"line_number":312,"context_line":"database if ``instance_list_cells_batch_strategy``` is set to ``fixed``."},{"line_number":313,"context_line":"This integral value will define the limit issued to each cell every time"},{"line_number":314,"context_line":"a batch of instances is requested, regardless of the number of cells in"},{"line_number":315,"context_line":"the system or any other factors. Per the general logic called out in"},{"line_number":316,"context_line":"in the documentation for ``instance_list_cells_batch_strategy``, the"},{"line_number":317,"context_line":"minimum value for this is 100 records per batch."},{"line_number":318,"context_line":""},{"line_number":319,"context_line":"Related options:"}],"source_content_type":"text/x-python","patch_set":13,"id":"3f79a3b5_44fbba77","line":316,"range":{"start_line":315,"start_character":66,"end_line":316,"end_character":2},"in_reply_to":"3f79a3b5_c7c7e595","updated":"2018-08-27 13:44:58.000000000","message":"Done","commit_id":"36e85e70fbce91c756d37cd9c5ed2320dc66d6f8"}],"nova/tests/unit/compute/test_instance_list.py":[{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"2fbd3b7d833653523f7a4cd944913e858dc80064","unresolved":false,"context_lines":[{"line_number":189,"context_line":"        self.flags(instance_list_cells_batch_fixed_size\u003d200, group\u003d\u0027api\u0027)"},{"line_number":190,"context_line":""},{"line_number":191,"context_line":"        # One cell, so batch at $limit"},{"line_number":192,"context_line":"        ret \u003d instance_list.get_instance_list_cells_batch_size(1000, [1])"},{"line_number":193,"context_line":"        self.assertEqual(1000, ret)"},{"line_number":194,"context_line":""},{"line_number":195,"context_line":"        # Two cells, so batch at 100"}],"source_content_type":"text/x-python","patch_set":10,"id":"3f79a3b5_6fb11544","line":192,"range":{"start_line":192,"start_character":70,"end_line":192,"end_character":71},"updated":"2018-08-24 14:46:35.000000000","message":"I guess these are just sentinels, which is OK. mock.sentinel.cell1 could be more clear.","commit_id":"d90f4008fe0c963de9c8e9b06bb4c80c5974ce89"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"ff5cd003546d070d95d59cde0b018cd509f5c5ed","unresolved":false,"context_lines":[{"line_number":189,"context_line":"        self.flags(instance_list_cells_batch_fixed_size\u003d200, group\u003d\u0027api\u0027)"},{"line_number":190,"context_line":""},{"line_number":191,"context_line":"        # One cell, so batch at $limit"},{"line_number":192,"context_line":"        ret \u003d instance_list.get_instance_list_cells_batch_size(1000, [1])"},{"line_number":193,"context_line":"        self.assertEqual(1000, ret)"},{"line_number":194,"context_line":""},{"line_number":195,"context_line":"        # Two cells, so batch at 100"}],"source_content_type":"text/x-python","patch_set":10,"id":"3f79a3b5_e7ab01d3","line":192,"range":{"start_line":192,"start_character":70,"end_line":192,"end_character":71},"in_reply_to":"3f79a3b5_3204e014","updated":"2018-08-24 20:40:19.000000000","message":"Yeah I realize the method is just checking list length. I wasn\u0027t a big deal, just something I had to think about.","commit_id":"d90f4008fe0c963de9c8e9b06bb4c80c5974ce89"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"401dafb9c79843d2b253fe8f8a276a240c3b9b40","unresolved":false,"context_lines":[{"line_number":189,"context_line":"        self.flags(instance_list_cells_batch_fixed_size\u003d200, group\u003d\u0027api\u0027)"},{"line_number":190,"context_line":""},{"line_number":191,"context_line":"        # One cell, so batch at $limit"},{"line_number":192,"context_line":"        ret \u003d instance_list.get_instance_list_cells_batch_size(1000, [1])"},{"line_number":193,"context_line":"        self.assertEqual(1000, ret)"},{"line_number":194,"context_line":""},{"line_number":195,"context_line":"        # Two cells, so batch at 100"}],"source_content_type":"text/x-python","patch_set":10,"id":"3f79a3b5_3204e014","line":192,"range":{"start_line":192,"start_character":70,"end_line":192,"end_character":71},"in_reply_to":"3f79a3b5_6fb11544","updated":"2018-08-24 15:43:08.000000000","message":"They\u0027re not really sentinels because they\u0027re not indicating anything to other code. I literally just need a list with a certain number of items in it. Using mock.sentinel makes this less readable to me, but what do I know, I\u0027ll change it.","commit_id":"d90f4008fe0c963de9c8e9b06bb4c80c5974ce89"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"2fbd3b7d833653523f7a4cd944913e858dc80064","unresolved":false,"context_lines":[{"line_number":192,"context_line":"        ret \u003d instance_list.get_instance_list_cells_batch_size(1000, [1])"},{"line_number":193,"context_line":"        self.assertEqual(1000, ret)"},{"line_number":194,"context_line":""},{"line_number":195,"context_line":"        # Two cells, so batch at 100"},{"line_number":196,"context_line":"        ret \u003d instance_list.get_instance_list_cells_batch_size(1000, [1, 2])"},{"line_number":197,"context_line":"        self.assertEqual(200, ret)"},{"line_number":198,"context_line":""}],"source_content_type":"text/x-python","patch_set":10,"id":"3f79a3b5_4f5219e0","line":195,"range":{"start_line":195,"start_character":33,"end_line":195,"end_character":36},"updated":"2018-08-24 14:46:35.000000000","message":"wrong","commit_id":"d90f4008fe0c963de9c8e9b06bb4c80c5974ce89"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"2fbd3b7d833653523f7a4cd944913e858dc80064","unresolved":false,"context_lines":[{"line_number":196,"context_line":"        ret \u003d instance_list.get_instance_list_cells_batch_size(1000, [1, 2])"},{"line_number":197,"context_line":"        self.assertEqual(200, ret)"},{"line_number":198,"context_line":""},{"line_number":199,"context_line":"        # Four cells, so batch at 100"},{"line_number":200,"context_line":"        ret \u003d instance_list.get_instance_list_cells_batch_size(1000, [1, 2,"},{"line_number":201,"context_line":"                                                                      3, 4])"},{"line_number":202,"context_line":"        self.assertEqual(200, ret)"}],"source_content_type":"text/x-python","patch_set":10,"id":"3f79a3b5_cf3e298a","line":199,"range":{"start_line":199,"start_character":34,"end_line":199,"end_character":37},"updated":"2018-08-24 14:46:35.000000000","message":"This would be true for distributed but that\u0027s not what this is testing.","commit_id":"d90f4008fe0c963de9c8e9b06bb4c80c5974ce89"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"2fbd3b7d833653523f7a4cd944913e858dc80064","unresolved":false,"context_lines":[{"line_number":205,"context_line":"        ret \u003d instance_list.get_instance_list_cells_batch_size(10, [1, 2, 3])"},{"line_number":206,"context_line":"        self.assertEqual(100, ret)"},{"line_number":207,"context_line":""},{"line_number":208,"context_line":"        # Three cells, small limit, so batch at limit"},{"line_number":209,"context_line":"        ret \u003d instance_list.get_instance_list_cells_batch_size(110, [1, 2, 3])"},{"line_number":210,"context_line":"        self.assertEqual(110, ret)"},{"line_number":211,"context_line":""}],"source_content_type":"text/x-python","patch_set":10,"id":"3f79a3b5_cf5709ca","line":208,"range":{"start_line":208,"start_character":23,"end_line":208,"end_character":34},"updated":"2018-08-24 14:46:35.000000000","message":"limit above floor?","commit_id":"d90f4008fe0c963de9c8e9b06bb4c80c5974ce89"},{"author":{"_account_id":4690,"name":"melanie witt","display_name":"melwitt","email":"melwittt@gmail.com","username":"melwitt"},"change_message_id":"d91cc5f630c381244b10171f0cc4c2d4b8227b8c","unresolved":false,"context_lines":[{"line_number":97,"context_line":"        mock_gi.assert_called_once_with(user_context, {}, None, None, [],"},{"line_number":98,"context_line":"                                        None, None,"},{"line_number":99,"context_line":"                                        cell_mappings\u003dmock_cm.return_value,"},{"line_number":100,"context_line":"                                        batch_size\u003d1000)"},{"line_number":101,"context_line":""},{"line_number":102,"context_line":"    @mock.patch(\u0027nova.context.CELLS\u0027, new\u003dFAKE_CELLS)"},{"line_number":103,"context_line":"    @mock.patch(\u0027nova.context.load_cells\u0027)"}],"source_content_type":"text/x-python","patch_set":14,"id":"3f79a3b5_65dd863f","line":100,"range":{"start_line":100,"start_character":51,"end_line":100,"end_character":55},"updated":"2018-08-27 23:49:00.000000000","message":"Note to self: this is 1000 because of instance_list_per_project_cells\u003dTrue, we\u0027re not doing any batching here. I was thinking this would be len(cells) \u003d\u003d 1, but can\u0027t see how above.\n\n(later)\n\nThis is a bit weird because what\u0027s being passed to as \u0027cell_mappings\u0027 is a MagicMock, so the len(cells) call in get_instance_list_cells_batch_size will return 0 because len(MagicMock()) \u003d\u003d 0, apparently.","commit_id":"c3a77f80b1863e114109af9c32ea01b205c1a735"}]}