)]}'
{"nova/compute/multi_cell_list.py":[{"author":{"_account_id":20722,"name":"Yikun Jiang","email":"yikunkero@gmail.com","username":"yikunkero"},"change_message_id":"3aaa8cb1e2b178dca3160e7d5b9514a173cbf4a2","unresolved":false,"context_lines":[{"line_number":270,"context_line":"                print(\u0027DAN: Doing batch of %s/%s\u0027 % (query_size, limit))"},{"line_number":271,"context_line":"                query_result \u003d self.get_by_filters("},{"line_number":272,"context_line":"                    ctx, filters,"},{"line_number":273,"context_line":"                    limit\u003dquery_size or None, marker\u003dlocal_marker,"},{"line_number":274,"context_line":"                    **kwargs)"},{"line_number":275,"context_line":""},{"line_number":276,"context_line":"                # Yield wrapped results from the batch, counting as we go"}],"source_content_type":"text/x-python","patch_set":2,"id":"3f79a3b5_0d761440","line":273,"range":{"start_line":273,"start_character":53,"end_line":273,"end_character":65},"updated":"2018-08-17 09:23:03.000000000","message":"So, this marker have 2 means:\n1. it\u0027s the original local_marker, that we get this using global marker.\n2. it\u0027s the batch_local_marker to help us find the follow records.","commit_id":"386ff8bd450c70fa6bb2e12219fbe0780ebe4f04"},{"author":{"_account_id":20722,"name":"Yikun Jiang","email":"yikunkero@gmail.com","username":"yikunkero"},"change_message_id":"3aaa8cb1e2b178dca3160e7d5b9514a173cbf4a2","unresolved":false,"context_lines":[{"line_number":278,"context_line":"                # local_marker each time so that local_marker is the end of"},{"line_number":279,"context_line":"                # this batch in order to find the next batch."},{"line_number":280,"context_line":"                for item in query_result:"},{"line_number":281,"context_line":"                    local_marker \u003d item[self.marker_identifier]"},{"line_number":282,"context_line":"                    yield RecordWrapper(self.sort_ctx, item)"},{"line_number":283,"context_line":"                    batch_count +\u003d 1"},{"line_number":284,"context_line":""}],"source_content_type":"text/x-python","patch_set":2,"id":"3f79a3b5_2d50b0d4","line":281,"updated":"2018-08-17 09:23:03.000000000","message":"A question about this local_marker, I thought that this ``local_marker`` is  the last one in all cells rather than in each cell, but it seems I\u0027m wrong?\n\nSo, I\u0027m not clear about how to stop the iteration? In my mind, only 2 conditions:\n1. we already got more than LIMIT records(N), and next time we get records from all cells is sorted after the tail of these records(N) we can stop the iteration.\n\nor to say 2. we got the last record of LIMIT records(N) as a global_marker, and if we cannot get any record using this global marker, we can stop the iteration.\n\nHowever, this local_marker seems the last record of each cell per thread, but it\u0027s not the global one. So, I don\u0027t understand your idea completely.","commit_id":"386ff8bd450c70fa6bb2e12219fbe0780ebe4f04"},{"author":{"_account_id":20722,"name":"Yikun Jiang","email":"yikunkero@gmail.com","username":"yikunkero"},"change_message_id":"3aaa8cb1e2b178dca3160e7d5b9514a173cbf4a2","unresolved":false,"context_lines":[{"line_number":279,"context_line":"                # this batch in order to find the next batch."},{"line_number":280,"context_line":"                for item in query_result:"},{"line_number":281,"context_line":"                    local_marker \u003d item[self.marker_identifier]"},{"line_number":282,"context_line":"                    yield RecordWrapper(self.sort_ctx, item)"},{"line_number":283,"context_line":"                    batch_count +\u003d 1"},{"line_number":284,"context_line":""},{"line_number":285,"context_line":"                print(\u0027DAN: returned %i\u0027 % batch_count)"}],"source_content_type":"text/x-python","patch_set":2,"id":"3f79a3b5_0017bcef","line":282,"updated":"2018-08-17 09:23:03.000000000","message":"Cool, good yield, and it\u0027s benefit for reducing the traversing and improving performance.","commit_id":"386ff8bd450c70fa6bb2e12219fbe0780ebe4f04"},{"author":{"_account_id":4690,"name":"melanie witt","display_name":"melwitt","email":"melwittt@gmail.com","username":"melwitt"},"change_message_id":"6f01b3264aa155ec675e5d4fd688ab3788812fc6","unresolved":false,"context_lines":[{"line_number":242,"context_line":"            if local_marker_prefix:"},{"line_number":243,"context_line":"                # Per above, if we had a matching marker object, that is"},{"line_number":244,"context_line":"                # the first result we should generate."},{"line_number":245,"context_line":"                yield RecordWrapper(self.sort_ctx, local_marker_prefix[0])"},{"line_number":246,"context_line":""},{"line_number":247,"context_line":"            # If a batch size was provided, use that as the limit per"},{"line_number":248,"context_line":"            # batch. If not, then ask for the entire $limit in a single"}],"source_content_type":"text/x-python","patch_set":4,"id":"3f79a3b5_e8335012","line":245,"updated":"2018-08-20 23:02:21.000000000","message":"Note to self: local_marker_prefix is a list of one instance, which is the marker we need to return as the first result, so it\u0027s being yielded here first.","commit_id":"8ba2483ad16013801155184a8a29f00273b218ed"},{"author":{"_account_id":4690,"name":"melanie witt","display_name":"melwitt","email":"melwittt@gmail.com","username":"melwitt"},"change_message_id":"9705966c757353731d477c63902dede2cb50e7ae","unresolved":false,"context_lines":[{"line_number":242,"context_line":"            if local_marker_prefix:"},{"line_number":243,"context_line":"                # Per above, if we had a matching marker object, that is"},{"line_number":244,"context_line":"                # the first result we should generate."},{"line_number":245,"context_line":"                yield RecordWrapper(self.sort_ctx, local_marker_prefix[0])"},{"line_number":246,"context_line":""},{"line_number":247,"context_line":"            # If a batch size was provided, use that as the limit per"},{"line_number":248,"context_line":"            # batch. If not, then ask for the entire $limit in a single"}],"source_content_type":"text/x-python","patch_set":4,"id":"3f79a3b5_0ec0157c","line":245,"in_reply_to":"3f79a3b5_ce2f7d86","updated":"2018-08-20 23:25:11.000000000","message":"Ack. Just wanted to make a comment note to help me (and potentially other reviewers) understand this bit.","commit_id":"8ba2483ad16013801155184a8a29f00273b218ed"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"f360100b2822e51d7e20529f6de1b91d8b7ea95f","unresolved":false,"context_lines":[{"line_number":242,"context_line":"            if local_marker_prefix:"},{"line_number":243,"context_line":"                # Per above, if we had a matching marker object, that is"},{"line_number":244,"context_line":"                # the first result we should generate."},{"line_number":245,"context_line":"                yield RecordWrapper(self.sort_ctx, local_marker_prefix[0])"},{"line_number":246,"context_line":""},{"line_number":247,"context_line":"            # If a batch size was provided, use that as the limit per"},{"line_number":248,"context_line":"            # batch. If not, then ask for the entire $limit in a single"}],"source_content_type":"text/x-python","patch_set":4,"id":"3f79a3b5_ce2f7d86","line":245,"in_reply_to":"3f79a3b5_e8335012","updated":"2018-08-20 23:08:59.000000000","message":"yep, which is why it\u0027s local_marker_prefix and not local_marker_actually. It\u0027s a list that needs to be prefixed to the list we\u0027re going to return, and was used that way when we just needed to itertools.chain() this with the result. We could snip out the [0] above but I don\u0027t think it will really be easier, and we might later want to yield this with the first batch from the db for latency reasons.","commit_id":"8ba2483ad16013801155184a8a29f00273b218ed"},{"author":{"_account_id":4690,"name":"melanie witt","display_name":"melwitt","email":"melwittt@gmail.com","username":"melwitt"},"change_message_id":"6f01b3264aa155ec675e5d4fd688ab3788812fc6","unresolved":false,"context_lines":[{"line_number":255,"context_line":"            # If limit was unlimited then keep querying batches until"},{"line_number":256,"context_line":"            # we run out of results. Otherwise, query until the total count"},{"line_number":257,"context_line":"            # we have returned exceeds the limit."},{"line_number":258,"context_line":"            while limit is None or return_count \u003c limit:"},{"line_number":259,"context_line":"                batch_count \u003d 0"},{"line_number":260,"context_line":""},{"line_number":261,"context_line":"                # Do not query a full batch if it would cause our total"}],"source_content_type":"text/x-python","patch_set":4,"id":"3f79a3b5_2829489a","line":258,"range":{"start_line":258,"start_character":18,"end_line":258,"end_character":31},"updated":"2018-08-20 23:02:21.000000000","message":"On L249, the comment says \"[limit]...which may be zero meaning unlimited\" which sounds like it\u0027s saying limit \u003d 0 means unlimited. So, don\u0027t we need to handle limit \u003d 0 here too as unlimited?","commit_id":"8ba2483ad16013801155184a8a29f00273b218ed"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"f360100b2822e51d7e20529f6de1b91d8b7ea95f","unresolved":false,"context_lines":[{"line_number":255,"context_line":"            # If limit was unlimited then keep querying batches until"},{"line_number":256,"context_line":"            # we run out of results. Otherwise, query until the total count"},{"line_number":257,"context_line":"            # we have returned exceeds the limit."},{"line_number":258,"context_line":"            while limit is None or return_count \u003c limit:"},{"line_number":259,"context_line":"                batch_count \u003d 0"},{"line_number":260,"context_line":""},{"line_number":261,"context_line":"                # Do not query a full batch if it would cause our total"}],"source_content_type":"text/x-python","patch_set":4,"id":"3f79a3b5_2e563108","line":258,"range":{"start_line":258,"start_character":18,"end_line":258,"end_character":31},"in_reply_to":"3f79a3b5_2829489a","updated":"2018-08-20 23:08:59.000000000","message":"No, this comment is leftover from before I realized that there\u0027s a bit of a reentrancy bug with using limit here, because the \"or zero meaning unlimited\" came from that. I fixed that below (L319) and need to snip this part of the comment out.","commit_id":"8ba2483ad16013801155184a8a29f00273b218ed"},{"author":{"_account_id":4690,"name":"melanie witt","display_name":"melwitt","email":"melwittt@gmail.com","username":"melwitt"},"change_message_id":"9705966c757353731d477c63902dede2cb50e7ae","unresolved":false,"context_lines":[{"line_number":255,"context_line":"            # If limit was unlimited then keep querying batches until"},{"line_number":256,"context_line":"            # we run out of results. Otherwise, query until the total count"},{"line_number":257,"context_line":"            # we have returned exceeds the limit."},{"line_number":258,"context_line":"            while limit is None or return_count \u003c limit:"},{"line_number":259,"context_line":"                batch_count \u003d 0"},{"line_number":260,"context_line":""},{"line_number":261,"context_line":"                # Do not query a full batch if it would cause our total"}],"source_content_type":"text/x-python","patch_set":4,"id":"3f79a3b5_4e804d47","line":258,"range":{"start_line":258,"start_character":18,"end_line":258,"end_character":31},"in_reply_to":"3f79a3b5_2e563108","updated":"2018-08-20 23:25:11.000000000","message":"Ah, I see -- in a previous version total_limit was just limit. Got it.","commit_id":"8ba2483ad16013801155184a8a29f00273b218ed"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"e38638de8012f06eaab933decf898f32efaf79f7","unresolved":false,"context_lines":[{"line_number":72,"context_line":"        if self._db_record in CELL_SENTINELS:"},{"line_number":73,"context_line":"            return -1"},{"line_number":74,"context_line":"        elif other._db_record in CELL_SENTINELS:"},{"line_number":75,"context_line":"            return 1"},{"line_number":76,"context_line":""},{"line_number":77,"context_line":"        r \u003d self._sort_ctx.compare_records(self._db_record,"},{"line_number":78,"context_line":"                                           other._db_record)"}],"source_content_type":"text/x-python","patch_set":8,"id":"3f79a3b5_78da7978","line":75,"updated":"2018-08-22 14:50:00.000000000","message":"I\u0027m adding a comment to this:\n\n        # NOTE(danms): This makes us always sort failure sentinels\n        # higher than actual results. We do this so that they bubble\n        # up in the get_records_sorted() feeder loop ahead of anything\n        # else, and so that the implementation of RecordSortContext\n        # never sees or has to handle the sentinels. If we did not\n        # sort these to the top then we could potentially return\n        # $limit results from good cells before we noticed the failed\n        # cells, and would not properly report them as failed for\n        # fix-up in the higher layers.","commit_id":"ce2559932913ccf834b086a066da60bfa6e3e352"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"7b860284061de34784717ed72980c0be5b128f97","unresolved":false,"context_lines":[{"line_number":218,"context_line":"            global_marker_values \u003d [global_marker_record[key]"},{"line_number":219,"context_line":"                                    for key in self.sort_ctx.sort_keys]"},{"line_number":220,"context_line":""},{"line_number":221,"context_line":"        def do_query(cctx):"},{"line_number":222,"context_line":"            \"\"\"Generate RecordWrapper(record) objects from a cell."},{"line_number":223,"context_line":""},{"line_number":224,"context_line":"            We do this inside the thread (created by"}],"source_content_type":"text/x-python","patch_set":8,"id":"3f79a3b5_d8794dfc","line":221,"range":{"start_line":221,"start_character":21,"end_line":221,"end_character":25},"updated":"2018-08-22 14:57:21.000000000","message":"I\u0027m moving this refactor out to its own patch to reduce the noise here.","commit_id":"ce2559932913ccf834b086a066da60bfa6e3e352"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"a06df57e0c918c86165da4d688d07051162b593a","unresolved":false,"context_lines":[{"line_number":23,"context_line":"from nova import exception"},{"line_number":24,"context_line":""},{"line_number":25,"context_line":"LOG \u003d logging.getLogger(__name__)"},{"line_number":26,"context_line":"CELL_SENTINELS \u003d (context.did_not_respond_sentinel,"},{"line_number":27,"context_line":"                  context.raised_exception_sentinel)"},{"line_number":28,"context_line":""},{"line_number":29,"context_line":""}],"source_content_type":"text/x-python","patch_set":9,"id":"3f79a3b5_5a420aad","line":26,"range":{"start_line":26,"start_character":0,"end_line":26,"end_character":14},"updated":"2018-08-22 20:18:47.000000000","message":"nit: maybe name this CELL_FAIL_SENTINELS?","commit_id":"ffc6efd22bc746e7e964c38247058c5843a39524"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"a06df57e0c918c86165da4d688d07051162b593a","unresolved":false,"context_lines":[{"line_number":69,"context_line":"        return self._ctx.cell_uuid"},{"line_number":70,"context_line":""},{"line_number":71,"context_line":"    def __lt__(self, other):"},{"line_number":72,"context_line":"        # NOTE(danms): This makes us always sort failure sentinels"},{"line_number":73,"context_line":"        # higher than actual results. We do this so that they bubble"},{"line_number":74,"context_line":"        # up in the get_records_sorted() feeder loop ahead of anything"},{"line_number":75,"context_line":"        # else, and so that the implementation of RecordSortContext"}],"source_content_type":"text/x-python","patch_set":9,"id":"3f79a3b5_5a28ca95","line":72,"updated":"2018-08-22 20:18:47.000000000","message":"So the idea here is before batching, we\u0027d hit the sentinel failures per-cell, but with batching we want to hit the sentinel failures per-batch within a cell, yes? And then stop immediately for that cell and bubble it up for the final resulting merge sort.","commit_id":"ffc6efd22bc746e7e964c38247058c5843a39524"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"ba83b43d0ee6735e8f5f6f433bbbf24a7deee28b","unresolved":false,"context_lines":[{"line_number":69,"context_line":"        return self._ctx.cell_uuid"},{"line_number":70,"context_line":""},{"line_number":71,"context_line":"    def __lt__(self, other):"},{"line_number":72,"context_line":"        # NOTE(danms): This makes us always sort failure sentinels"},{"line_number":73,"context_line":"        # higher than actual results. We do this so that they bubble"},{"line_number":74,"context_line":"        # up in the get_records_sorted() feeder loop ahead of anything"},{"line_number":75,"context_line":"        # else, and so that the implementation of RecordSortContext"}],"source_content_type":"text/x-python","patch_set":9,"id":"3f79a3b5_35f07f91","line":72,"in_reply_to":"3f79a3b5_5a28ca95","updated":"2018-08-22 20:47:12.000000000","message":"Prior to this patch, we waited for all cells to either return or fail before we started to process/sort the results. We noted which cells failed at that point, removed their result from the list we passed through to heapq. Now, we don\u0027t wait, and get the sentinels in-band with the results themselves. We need to sort those before anything else as soon as they come up so we see them.","commit_id":"ffc6efd22bc746e7e964c38247058c5843a39524"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"a06df57e0c918c86165da4d688d07051162b593a","unresolved":false,"context_lines":[{"line_number":99,"context_line":"    be handled by the main get_objects_sorted() feeder loop quickly and"},{"line_number":100,"context_line":"    gracefully."},{"line_number":101,"context_line":"    \"\"\""},{"line_number":102,"context_line":"    with eventlet.timeout.Timeout(context.CELL_TIMEOUT, exception.CellTimeout):"},{"line_number":103,"context_line":"        try:"},{"line_number":104,"context_line":"            for record in fn(ctx, *args, **kwargs):"},{"line_number":105,"context_line":"                yield record"}],"source_content_type":"text/x-python","patch_set":9,"id":"3f79a3b5_1acd7230","line":102,"updated":"2018-08-22 20:18:47.000000000","message":"This made me think of https://review.openstack.org/#/c/592285/ but shouldn\u0027t matter here.","commit_id":"ffc6efd22bc746e7e964c38247058c5843a39524"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"a06df57e0c918c86165da4d688d07051162b593a","unresolved":false,"context_lines":[{"line_number":107,"context_line":"            # Here, we yield a RecordWrapper (no sort_ctx needed since"},{"line_number":108,"context_line":"            # we won\u0027t call into the implementation\u0027s comparison routines)"},{"line_number":109,"context_line":"            # wrapping the sentinel indicating timeout."},{"line_number":110,"context_line":"            yield RecordWrapper(ctx, None, context.did_not_respond_sentinel)"},{"line_number":111,"context_line":"            raise StopIteration"},{"line_number":112,"context_line":"        except Exception:"},{"line_number":113,"context_line":"            # Here, we yield a RecordWrapper (no sort_ctx needed since"}],"source_content_type":"text/x-python","patch_set":9,"id":"3f79a3b5_5a53ea39","line":110,"range":{"start_line":110,"start_character":37,"end_line":110,"end_character":41},"updated":"2018-08-22 20:18:47.000000000","message":"Thinking out loud, it might be nice to have a NonCallableSortContext that, if its compare_result method is called, raises a predictable exception to indicate a programming error, rather than just a NoneType. But maybe that\u0027s essentially the same thing - it\u0027s a programming error either way. I\u0027m just a bit uneasy with the tight coupling of how this assumes how RecordWrapper is implemented today.","commit_id":"ffc6efd22bc746e7e964c38247058c5843a39524"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"ba83b43d0ee6735e8f5f6f433bbbf24a7deee28b","unresolved":false,"context_lines":[{"line_number":107,"context_line":"            # Here, we yield a RecordWrapper (no sort_ctx needed since"},{"line_number":108,"context_line":"            # we won\u0027t call into the implementation\u0027s comparison routines)"},{"line_number":109,"context_line":"            # wrapping the sentinel indicating timeout."},{"line_number":110,"context_line":"            yield RecordWrapper(ctx, None, context.did_not_respond_sentinel)"},{"line_number":111,"context_line":"            raise StopIteration"},{"line_number":112,"context_line":"        except Exception:"},{"line_number":113,"context_line":"            # Here, we yield a RecordWrapper (no sort_ctx needed since"}],"source_content_type":"text/x-python","patch_set":9,"id":"3f79a3b5_f51447a2","line":110,"range":{"start_line":110,"start_character":37,"end_line":110,"end_character":41},"in_reply_to":"3f79a3b5_5a53ea39","updated":"2018-08-22 20:47:12.000000000","message":"Neither of these are override-able by the implementation class (i.e. for instances), so I\u0027m not sure it\u0027s a huge deal. I could make this a staticmethod on RecordWrapper so they\u0027re (one line) closer, if you think it would help. I could also document that sort_ctx can be None in the sentinel case for RecordWrapper and/or check for None after the sentinel check and do something predictable. I will do whatever you want.","commit_id":"ffc6efd22bc746e7e964c38247058c5843a39524"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"fbdefcac0a98fe708d266b82dc27e379afe095f5","unresolved":false,"context_lines":[{"line_number":107,"context_line":"            # Here, we yield a RecordWrapper (no sort_ctx needed since"},{"line_number":108,"context_line":"            # we won\u0027t call into the implementation\u0027s comparison routines)"},{"line_number":109,"context_line":"            # wrapping the sentinel indicating timeout."},{"line_number":110,"context_line":"            yield RecordWrapper(ctx, None, context.did_not_respond_sentinel)"},{"line_number":111,"context_line":"            raise StopIteration"},{"line_number":112,"context_line":"        except Exception:"},{"line_number":113,"context_line":"            # Here, we yield a RecordWrapper (no sort_ctx needed since"}],"source_content_type":"text/x-python","patch_set":9,"id":"3f79a3b5_ffebc174","line":110,"range":{"start_line":110,"start_character":37,"end_line":110,"end_character":41},"in_reply_to":"3f79a3b5_f51447a2","updated":"2018-08-23 01:35:28.000000000","message":"Nah it\u0027s fine, I was mostly just thinking out loud.","commit_id":"ffc6efd22bc746e7e964c38247058c5843a39524"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"a06df57e0c918c86165da4d688d07051162b593a","unresolved":false,"context_lines":[{"line_number":347,"context_line":""},{"line_number":348,"context_line":"        # NOTE(danms): The calls to do_query() will return immediately with"},{"line_number":349,"context_line":"        # a generator. There is no point in us checking the results for"},{"line_number":350,"context_line":"        # failure or timeout anymore since we have not actually run any code"},{"line_number":351,"context_line":"        # in do_query() until the first iteration below. The query_wrapper()"},{"line_number":352,"context_line":"        # utility above handles inline translation of failures and timeouts"},{"line_number":353,"context_line":"        # to sentinels which will be generated and consumed just like any"}],"source_content_type":"text/x-python","patch_set":9,"id":"3f79a3b5_1a2012ac","line":350,"range":{"start_line":350,"start_character":29,"end_line":350,"end_character":36},"updated":"2018-08-22 20:18:47.000000000","message":"nix this - it means something in the context of this review, but not historically, i.e. when I\u0027m looking at this in a year.","commit_id":"ffc6efd22bc746e7e964c38247058c5843a39524"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"a06df57e0c918c86165da4d688d07051162b593a","unresolved":false,"context_lines":[{"line_number":349,"context_line":"        # a generator. There is no point in us checking the results for"},{"line_number":350,"context_line":"        # failure or timeout anymore since we have not actually run any code"},{"line_number":351,"context_line":"        # in do_query() until the first iteration below. The query_wrapper()"},{"line_number":352,"context_line":"        # utility above handles inline translation of failures and timeouts"},{"line_number":353,"context_line":"        # to sentinels which will be generated and consumed just like any"},{"line_number":354,"context_line":"        # normal result below."},{"line_number":355,"context_line":"        if self.cells:"}],"source_content_type":"text/x-python","patch_set":9,"id":"3f79a3b5_7a0ee63b","line":352,"range":{"start_line":352,"start_character":18,"end_line":352,"end_character":23},"updated":"2018-08-22 20:18:47.000000000","message":"nix","commit_id":"ffc6efd22bc746e7e964c38247058c5843a39524"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"a06df57e0c918c86165da4d688d07051162b593a","unresolved":false,"context_lines":[{"line_number":381,"context_line":"            if item._db_record in CELL_SENTINELS:"},{"line_number":382,"context_line":"                LOG.warning(\u0027Cell %s is not responding and hence is \u0027"},{"line_number":383,"context_line":"                            \u0027being omitted from the results\u0027,"},{"line_number":384,"context_line":"                            item.cell_uuid)"},{"line_number":385,"context_line":"                continue"},{"line_number":386,"context_line":""},{"line_number":387,"context_line":"            yield item._db_record"}],"source_content_type":"text/x-python","patch_set":9,"id":"3f79a3b5_152ea391","line":384,"range":{"start_line":384,"start_character":28,"end_line":384,"end_character":42},"updated":"2018-08-22 20:18:47.000000000","message":"OK here is where the RecordWrapper.cell_uuid pays off.","commit_id":"ffc6efd22bc746e7e964c38247058c5843a39524"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"ba83b43d0ee6735e8f5f6f433bbbf24a7deee28b","unresolved":false,"context_lines":[{"line_number":381,"context_line":"            if item._db_record in CELL_SENTINELS:"},{"line_number":382,"context_line":"                LOG.warning(\u0027Cell %s is not responding and hence is \u0027"},{"line_number":383,"context_line":"                            \u0027being omitted from the results\u0027,"},{"line_number":384,"context_line":"                            item.cell_uuid)"},{"line_number":385,"context_line":"                continue"},{"line_number":386,"context_line":""},{"line_number":387,"context_line":"            yield item._db_record"}],"source_content_type":"text/x-python","patch_set":9,"id":"3f79a3b5_b57fafdd","line":384,"range":{"start_line":384,"start_character":28,"end_line":384,"end_character":42},"in_reply_to":"3f79a3b5_152ea391","updated":"2018-08-22 20:47:12.000000000","message":"Here and again in the patch where I record which cells failed.","commit_id":"ffc6efd22bc746e7e964c38247058c5843a39524"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"b5e8ffd3a689c8bf4cba0e98e5eb795f21fd84b3","unresolved":false,"context_lines":[{"line_number":74,"context_line":"        # cells, and would not properly report them as failed for"},{"line_number":75,"context_line":"        # fix-up in the higher layers."},{"line_number":76,"context_line":"        if self._db_record in CELL_FAIL_SENTINELS:"},{"line_number":77,"context_line":"            return -1"},{"line_number":78,"context_line":"        elif other._db_record in CELL_FAIL_SENTINELS:"},{"line_number":79,"context_line":"            return 1"},{"line_number":80,"context_line":""},{"line_number":81,"context_line":"        r \u003d self._sort_ctx.compare_records(self._db_record,"},{"line_number":82,"context_line":"                                           other._db_record)"}],"source_content_type":"text/x-python","patch_set":10,"id":"3f79a3b5_a0965a4b","line":79,"range":{"start_line":77,"start_character":0,"end_line":79,"end_character":20},"updated":"2018-08-24 16:08:05.000000000","message":"This is a bug. I was thinking cmp() interface here, but this is __lt__ which returns a boolean. So, this doesn\u0027t break anything it just doesn\u0027t sort the sentinels to the top like I wanted. It\u0027s not until the later patch that I have the data to really test this easily, and working on that test I realized this isn\u0027t quite right.\n\nTrivial fix coming.","commit_id":"9c5af5bb198e9aa108b2233142ca0c50a0b5b60c"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"c06678d0cb85e4b220747979372c9895dfae3a1a","unresolved":false,"context_lines":[{"line_number":74,"context_line":"        # cells, and would not properly report them as failed for"},{"line_number":75,"context_line":"        # fix-up in the higher layers."},{"line_number":76,"context_line":"        if self._db_record in CELL_FAIL_SENTINELS:"},{"line_number":77,"context_line":"            return -1"},{"line_number":78,"context_line":"        elif other._db_record in CELL_FAIL_SENTINELS:"},{"line_number":79,"context_line":"            return 1"},{"line_number":80,"context_line":""},{"line_number":81,"context_line":"        r \u003d self._sort_ctx.compare_records(self._db_record,"},{"line_number":82,"context_line":"                                           other._db_record)"}],"source_content_type":"text/x-python","patch_set":10,"id":"3f79a3b5_6cbd9800","line":79,"range":{"start_line":77,"start_character":0,"end_line":79,"end_character":20},"in_reply_to":"3f79a3b5_a0965a4b","updated":"2018-08-24 20:18:23.000000000","message":"I must have also looked at this and thought cmp() because I did question why one failure was -1 and the other was 1, but apparently didn\u0027t think enough to ask why...","commit_id":"9c5af5bb198e9aa108b2233142ca0c50a0b5b60c"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"3e4871450f46fc78900bd2bf92563ba85b92fb48","unresolved":false,"context_lines":[{"line_number":76,"context_line":"        if self._db_record in CELL_FAIL_SENTINELS:"},{"line_number":77,"context_line":"            return True"},{"line_number":78,"context_line":"        elif other._db_record in CELL_FAIL_SENTINELS:"},{"line_number":79,"context_line":"            return False"},{"line_number":80,"context_line":""},{"line_number":81,"context_line":"        r \u003d self._sort_ctx.compare_records(self._db_record,"},{"line_number":82,"context_line":"                                           other._db_record)"}],"source_content_type":"text/x-python","patch_set":12,"id":"3f79a3b5_41e2e8d6","line":79,"updated":"2018-08-24 18:21:40.000000000","message":"✓","commit_id":"3850a215b742a66d40c979b3ecfa57158fea8b15"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"3e4871450f46fc78900bd2bf92563ba85b92fb48","unresolved":false,"context_lines":[{"line_number":126,"context_line":"    def __init__(self, sort_ctx, cells\u003dNone, batch_size\u003dNone):"},{"line_number":127,"context_line":"        self.sort_ctx \u003d sort_ctx"},{"line_number":128,"context_line":"        self.cells \u003d cells"},{"line_number":129,"context_line":"        self.batch_size \u003d batch_size"},{"line_number":130,"context_line":""},{"line_number":131,"context_line":"    @property"},{"line_number":132,"context_line":"    @abc.abstractmethod"}],"source_content_type":"text/x-python","patch_set":12,"id":"3f79a3b5_a16e5c2c","line":129,"updated":"2018-08-24 18:21:40.000000000","message":"nit: might be nice to docstring this new parameter to indicate that this represents the per-cell-fetch limit for listing operations.","commit_id":"3850a215b742a66d40c979b3ecfa57158fea8b15"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"3e4871450f46fc78900bd2bf92563ba85b92fb48","unresolved":false,"context_lines":[{"line_number":248,"context_line":"            marker_id \u003d self.marker_identifier"},{"line_number":249,"context_line":""},{"line_number":250,"context_line":"            if marker:"},{"line_number":251,"context_line":"                # FIXME(danms): If we knew which cell we were in here, we could"},{"line_number":252,"context_line":"                # avoid looking up the marker again. But, we don\u0027t currently."},{"line_number":253,"context_line":""},{"line_number":254,"context_line":"                local_marker \u003d self.get_marker_by_values(cctx,"},{"line_number":255,"context_line":"                                                         global_marker_values)"}],"source_content_type":"text/x-python","patch_set":12,"id":"3f79a3b5_41cc2834","line":252,"range":{"start_line":251,"start_character":16,"end_line":252,"end_character":77},"updated":"2018-08-24 18:21:40.000000000","message":"You now know which cell you\u0027re in here, though, right? Per a previous patch in this series, you\u0027ve modded do_query() to contain the cctx, not ctx, and pass it to get_marker_by_values() on 254 below...","commit_id":"3850a215b742a66d40c979b3ecfa57158fea8b15"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"46f3737d9005f3bd5650ee9e86e1f0a947b6b96a","unresolved":false,"context_lines":[{"line_number":248,"context_line":"            marker_id \u003d self.marker_identifier"},{"line_number":249,"context_line":""},{"line_number":250,"context_line":"            if marker:"},{"line_number":251,"context_line":"                # FIXME(danms): If we knew which cell we were in here, we could"},{"line_number":252,"context_line":"                # avoid looking up the marker again. But, we don\u0027t currently."},{"line_number":253,"context_line":""},{"line_number":254,"context_line":"                local_marker \u003d self.get_marker_by_values(cctx,"},{"line_number":255,"context_line":"                                                         global_marker_values)"}],"source_content_type":"text/x-python","patch_set":12,"id":"3f79a3b5_6cc35839","line":252,"range":{"start_line":251,"start_character":16,"end_line":252,"end_character":77},"in_reply_to":"3f79a3b5_0c10a4ff","updated":"2018-08-24 20:27:01.000000000","message":"And even still, we have the context but we don\u0027t know the actual cell we\u0027re in, just how to talk to it. But yeah, later in the series after we learn how to tell, this gets removed.","commit_id":"3850a215b742a66d40c979b3ecfa57158fea8b15"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"188b90c67d76d933ac78b09e70408fdc3849ce53","unresolved":false,"context_lines":[{"line_number":248,"context_line":"            marker_id \u003d self.marker_identifier"},{"line_number":249,"context_line":""},{"line_number":250,"context_line":"            if marker:"},{"line_number":251,"context_line":"                # FIXME(danms): If we knew which cell we were in here, we could"},{"line_number":252,"context_line":"                # avoid looking up the marker again. But, we don\u0027t currently."},{"line_number":253,"context_line":""},{"line_number":254,"context_line":"                local_marker \u003d self.get_marker_by_values(cctx,"},{"line_number":255,"context_line":"                                                         global_marker_values)"}],"source_content_type":"text/x-python","patch_set":12,"id":"3f79a3b5_0c10a4ff","line":252,"range":{"start_line":251,"start_character":16,"end_line":252,"end_character":77},"in_reply_to":"3f79a3b5_41cc2834","updated":"2018-08-24 20:23:30.000000000","message":"This gets removed/fixed later in the series.","commit_id":"3850a215b742a66d40c979b3ecfa57158fea8b15"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"3e4871450f46fc78900bd2bf92563ba85b92fb48","unresolved":false,"context_lines":[{"line_number":308,"context_line":""},{"line_number":309,"context_line":"                # Do not query a full batch if it would cause our total"},{"line_number":310,"context_line":"                # to exceed the limit"},{"line_number":311,"context_line":"                if limit:"},{"line_number":312,"context_line":"                    query_size \u003d min(batch_size, limit - return_count)"},{"line_number":313,"context_line":"                else:"},{"line_number":314,"context_line":"                    query_size \u003d batch_size"},{"line_number":315,"context_line":""},{"line_number":316,"context_line":"                # Get one batch"},{"line_number":317,"context_line":"                query_result \u003d self.get_by_filters("}],"source_content_type":"text/x-python","patch_set":12,"id":"3f79a3b5_01b95077","line":314,"range":{"start_line":311,"start_character":16,"end_line":314,"end_character":43},"updated":"2018-08-24 18:21:40.000000000","message":"note that batch_size can be None, since this:\n\n batch_size \u003d self.batch_size or limit\n\ncan result in None (since both self.batch_size and limit may be None).\n\nwhich means query_size may also be None.","commit_id":"3850a215b742a66d40c979b3ecfa57158fea8b15"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"3e4871450f46fc78900bd2bf92563ba85b92fb48","unresolved":false,"context_lines":[{"line_number":316,"context_line":"                # Get one batch"},{"line_number":317,"context_line":"                query_result \u003d self.get_by_filters("},{"line_number":318,"context_line":"                    cctx, filters,"},{"line_number":319,"context_line":"                    limit\u003dquery_size or None, marker\u003dlocal_marker,"},{"line_number":320,"context_line":"                    **kwargs)"},{"line_number":321,"context_line":""},{"line_number":322,"context_line":"                # Yield wrapped results from the batch, counting as we go"}],"source_content_type":"text/x-python","patch_set":12,"id":"3f79a3b5_016e30ff","line":319,"range":{"start_line":319,"start_character":27,"end_line":319,"end_character":44},"updated":"2018-08-24 18:21:40.000000000","message":"and therefore I\u0027m not sure if this is necessary (can just be limit\u003dquery_size)","commit_id":"3850a215b742a66d40c979b3ecfa57158fea8b15"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"3e4871450f46fc78900bd2bf92563ba85b92fb48","unresolved":false,"context_lines":[{"line_number":346,"context_line":"        # run any code in do_query() until the first iteration"},{"line_number":347,"context_line":"        # below. The query_wrapper() utility handles inline"},{"line_number":348,"context_line":"        # translation of failures and timeouts to sentinels which will"},{"line_number":349,"context_line":"        # be generated and consumed just like any normal result below."},{"line_number":350,"context_line":"        if self.cells:"},{"line_number":351,"context_line":"            results \u003d context.scatter_gather_cells(ctx, self.cells,"},{"line_number":352,"context_line":"                                                   context.CELL_TIMEOUT,"}],"source_content_type":"text/x-python","patch_set":12,"id":"3f79a3b5_a1617c07","line":349,"updated":"2018-08-24 18:21:40.000000000","message":"✓ nice comment.","commit_id":"3850a215b742a66d40c979b3ecfa57158fea8b15"}],"nova/tests/functional/compute/test_instance_list.py":[{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"354fa4125e3bceb9ce88958ee8c54c920a06fdfa","unresolved":false,"context_lines":[{"line_number":146,"context_line":"        self.assertEqual(sorted(uuids), uuids)"},{"line_number":147,"context_line":"        self.assertEqual(len(self.instances), len(uuids))"},{"line_number":148,"context_line":""},{"line_number":149,"context_line":"    def test_get_sorted_with_large_limit_batched(self):"},{"line_number":150,"context_line":"        insts \u003d instance_list.get_instances_sorted(self.context, {},"},{"line_number":151,"context_line":"                                                   5000, None,"},{"line_number":152,"context_line":"                                                   [], [\u0027uuid\u0027], [\u0027asc\u0027],"}],"source_content_type":"text/x-python","patch_set":4,"id":"3f79a3b5_a205827c","line":149,"range":{"start_line":149,"start_character":8,"end_line":149,"end_character":48},"updated":"2018-08-20 14:13:22.000000000","message":"So this is the same as test_get_sorted_with_large_limit which shows we eventually get the right result, but doesn\u0027t otherwise tell us anything about the batch stuff being done, right? Like if we removed the patch code (just didn\u0027t honor the batch_size param), this would still pass, right? Is there some way to assert that we\u0027re making DB calls with the batch size if provided? I\u0027m not sure how easy that is without mocking out the get_by_filters call to pass through to the real DB API method but keep track of how it was called (assert the limit passed was 2 instead of unlimited).\n\n(later)\n\nI guess that\u0027s what TestBatching is for. So nevermind.","commit_id":"8ba2483ad16013801155184a8a29f00273b218ed"}],"nova/tests/unit/compute/test_multi_cell_list.py":[{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"a06df57e0c918c86165da4d688d07051162b593a","unresolved":false,"context_lines":[{"line_number":279,"context_line":"        self.assertEqual(limit_expected, summary[\u0027limit_by_cell\u0027])"},{"line_number":280,"context_line":""},{"line_number":281,"context_line":""},{"line_number":282,"context_line":"class FailyLister(TestLister):"},{"line_number":283,"context_line":"    def __init__(self, *a, **k):"},{"line_number":284,"context_line":"        super(FailyLister, self).__init__(*a, **k)"},{"line_number":285,"context_line":"        self._fails \u003d [context.did_not_respond_sentinel,"}],"source_content_type":"text/x-python","patch_set":9,"id":"3f79a3b5_552f3b87","line":282,"range":{"start_line":282,"start_character":6,"end_line":282,"end_character":11},"updated":"2018-08-22 20:18:47.000000000","message":"I can only imagine how our Chinese friends are trying to parse this. Maybe rename to FailureLister? Or at least add a docstring.\n\nI\u0027m extremely culturally sensitive you know.","commit_id":"ffc6efd22bc746e7e964c38247058c5843a39524"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"188b90c67d76d933ac78b09e70408fdc3849ce53","unresolved":false,"context_lines":[{"line_number":115,"context_line":"        ctx \u003d context.RequestContext()"},{"line_number":116,"context_line":"        ctx.cell_uuid \u003d uuids.cell"},{"line_number":117,"context_line":""},{"line_number":118,"context_line":"        # Should sort by key1"},{"line_number":119,"context_line":"        sort_ctx \u003d multi_cell_list.RecordSortContext([\u0027key0\u0027, \u0027key1\u0027],"},{"line_number":120,"context_line":"                                                     [\u0027asc\u0027, \u0027asc\u0027])"},{"line_number":121,"context_line":"        iw1 \u003d multi_cell_list.RecordWrapper(ctx, sort_ctx, inst1)"}],"source_content_type":"text/x-python","patch_set":12,"id":"3f79a3b5_8c14b4ea","line":118,"range":{"start_line":118,"start_character":25,"end_line":118,"end_character":29},"updated":"2018-08-24 20:23:30.000000000","message":"But you\u0027re passing 2 sort keys?","commit_id":"3850a215b742a66d40c979b3ecfa57158fea8b15"}]}