)]}'
{"nova/cmd/manage.py":[{"author":{"_account_id":782,"name":"John Garbutt","email":"john@johngarbutt.com","username":"johngarbutt"},"change_message_id":"6f541c12cd29d23f91bd573a2ca817025f337f02","unresolved":false,"context_lines":[{"line_number":947,"context_line":"class DbCommands(object):"},{"line_number":948,"context_line":"    \"\"\"Class for managing the main database.\"\"\""},{"line_number":949,"context_line":""},{"line_number":950,"context_line":"    online_migrations \u003d ("},{"line_number":951,"context_line":"        db.OnlineDataMigration(objects.PciDevice,"},{"line_number":952,"context_line":"                               db.sqlalchemy.models.PciDevice),"},{"line_number":953,"context_line":"    )"}],"source_content_type":"text/x-python","patch_set":2,"id":"3a57f1b5_d1f9fa4e","line":950,"updated":"2016-02-10 09:48:17.000000000","message":"oh, thats quite nice actually.","commit_id":"ebad1149ce68d43d4f7d1ae33f1f6af93c16c7a3"},{"author":{"_account_id":782,"name":"John Garbutt","email":"john@johngarbutt.com","username":"johngarbutt"},"change_message_id":"9d8764d0179c6a785d042a5ec973b80eb22f7eed","unresolved":false,"context_lines":[{"line_number":947,"context_line":"class DbCommands(object):"},{"line_number":948,"context_line":"    \"\"\"Class for managing the main database.\"\"\""},{"line_number":949,"context_line":""},{"line_number":950,"context_line":"    online_migrations \u003d ("},{"line_number":951,"context_line":"        db.OnlineDataMigration(objects.PciDevice,"},{"line_number":952,"context_line":"                               db.sqlalchemy.models.PciDevice),"},{"line_number":953,"context_line":"    )"}],"source_content_type":"text/x-python","patch_set":2,"id":"3a57f1b5_7c2c9d4a","line":950,"in_reply_to":"3a57f1b5_d1f9fa4e","updated":"2016-02-10 09:55:31.000000000","message":"So the bit I don\u0027t like is having to pass in the DB model.\n\nI think all the DB queries should stay encapsulated in the DB object.\n\nBut the explicit list of objects here, rather than looping around all objects in the registry, seems a good plan.\n\nIf folks hit issues, they could hack the list to re-order which ones are done first (well, we could tune that in tree also).","commit_id":"ebad1149ce68d43d4f7d1ae33f1f6af93c16c7a3"},{"author":{"_account_id":782,"name":"John Garbutt","email":"john@johngarbutt.com","username":"johngarbutt"},"change_message_id":"fd7f663b9be8a332e92be261e744dd10d73bdadb","unresolved":false,"context_lines":[{"line_number":924,"context_line":"    \"\"\"Class for managing the main database.\"\"\""},{"line_number":925,"context_line":""},{"line_number":926,"context_line":"    online_migrations \u003d ("},{"line_number":927,"context_line":"        db.pcidevice_online_data_migration,"},{"line_number":928,"context_line":"    )"},{"line_number":929,"context_line":""},{"line_number":930,"context_line":"    def __init__(self):"}],"source_content_type":"text/x-python","patch_set":5,"id":"dae33548_f4d5cbe4","line":927,"updated":"2016-02-18 12:20:42.000000000","message":"honestly, my preference is still for a link to an object method, to share the code, but this works.","commit_id":"60f7db9f65910e03cce19d9776354e7eac803564"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"02063d1324eb1ed06725b07ce8c6c0527c4238f4","unresolved":false,"context_lines":[{"line_number":924,"context_line":"    \"\"\"Class for managing the main database.\"\"\""},{"line_number":925,"context_line":""},{"line_number":926,"context_line":"    online_migrations \u003d ("},{"line_number":927,"context_line":"        db.pcidevice_online_data_migration,"},{"line_number":928,"context_line":"    )"},{"line_number":929,"context_line":""},{"line_number":930,"context_line":"    def __init__(self):"}],"source_content_type":"text/x-python","patch_set":5,"id":"dae33548_53111b51","line":927,"in_reply_to":"dae33548_2b0b4925","updated":"2016-02-18 16:38:53.000000000","message":"Yeah, see my previous comments. These don\u0027t always correspond to one object. For example, the ComputeNode, Inventory, ResourceProvider, Instant, and Allocation migrations we have coming up :)","commit_id":"60f7db9f65910e03cce19d9776354e7eac803564"},{"author":{"_account_id":782,"name":"John Garbutt","email":"john@johngarbutt.com","username":"johngarbutt"},"change_message_id":"be0fc0bd01084c8c2158625bb11ab23d6f7c3ce1","unresolved":false,"context_lines":[{"line_number":924,"context_line":"    \"\"\"Class for managing the main database.\"\"\""},{"line_number":925,"context_line":""},{"line_number":926,"context_line":"    online_migrations \u003d ("},{"line_number":927,"context_line":"        db.pcidevice_online_data_migration,"},{"line_number":928,"context_line":"    )"},{"line_number":929,"context_line":""},{"line_number":930,"context_line":"    def __init__(self):"}],"source_content_type":"text/x-python","patch_set":5,"id":"dae33548_2b0b4925","line":927,"in_reply_to":"dae33548_4f4dfe98","updated":"2016-02-18 13:30:29.000000000","message":"OK, fair enough.","commit_id":"60f7db9f65910e03cce19d9776354e7eac803564"},{"author":{"_account_id":782,"name":"John Garbutt","email":"john@johngarbutt.com","username":"johngarbutt"},"change_message_id":"8fc30ecd230be79cf942fe4011a513ae0725d623","unresolved":false,"context_lines":[{"line_number":924,"context_line":"    \"\"\"Class for managing the main database.\"\"\""},{"line_number":925,"context_line":""},{"line_number":926,"context_line":"    online_migrations \u003d ("},{"line_number":927,"context_line":"        db.pcidevice_online_data_migration,"},{"line_number":928,"context_line":"    )"},{"line_number":929,"context_line":""},{"line_number":930,"context_line":"    def __init__(self):"}],"source_content_type":"text/x-python","patch_set":5,"id":"dae33548_3c6b831f","line":927,"in_reply_to":"dae33548_53111b51","updated":"2016-02-18 18:00:50.000000000","message":"yeah, totally makes sense. I was forgetting that before.","commit_id":"60f7db9f65910e03cce19d9776354e7eac803564"},{"author":{"_account_id":5511,"name":"Nikola Dipanov","email":"ndipanov@redhat.com","username":"ndipanov"},"change_message_id":"a8f10db9a64c3587bf53cf4ff9a3ac01e3d059dc","unresolved":false,"context_lines":[{"line_number":924,"context_line":"    \"\"\"Class for managing the main database.\"\"\""},{"line_number":925,"context_line":""},{"line_number":926,"context_line":"    online_migrations \u003d ("},{"line_number":927,"context_line":"        db.pcidevice_online_data_migration,"},{"line_number":928,"context_line":"    )"},{"line_number":929,"context_line":""},{"line_number":930,"context_line":"    def __init__(self):"}],"source_content_type":"text/x-python","patch_set":5,"id":"dae33548_4f4dfe98","line":927,"in_reply_to":"dae33548_f4d5cbe4","updated":"2016-02-18 12:30:00.000000000","message":"Dan did not want this in the objects a of yet since in the general case it\u0027s not a single object that gets migrated but potentially several","commit_id":"60f7db9f65910e03cce19d9776354e7eac803564"}],"nova/db/sqlalchemy/api.py":[{"author":{"_account_id":5511,"name":"Nikola Dipanov","email":"ndipanov@redhat.com","username":"ndipanov"},"change_message_id":"b16bb9392115dbe2ceae8f2400dd8bd878521432","unresolved":false,"context_lines":[{"line_number":6160,"context_line":"    else:"},{"line_number":6161,"context_line":"        query \u003d model_query(context, models.PciDevice)"},{"line_number":6162,"context_line":"        if max_count is not None:"},{"line_number":6163,"context_line":"            results \u003d list(query.limit(max_count))"},{"line_number":6164,"context_line":"        else:"},{"line_number":6165,"context_line":"            results \u003d query.all()"},{"line_number":6166,"context_line":""}],"source_content_type":"text/x-python","patch_set":5,"id":"dae33548_e47b4bf3","line":6163,"range":{"start_line":6163,"start_character":22,"end_line":6163,"end_character":50},"updated":"2016-02-18 15:35:09.000000000","message":"As disscussed on IRC - we might actually want to make sure we are only getting the viable migration candidates here to increase the likelyhood of hits","commit_id":"60f7db9f65910e03cce19d9776354e7eac803564"},{"author":{"_account_id":782,"name":"John Garbutt","email":"john@johngarbutt.com","username":"johngarbutt"},"change_message_id":"fd7f663b9be8a332e92be261e744dd10d73bdadb","unresolved":false,"context_lines":[{"line_number":6162,"context_line":"        if max_count is not None:"},{"line_number":6163,"context_line":"            results \u003d list(query.limit(max_count))"},{"line_number":6164,"context_line":"        else:"},{"line_number":6165,"context_line":"            results \u003d query.all()"},{"line_number":6166,"context_line":""},{"line_number":6167,"context_line":"        for db_dict in results:"},{"line_number":6168,"context_line":"            count_all +\u003d 1"}],"source_content_type":"text/x-python","patch_set":5,"id":"dae33548_6fcc42bb","line":6165,"updated":"2016-02-18 12:20:42.000000000","message":"Do we really want a max batch size here, and modify the outer loop to call us again? Just to keep the memory usage low-ish. Maybe I am miss-understanding whats going on here?","commit_id":"60f7db9f65910e03cce19d9776354e7eac803564"},{"author":{"_account_id":5511,"name":"Nikola Dipanov","email":"ndipanov@redhat.com","username":"ndipanov"},"change_message_id":"b16bb9392115dbe2ceae8f2400dd8bd878521432","unresolved":false,"context_lines":[{"line_number":6162,"context_line":"        if max_count is not None:"},{"line_number":6163,"context_line":"            results \u003d list(query.limit(max_count))"},{"line_number":6164,"context_line":"        else:"},{"line_number":6165,"context_line":"            results \u003d query.all()"},{"line_number":6166,"context_line":""},{"line_number":6167,"context_line":"        for db_dict in results:"},{"line_number":6168,"context_line":"            count_all +\u003d 1"}],"source_content_type":"text/x-python","patch_set":5,"id":"dae33548_ce2bc680","line":6165,"in_reply_to":"dae33548_6fcc42bb","updated":"2016-02-18 15:35:09.000000000","message":"Well we lose the ability to do a true \"one shot\" migration then for folks who do not mind the downtime.","commit_id":"60f7db9f65910e03cce19d9776354e7eac803564"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"02063d1324eb1ed06725b07ce8c6c0527c4238f4","unresolved":false,"context_lines":[{"line_number":6162,"context_line":"        if max_count is not None:"},{"line_number":6163,"context_line":"            results \u003d list(query.limit(max_count))"},{"line_number":6164,"context_line":"        else:"},{"line_number":6165,"context_line":"            results \u003d query.all()"},{"line_number":6166,"context_line":""},{"line_number":6167,"context_line":"        for db_dict in results:"},{"line_number":6168,"context_line":"            count_all +\u003d 1"}],"source_content_type":"text/x-python","patch_set":5,"id":"dae33548_d3524b12","line":6165,"in_reply_to":"dae33548_ce2bc680","updated":"2016-02-18 16:38:53.000000000","message":"Looping the caller until it\u0027s done would give us the same result. John\u0027s point is that if we have a billion results from the query, we\u0027ll just OOM instead of making progress, which is probably a good point.","commit_id":"60f7db9f65910e03cce19d9776354e7eac803564"},{"author":{"_account_id":782,"name":"John Garbutt","email":"john@johngarbutt.com","username":"johngarbutt"},"change_message_id":"8fc30ecd230be79cf942fe4011a513ae0725d623","unresolved":false,"context_lines":[{"line_number":6162,"context_line":"        if max_count is not None:"},{"line_number":6163,"context_line":"            results \u003d list(query.limit(max_count))"},{"line_number":6164,"context_line":"        else:"},{"line_number":6165,"context_line":"            results \u003d query.all()"},{"line_number":6166,"context_line":""},{"line_number":6167,"context_line":"        for db_dict in results:"},{"line_number":6168,"context_line":"            count_all +\u003d 1"}],"source_content_type":"text/x-python","patch_set":5,"id":"dae33548_7c718b0b","line":6165,"in_reply_to":"dae33548_d3524b12","updated":"2016-02-18 18:00:50.000000000","message":"+1 on Dan\u0027s point.","commit_id":"60f7db9f65910e03cce19d9776354e7eac803564"},{"author":{"_account_id":782,"name":"John Garbutt","email":"john@johngarbutt.com","username":"johngarbutt"},"change_message_id":"fd7f663b9be8a332e92be261e744dd10d73bdadb","unresolved":false,"context_lines":[{"line_number":6170,"context_line":"                context, pci_dev_obj.PciDevice(), db_dict)"},{"line_number":6171,"context_line":"            if pci_dev.obj_what_changed():"},{"line_number":6172,"context_line":"                pci_dev.save()"},{"line_number":6173,"context_line":"                count_hit +\u003d 1"},{"line_number":6174,"context_line":"    return count_all, count_hit"},{"line_number":6175,"context_line":""},{"line_number":6176,"context_line":""}],"source_content_type":"text/x-python","patch_set":5,"id":"dae33548_4f021e0b","line":6173,"updated":"2016-02-18 12:20:42.000000000","message":"I was expecting the max_count to limit the hits, not the query.\n\nMy assumption was you keep calling the script with the same limit number, say 100, and it slowly works through all possible data_migrations.\n\nI think with this you have to keep increasing the number with each call I guess?\n\nI have a feeling I miss-read that first patch slightly. oops.","commit_id":"60f7db9f65910e03cce19d9776354e7eac803564"},{"author":{"_account_id":782,"name":"John Garbutt","email":"john@johngarbutt.com","username":"johngarbutt"},"change_message_id":"8fc30ecd230be79cf942fe4011a513ae0725d623","unresolved":false,"context_lines":[{"line_number":6170,"context_line":"                context, pci_dev_obj.PciDevice(), db_dict)"},{"line_number":6171,"context_line":"            if pci_dev.obj_what_changed():"},{"line_number":6172,"context_line":"                pci_dev.save()"},{"line_number":6173,"context_line":"                count_hit +\u003d 1"},{"line_number":6174,"context_line":"    return count_all, count_hit"},{"line_number":6175,"context_line":""},{"line_number":6176,"context_line":""}],"source_content_type":"text/x-python","patch_set":5,"id":"dae33548_fcd2bbcf","line":6173,"in_reply_to":"dae33548_4ec85655","updated":"2016-02-18 18:00:50.000000000","message":"Sorry, I was really trying to say, each call with the max set should make progress, re issue on line 6163.","commit_id":"60f7db9f65910e03cce19d9776354e7eac803564"},{"author":{"_account_id":5511,"name":"Nikola Dipanov","email":"ndipanov@redhat.com","username":"ndipanov"},"change_message_id":"b16bb9392115dbe2ceae8f2400dd8bd878521432","unresolved":false,"context_lines":[{"line_number":6170,"context_line":"                context, pci_dev_obj.PciDevice(), db_dict)"},{"line_number":6171,"context_line":"            if pci_dev.obj_what_changed():"},{"line_number":6172,"context_line":"                pci_dev.save()"},{"line_number":6173,"context_line":"                count_hit +\u003d 1"},{"line_number":6174,"context_line":"    return count_all, count_hit"},{"line_number":6175,"context_line":""},{"line_number":6176,"context_line":""}],"source_content_type":"text/x-python","patch_set":5,"id":"dae33548_4ec85655","line":6173,"in_reply_to":"dae33548_4f021e0b","updated":"2016-02-18 15:35:09.000000000","message":"well we kind of assumed when we did these previously that operators would like to run these periodically with a max_number appropriately set so that it doesn\u0027t cause too much load. I\u0027m not sure we want to make this smarter than that?","commit_id":"60f7db9f65910e03cce19d9776354e7eac803564"},{"author":{"_account_id":782,"name":"John Garbutt","email":"john@johngarbutt.com","username":"johngarbutt"},"change_message_id":"9c7a7bf5c212cb5679e4a26f8607cac90e3accaa","unresolved":false,"context_lines":[{"line_number":6163,"context_line":"        if max_count is not None:"},{"line_number":6164,"context_line":"            results \u003d list(query.limit(max_count))"},{"line_number":6165,"context_line":"        else:"},{"line_number":6166,"context_line":"            results \u003d query.all()"},{"line_number":6167,"context_line":""},{"line_number":6168,"context_line":"        for db_dict in results:"},{"line_number":6169,"context_line":"            count_all +\u003d 1"}],"source_content_type":"text/x-python","patch_set":6,"id":"dae33548_3c49c331","line":6166,"updated":"2016-02-18 17:58:12.000000000","message":"Still a bit worried about not batching this somehow, but at least its now only \"all things that need converting\".","commit_id":"b939ea17487e283a2acc6effeb43b97d611dc41f"},{"author":{"_account_id":5511,"name":"Nikola Dipanov","email":"ndipanov@redhat.com","username":"ndipanov"},"change_message_id":"2494e7ad0cb9d268288ffed3ab8da09b26aef465","unresolved":false,"context_lines":[{"line_number":6163,"context_line":"        if max_count is not None:"},{"line_number":6164,"context_line":"            results \u003d list(query.limit(max_count))"},{"line_number":6165,"context_line":"        else:"},{"line_number":6166,"context_line":"            results \u003d query.all()"},{"line_number":6167,"context_line":""},{"line_number":6168,"context_line":"        for db_dict in results:"},{"line_number":6169,"context_line":"            count_all +\u003d 1"}],"source_content_type":"text/x-python","patch_set":6,"id":"bae84128_3f306756","line":6166,"in_reply_to":"dae33548_07aaa884","updated":"2016-02-19 18:14:17.000000000","message":"I am against setting a max batch size. There is already a way for a user to limit how much data they want to get, so if they are concerned about memory usage they can use that.\n\nI am not sure slicing up .all() into batches really reduces memory consumption. We\u0027d need to test it out, there is a lot of pretty complex code working under the hood here (sqla, oslodb, sessions) + whatever CPython does and I would not bet that it\u0027s as straightforward as batches \u003d\u003d less memeory consumption.\n\nOne PciDevice is probably about 1KiB at worst (data is likely to be less almost always). Assuming 1000 per compute node (veeery generous) that\u0027s a megabyte per compute host. My laptop would handle migrating a cloud of 8000 compute hosts and I have a hipster little thin laptop.","commit_id":"b939ea17487e283a2acc6effeb43b97d611dc41f"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"ce82c443dc70859b26a810328a818fe2d967947b","unresolved":false,"context_lines":[{"line_number":6163,"context_line":"        if max_count is not None:"},{"line_number":6164,"context_line":"            results \u003d list(query.limit(max_count))"},{"line_number":6165,"context_line":"        else:"},{"line_number":6166,"context_line":"            results \u003d query.all()"},{"line_number":6167,"context_line":""},{"line_number":6168,"context_line":"        for db_dict in results:"},{"line_number":6169,"context_line":"            count_all +\u003d 1"}],"source_content_type":"text/x-python","patch_set":6,"id":"dae33548_07aaa884","line":6166,"in_reply_to":"dae33548_3c49c331","updated":"2016-02-18 18:11:34.000000000","message":"Yeah, thinking about this more, especially in the pci device case, I\u0027m concerned about the sheer number of pci device records we\u0027d pull doing this .all() even on a smallish number of compute nodes. I think batching is probably a really good idea.","commit_id":"b939ea17487e283a2acc6effeb43b97d611dc41f"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"e89c6dd3cdb8cc15aceb424922d8c79e65d7e65a","unresolved":false,"context_lines":[{"line_number":6163,"context_line":"        if max_count is not None:"},{"line_number":6164,"context_line":"            results \u003d list(query.limit(max_count))"},{"line_number":6165,"context_line":"        else:"},{"line_number":6166,"context_line":"            results \u003d query.all()"},{"line_number":6167,"context_line":""},{"line_number":6168,"context_line":"        for db_dict in results:"},{"line_number":6169,"context_line":"            count_all +\u003d 1"}],"source_content_type":"text/x-python","patch_set":7,"id":"bae84128_a1ccce6a","line":6166,"updated":"2016-02-23 15:49:51.000000000","message":"So, I still feel like this is asking for a problem. At the minimum I think we need to have the release note mention something about not just running all of them.\n\nI *know* that puppet scripts are going to just run this with an unbounded maximum and then things are going to fail for people who are running this on their smallest machine in the middle of a puppet apply.","commit_id":"5964073a7d598b4825fa38cf3fcd6697689c93a5"},{"author":{"_account_id":782,"name":"John Garbutt","email":"john@johngarbutt.com","username":"johngarbutt"},"change_message_id":"b4e632720677bee4c3cbb614031742d44dd27169","unresolved":false,"context_lines":[{"line_number":6163,"context_line":"        if max_count is not None:"},{"line_number":6164,"context_line":"            results \u003d list(query.limit(max_count))"},{"line_number":6165,"context_line":"        else:"},{"line_number":6166,"context_line":"            results \u003d query.all()"},{"line_number":6167,"context_line":""},{"line_number":6168,"context_line":"        for db_dict in results:"},{"line_number":6169,"context_line":"            count_all +\u003d 1"}],"source_content_type":"text/x-python","patch_set":7,"id":"bae84128_c46760b5","line":6166,"in_reply_to":"bae84128_a1ccce6a","updated":"2016-02-23 16:01:05.000000000","message":"It does feel way too risky like this.\n\nLets just get this in, and see how bad things are in reality.","commit_id":"5964073a7d598b4825fa38cf3fcd6697689c93a5"},{"author":{"_account_id":5511,"name":"Nikola Dipanov","email":"ndipanov@redhat.com","username":"ndipanov"},"change_message_id":"f8db1128d87057563bcc9f1168512b5a9103febd","unresolved":false,"context_lines":[{"line_number":6163,"context_line":"        if max_count is not None:"},{"line_number":6164,"context_line":"            results \u003d list(query.limit(max_count))"},{"line_number":6165,"context_line":"        else:"},{"line_number":6166,"context_line":"            results \u003d query.all()"},{"line_number":6167,"context_line":""},{"line_number":6168,"context_line":"        for db_dict in results:"},{"line_number":6169,"context_line":"            count_all +\u003d 1"}],"source_content_type":"text/x-python","patch_set":7,"id":"7af24918_c2d886f2","line":6166,"in_reply_to":"bae84128_c46760b5","updated":"2016-03-01 17:53:48.000000000","message":"I can drop this now that we always have a max_count no worries","commit_id":"5964073a7d598b4825fa38cf3fcd6697689c93a5"}]}