)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":10263,"name":"Vilobh Meshram","email":"vilobhmm@yahoo-inc.com","username":"vilobhmm"},"change_message_id":"ad677f1ddcb29e71ce393740dfbc7512068d6c5e","unresolved":false,"context_lines":[{"line_number":7,"context_line":"Backlog: A proposal for how to do quotas differently"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"This aims to simplify the quota tracking process and make it more"},{"line_number":10,"context_line":"accurate.  It will accomplish this by associating quota usage with the"},{"line_number":11,"context_line":"consuming resource, and not basing quota checks on what is now"},{"line_number":12,"context_line":"essentially a cache."},{"line_number":13,"context_line":""}],"source_content_type":"text/x-gerrit-commit-message","patch_set":4,"id":"3a29b11f_b344d649","line":10,"updated":"2015-10-22 19:07:06.000000000","message":"nit : extra space","commit_id":"ca749f4a01af859b1d4f202f9359d813e6e8c5b5"}],"specs/backlog/approved/quotas-reimagined.rst":[{"author":{"_account_id":1297,"name":"Joshua Harlow","email":"jxharlow@godaddy.com","username":"harlowja"},"change_message_id":"5619dc081a4c1821f3ace5b73c292d33e2a8a786","unresolved":false,"context_lines":[{"line_number":21,"context_line":""},{"line_number":22,"context_line":"As stated above, quotas often end up being inaccurate in Nova.  There are many"},{"line_number":23,"context_line":"safeguards in place, such as periodic resyncs, to help combat this but it would"},{"line_number":24,"context_line":"be nice to solve this in a more fundamental way."},{"line_number":25,"context_line":""},{"line_number":26,"context_line":"Use Cases"},{"line_number":27,"context_line":"----------"}],"source_content_type":"text/x-rst","patch_set":4,"id":"3a29b11f_331be682","line":24,"updated":"2015-10-22 18:21:34.000000000","message":"+10","commit_id":"ca749f4a01af859b1d4f202f9359d813e6e8c5b5"},{"author":{"_account_id":10263,"name":"Vilobh Meshram","email":"vilobhmm@yahoo-inc.com","username":"vilobhmm"},"change_message_id":"ad677f1ddcb29e71ce393740dfbc7512068d6c5e","unresolved":false,"context_lines":[{"line_number":41,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":42,"context_line":""},{"line_number":43,"context_line":"Right now quotas are tracked as a single number, a hopefully up to date count"},{"line_number":44,"context_line":"of resources in use.  Updates are applied by creating deltas as part of a quota"},{"line_number":45,"context_line":"check when adding/removing resources and then commiting them at some later"},{"line_number":46,"context_line":"point.  This is problematic because quota updates are disconnected from the"},{"line_number":47,"context_line":"resource being created/deleted.  If an operation is interrupted there is no way"}],"source_content_type":"text/x-rst","patch_set":4,"id":"3a29b11f_b3a616ac","line":44,"updated":"2015-10-22 19:07:06.000000000","message":"not : two spaces","commit_id":"ca749f4a01af859b1d4f202f9359d813e6e8c5b5"},{"author":{"_account_id":10263,"name":"Vilobh Meshram","email":"vilobhmm@yahoo-inc.com","username":"vilobhmm"},"change_message_id":"ad677f1ddcb29e71ce393740dfbc7512068d6c5e","unresolved":false,"context_lines":[{"line_number":45,"context_line":"check when adding/removing resources and then commiting them at some later"},{"line_number":46,"context_line":"point.  This is problematic because quota updates are disconnected from the"},{"line_number":47,"context_line":"resource being created/deleted.  If an operation is interrupted there is no way"},{"line_number":48,"context_line":"to find the associated delta and ensure it is cleaned up.  There is an expire"},{"line_number":49,"context_line":"time which does provides some mechanism for eventual clean up."},{"line_number":50,"context_line":""},{"line_number":51,"context_line":"Rather than tracking a single number and relying on delta updates to keep it up"}],"source_content_type":"text/x-rst","patch_set":4,"id":"3a29b11f_b3cdb667","line":48,"updated":"2015-10-22 19:07:06.000000000","message":"Can you please elaborate here ?","commit_id":"ca749f4a01af859b1d4f202f9359d813e6e8c5b5"},{"author":{"_account_id":1297,"name":"Joshua Harlow","email":"jxharlow@godaddy.com","username":"harlowja"},"change_message_id":"5619dc081a4c1821f3ace5b73c292d33e2a8a786","unresolved":false,"context_lines":[{"line_number":46,"context_line":"point.  This is problematic because quota updates are disconnected from the"},{"line_number":47,"context_line":"resource being created/deleted.  If an operation is interrupted there is no way"},{"line_number":48,"context_line":"to find the associated delta and ensure it is cleaned up.  There is an expire"},{"line_number":49,"context_line":"time which does provides some mechanism for eventual clean up."},{"line_number":50,"context_line":""},{"line_number":51,"context_line":"Rather than tracking a single number and relying on delta updates to keep it up"},{"line_number":52,"context_line":"to date we should have a small quota object for each resource that uses quota."}],"source_content_type":"text/x-rst","patch_set":4,"id":"3a29b11f_9300ba03","line":49,"updated":"2015-10-22 18:21:34.000000000","message":"Except if the expiry time is so short that the resource is still building, then the expiry time is actually a bad thing right? I\u0027ve always wondered who actually depends on the expiry time, and how has that worked out for them?","commit_id":"ca749f4a01af859b1d4f202f9359d813e6e8c5b5"},{"author":{"_account_id":10263,"name":"Vilobh Meshram","email":"vilobhmm@yahoo-inc.com","username":"vilobhmm"},"change_message_id":"ad677f1ddcb29e71ce393740dfbc7512068d6c5e","unresolved":false,"context_lines":[{"line_number":58,"context_line":"interruption during that we can easily audit usage records for all resources."},{"line_number":59,"context_line":""},{"line_number":60,"context_line":"Additionally we should get rid of the quota_usages table and base quota"},{"line_number":61,"context_line":"decisions on usage information gathered by summing all current usage objects."},{"line_number":62,"context_line":"This eliminates one spot where data can drift from actual usage."},{"line_number":63,"context_line":""},{"line_number":64,"context_line":"Quota checks for adding resources would be modified to doing a usage addition"}],"source_content_type":"text/x-rst","patch_set":4,"id":"3a29b11f_f381eec3","line":61,"updated":"2015-10-22 19:07:06.000000000","message":"Are we saying that we completely getting rid of hard-limit in nova.quotas or planning to make it an attribute of the respective resource object.\n\nWill we have hardlimit in nova.quotas or be an attribute of resouce ?\n\nLets say admin spin\u0027s up 2 instances of small flavor (2VCPU-4GRAM-10GDisk)\n\ndoes that mean \n\nresource.ram \u003d 8G (usage) \nresource.vcpu \u003d 4 vcpu (usage)\n\n\nSo sum of current usage \u003d 8G (RAM)\n\nLets say admin issues nova quota-update to increase the quota for a project for RAM to be 40G (earlier was 30G) so will the usages now track against the hard-limit of 40G.","commit_id":"ca749f4a01af859b1d4f202f9359d813e6e8c5b5"},{"author":{"_account_id":10263,"name":"Vilobh Meshram","email":"vilobhmm@yahoo-inc.com","username":"vilobhmm"},"change_message_id":"ad677f1ddcb29e71ce393740dfbc7512068d6c5e","unresolved":false,"context_lines":[{"line_number":65,"context_line":"and then checking the usage against the quota.  If it was over at that point"},{"line_number":66,"context_line":"the check fails and the usage is removed, if it succeeds then resource creation"},{"line_number":67,"context_line":"can continue.  This check would need to be atomic using something like a"},{"line_number":68,"context_line":"compare-and-swap pattern."},{"line_number":69,"context_line":""},{"line_number":70,"context_line":"Alternatives"},{"line_number":71,"context_line":"------------"}],"source_content_type":"text/x-rst","patch_set":4,"id":"3a29b11f_d35cf24c","line":68,"updated":"2015-10-22 19:07:06.000000000","message":"So if I understand correctly,\n\n1. Boot request -\u003e \n\n2. Quota Validation (as part of checking the respective resource usage options); Lets say the request is within limits -\u003e  \n\n3. Boot instance (will go to compute) -\u003e \n\n    if success ( commit) -\u003e \n  \n         resource.save() {to reflect objects state in db} ; \n\n    else (rollback):\n     \n         do not save anything in db for this resource object\n\nIf the above understanding is correct can we guarantee correctness of quota usage in case of concurrent boot requests ?","commit_id":"ca749f4a01af859b1d4f202f9359d813e6e8c5b5"},{"author":{"_account_id":1297,"name":"Joshua Harlow","email":"jxharlow@godaddy.com","username":"harlowja"},"change_message_id":"72f14a8c7c8de0f1505e10f0072e48b8aa18a26b","unresolved":false,"context_lines":[{"line_number":65,"context_line":"and then checking the usage against the quota.  If it was over at that point"},{"line_number":66,"context_line":"the check fails and the usage is removed, if it succeeds then resource creation"},{"line_number":67,"context_line":"can continue.  This check would need to be atomic using something like a"},{"line_number":68,"context_line":"compare-and-swap pattern."},{"line_number":69,"context_line":""},{"line_number":70,"context_line":"Alternatives"},{"line_number":71,"context_line":"------------"}],"source_content_type":"text/x-rst","patch_set":4,"id":"3a29b11f_4df11a94","line":68,"in_reply_to":"3a29b11f_d35cf24c","updated":"2015-10-22 23:30:45.000000000","message":"I thought it was:\n\n1. Boot request\n\n  with atomic-compare-and-update/select-for-update/other:\n      2. Quota validation\n      3. Quota commit\n\n4. Schedule instance -\u003e scheduler\n\n5. Boot instance -\u003e compute\n\nThat way #2 and #3 are in one place (instead of being in 2 places), and happen before further work...","commit_id":"ca749f4a01af859b1d4f202f9359d813e6e8c5b5"},{"author":{"_account_id":9664,"name":"amrith","email":"amrith.kumar@gmail.com","username":"amrith"},"change_message_id":"e445dd544c4396bfd8662bee9d3c042daf0ad873","unresolved":false,"context_lines":[{"line_number":65,"context_line":"and then checking the usage against the quota.  If it was over at that point"},{"line_number":66,"context_line":"the check fails and the usage is removed, if it succeeds then resource creation"},{"line_number":67,"context_line":"can continue.  This check would need to be atomic using something like a"},{"line_number":68,"context_line":"compare-and-swap pattern."},{"line_number":69,"context_line":""},{"line_number":70,"context_line":"Alternatives"},{"line_number":71,"context_line":"------------"}],"source_content_type":"text/x-rst","patch_set":4,"id":"ba8a016a_9dc0722c","line":68,"in_reply_to":"3a29b11f_d35cf24c","updated":"2015-11-20 18:53:56.000000000","message":"So long as quota validation is atomic and validation records the reservation in an atomic operation, YES!\n\nSpecifically responding to Joshua Harlow, Item #3 is Quota Reservation not Commit.\n\nThe flow would be, I think,\n\n1. Boot Request\n\n2. Atomic Validation and reservation\n\n    - Validation\n    \n    - Reservation\n\n3. Actually Boot\n\n4. Convert Reservation to Commitment on Success.\n\nDelete Reservation on failure.","commit_id":"ca749f4a01af859b1d4f202f9359d813e6e8c5b5"},{"author":{"_account_id":10263,"name":"Vilobh Meshram","email":"vilobhmm@yahoo-inc.com","username":"vilobhmm"},"change_message_id":"ad677f1ddcb29e71ce393740dfbc7512068d6c5e","unresolved":false,"context_lines":[{"line_number":71,"context_line":"------------"},{"line_number":72,"context_line":""},{"line_number":73,"context_line":"We could keep things as they are and continue to track down the edge cases"},{"line_number":74,"context_line":"where quotas are getting out of sync.  Or refresh quotas more continously,"},{"line_number":75,"context_line":"which starts to look like this proposal."},{"line_number":76,"context_line":""},{"line_number":77,"context_line":"Data model impact"}],"source_content_type":"text/x-rst","patch_set":4,"id":"3a29b11f_9352fa33","line":74,"updated":"2015-10-22 19:07:06.000000000","message":"nit : Two spaces","commit_id":"ca749f4a01af859b1d4f202f9359d813e6e8c5b5"},{"author":{"_account_id":10263,"name":"Vilobh Meshram","email":"vilobhmm@yahoo-inc.com","username":"vilobhmm"},"change_message_id":"ad677f1ddcb29e71ce393740dfbc7512068d6c5e","unresolved":false,"context_lines":[{"line_number":77,"context_line":"Data model impact"},{"line_number":78,"context_line":"-----------------"},{"line_number":79,"context_line":""},{"line_number":80,"context_line":"A new table will be added to store quota usage objects. It will have the"},{"line_number":81,"context_line":"following columns: created_at, updated_at, deleted_at, id, uuid, project_id,"},{"line_number":82,"context_line":"user_id, resource, usage, instance_uuid, deleted."},{"line_number":83,"context_line":""}],"source_content_type":"text/x-rst","patch_set":4,"id":"3a29b11f_b9e0372a","line":80,"updated":"2015-10-22 19:07:06.000000000","message":"Why not include these details is nova.quotas and why we need a separate table ?","commit_id":"ca749f4a01af859b1d4f202f9359d813e6e8c5b5"},{"author":{"_account_id":10263,"name":"Vilobh Meshram","email":"vilobhmm@yahoo-inc.com","username":"vilobhmm"},"change_message_id":"ad677f1ddcb29e71ce393740dfbc7512068d6c5e","unresolved":false,"context_lines":[{"line_number":79,"context_line":""},{"line_number":80,"context_line":"A new table will be added to store quota usage objects. It will have the"},{"line_number":81,"context_line":"following columns: created_at, updated_at, deleted_at, id, uuid, project_id,"},{"line_number":82,"context_line":"user_id, resource, usage, instance_uuid, deleted."},{"line_number":83,"context_line":""},{"line_number":84,"context_line":"Each instance will have quota usage objects associated with it that represent"},{"line_number":85,"context_line":"the resources it is consuming. Quota usage for a project/user will be"}],"source_content_type":"text/x-rst","patch_set":4,"id":"3a29b11f_160b3825","line":82,"updated":"2015-10-22 19:07:06.000000000","message":"May be creating a table format is easy for readers than simply including text IMHO. As how you might see desc \u003ctablename\u003e output. But its up to you not mandatory.","commit_id":"ca749f4a01af859b1d4f202f9359d813e6e8c5b5"},{"author":{"_account_id":1297,"name":"Joshua Harlow","email":"jxharlow@godaddy.com","username":"harlowja"},"change_message_id":"f99d645f9a55bfcb7a77eabf30cf8dbe04f42f28","unresolved":false,"context_lines":[{"line_number":79,"context_line":""},{"line_number":80,"context_line":"A new table will be added to store quota usage objects. It will have the"},{"line_number":81,"context_line":"following columns: created_at, updated_at, deleted_at, id, uuid, project_id,"},{"line_number":82,"context_line":"user_id, resource, usage, instance_uuid, deleted."},{"line_number":83,"context_line":""},{"line_number":84,"context_line":"Each instance will have quota usage objects associated with it that represent"},{"line_number":85,"context_line":"the resources it is consuming. Quota usage for a project/user will be"}],"source_content_type":"text/x-rst","patch_set":4,"id":"3a29b11f_ad6b0e1d","line":82,"in_reply_to":"3a29b11f_160b3825","updated":"2015-10-22 23:32:35.000000000","message":"http://truben.no/table/ can make nice RST tables with a nice UI. perhaps use it..","commit_id":"ca749f4a01af859b1d4f202f9359d813e6e8c5b5"},{"author":{"_account_id":10263,"name":"Vilobh Meshram","email":"vilobhmm@yahoo-inc.com","username":"vilobhmm"},"change_message_id":"ad677f1ddcb29e71ce393740dfbc7512068d6c5e","unresolved":false,"context_lines":[{"line_number":81,"context_line":"following columns: created_at, updated_at, deleted_at, id, uuid, project_id,"},{"line_number":82,"context_line":"user_id, resource, usage, instance_uuid, deleted."},{"line_number":83,"context_line":""},{"line_number":84,"context_line":"Each instance will have quota usage objects associated with it that represent"},{"line_number":85,"context_line":"the resources it is consuming. Quota usage for a project/user will be"},{"line_number":86,"context_line":"calculated by summing the resources associated with them."},{"line_number":87,"context_line":""}],"source_content_type":"text/x-rst","patch_set":4,"id":"3a29b11f_b94ef738","line":84,"updated":"2015-10-22 19:07:06.000000000","message":"How will each instance keep track of these quota objects ? Can you please add more details here ? I understand the instance_uuid in the new table you are proposing will be referring to each instance but needs to be spelled out IMHO.","commit_id":"ca749f4a01af859b1d4f202f9359d813e6e8c5b5"},{"author":{"_account_id":9664,"name":"amrith","email":"amrith.kumar@gmail.com","username":"amrith"},"change_message_id":"e445dd544c4396bfd8662bee9d3c042daf0ad873","unresolved":false,"context_lines":[{"line_number":88,"context_line":"REST API impact"},{"line_number":89,"context_line":"---------------"},{"line_number":90,"context_line":""},{"line_number":91,"context_line":"None"},{"line_number":92,"context_line":""},{"line_number":93,"context_line":"Security impact"},{"line_number":94,"context_line":"---------------"}],"source_content_type":"text/x-rst","patch_set":4,"id":"ba8a016a_ddf59acb","line":91,"updated":"2015-11-20 18:53:56.000000000","message":"Nice!","commit_id":"ca749f4a01af859b1d4f202f9359d813e6e8c5b5"},{"author":{"_account_id":10263,"name":"Vilobh Meshram","email":"vilobhmm@yahoo-inc.com","username":"vilobhmm"},"change_message_id":"ad677f1ddcb29e71ce393740dfbc7512068d6c5e","unresolved":false,"context_lines":[{"line_number":109,"context_line":"------------------"},{"line_number":110,"context_line":""},{"line_number":111,"context_line":"There could be a slight performance impact incurred by needing to sum the"},{"line_number":112,"context_line":"usages each time a quota is checked.  But given a simple data model this should"},{"line_number":113,"context_line":"be fast in practice."},{"line_number":114,"context_line":""},{"line_number":115,"context_line":"It should be noted that in the case where there is a lot of contention, the"}],"source_content_type":"text/x-rst","patch_set":4,"id":"3a29b11f_d9e5f308","line":112,"updated":"2015-10-22 19:07:06.000000000","message":"Quota be checked mostly in case of write ops right ?\nquota-update, quota-delete, instance creation/deletion ? Or are there any other ops that might need it. Or does this statement generalizes to all the operations (even read quota-show, etc) as well ?","commit_id":"ca749f4a01af859b1d4f202f9359d813e6e8c5b5"},{"author":{"_account_id":1297,"name":"Joshua Harlow","email":"jxharlow@godaddy.com","username":"harlowja"},"change_message_id":"5619dc081a4c1821f3ace5b73c292d33e2a8a786","unresolved":false,"context_lines":[{"line_number":110,"context_line":""},{"line_number":111,"context_line":"There could be a slight performance impact incurred by needing to sum the"},{"line_number":112,"context_line":"usages each time a quota is checked.  But given a simple data model this should"},{"line_number":113,"context_line":"be fast in practice."},{"line_number":114,"context_line":""},{"line_number":115,"context_line":"It should be noted that in the case where there is a lot of contention, the"},{"line_number":116,"context_line":"back off approach of deleting quota objects when calculated usage is over the"}],"source_content_type":"text/x-rst","patch_set":4,"id":"3a29b11f_162258a3","line":113,"updated":"2015-10-22 18:21:34.000000000","message":"Hopefully the sum() process and such can actually be handled by the database entirely, so that should be ok IMHO.","commit_id":"ca749f4a01af859b1d4f202f9359d813e6e8c5b5"},{"author":{"_account_id":10263,"name":"Vilobh Meshram","email":"vilobhmm@yahoo-inc.com","username":"vilobhmm"},"change_message_id":"ad677f1ddcb29e71ce393740dfbc7512068d6c5e","unresolved":false,"context_lines":[{"line_number":113,"context_line":"be fast in practice."},{"line_number":114,"context_line":""},{"line_number":115,"context_line":"It should be noted that in the case where there is a lot of contention, the"},{"line_number":116,"context_line":"back off approach of deleting quota objects when calculated usage is over the"},{"line_number":117,"context_line":"quota limit could cause a user to experience a lot of build failures when they"},{"line_number":118,"context_line":"get close to the quota limit."},{"line_number":119,"context_line":""}],"source_content_type":"text/x-rst","patch_set":4,"id":"3a29b11f_d993b359","line":116,"updated":"2015-10-22 19:07:06.000000000","message":"Will this situation arise in first case ? Can you explain how ?","commit_id":"ca749f4a01af859b1d4f202f9359d813e6e8c5b5"}],"specs/backlog/quotas-reimagined.rst":[{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"466d4e18dd08f1ee2994c803672817526fe55a82","unresolved":false,"context_lines":[{"line_number":46,"context_line":"point.  This is problematic because quota updates are disconnected from the"},{"line_number":47,"context_line":"resource being created/deleted.  If an operation is interrupted there is no way"},{"line_number":48,"context_line":"to find the associated delta and ensure it is cleaned up.  There is an expire"},{"line_number":49,"context_line":"time which does provides some mechanism for eventual clean up."},{"line_number":50,"context_line":""},{"line_number":51,"context_line":"Rather than tracking a single number and relying on delta updates to keep it up"},{"line_number":52,"context_line":"to date we should have a small quota object for each resource that uses quota."}],"source_content_type":"text/x-rst","patch_set":2,"id":"7a2fa921_39a8e71f","line":49,"updated":"2015-10-08 15:53:56.000000000","message":"\"eventually consistent\" eh ? ;-)","commit_id":"3835b779286899a580b99f00122c9459c7183f77"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"466d4e18dd08f1ee2994c803672817526fe55a82","unresolved":false,"context_lines":[{"line_number":49,"context_line":"time which does provides some mechanism for eventual clean up."},{"line_number":50,"context_line":""},{"line_number":51,"context_line":"Rather than tracking a single number and relying on delta updates to keep it up"},{"line_number":52,"context_line":"to date we should have a small quota object for each resource that uses quota."},{"line_number":53,"context_line":"This could just be a simple db row, similar to \u0027reservations\u0027, with the"},{"line_number":54,"context_line":"addition of a reference to the object using the quota.  By associating the"},{"line_number":55,"context_line":"quota with the object using it we can at any point look up the usage"}],"source_content_type":"text/x-rst","patch_set":2,"id":"7a2fa921_04c9fe25","line":52,"range":{"start_line":52,"start_character":31,"end_line":52,"end_character":43},"updated":"2015-10-08 15:53:56.000000000","message":"So you would see a Quota object nested to a specific resource ?\n\nWhat about a QuotaMixin rather provided to the resource we want to calculate ? It would just provide the fields we want and the actions we would like (create, update, save, get_by_project) nope?","commit_id":"3835b779286899a580b99f00122c9459c7183f77"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"0340dcd9ea062760867e43c019084faea622fc63","unresolved":false,"context_lines":[{"line_number":49,"context_line":"time which does provides some mechanism for eventual clean up."},{"line_number":50,"context_line":""},{"line_number":51,"context_line":"Rather than tracking a single number and relying on delta updates to keep it up"},{"line_number":52,"context_line":"to date we should have a small quota object for each resource that uses quota."},{"line_number":53,"context_line":"This could just be a simple db row, similar to \u0027reservations\u0027, with the"},{"line_number":54,"context_line":"addition of a reference to the object using the quota.  By associating the"},{"line_number":55,"context_line":"quota with the object using it we can at any point look up the usage"}],"source_content_type":"text/x-rst","patch_set":2,"id":"5a2ca52d_469a3216","line":52,"range":{"start_line":52,"start_character":31,"end_line":52,"end_character":43},"in_reply_to":"7a2fa921_04c9fe25","updated":"2015-10-13 14:40:56.000000000","message":"To be clear, that point is an implementation detail that we can leave for later.","commit_id":"3835b779286899a580b99f00122c9459c7183f77"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"466d4e18dd08f1ee2994c803672817526fe55a82","unresolved":false,"context_lines":[{"line_number":65,"context_line":"and then checking the usage against the quota.  If it was over at that point"},{"line_number":66,"context_line":"the check fails and the usage is removed, if it succeeds then resource creation"},{"line_number":67,"context_line":"can continue.  This check would need to be atomic using something like a"},{"line_number":68,"context_line":"compare-and-swap pattern."},{"line_number":69,"context_line":""},{"line_number":70,"context_line":"Alternatives"},{"line_number":71,"context_line":"------------"}],"source_content_type":"text/x-rst","patch_set":2,"id":"7a2fa921_c46a26ed","line":68,"updated":"2015-10-08 15:53:56.000000000","message":"I\u0027d be interested in seeing how we could lookup between the old and the new style. Given that we\u0027d like to have incremental updates, I guess we probably need to have some way to look at the old model before removing it.","commit_id":"3835b779286899a580b99f00122c9459c7183f77"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"9b9ea47c5ee25d4536eaedfaaff08b36d9638e73","unresolved":false,"context_lines":[{"line_number":65,"context_line":"and then checking the usage against the quota.  If it was over at that point"},{"line_number":66,"context_line":"the check fails and the usage is removed, if it succeeds then resource creation"},{"line_number":67,"context_line":"can continue.  This check would need to be atomic using something like a"},{"line_number":68,"context_line":"compare-and-swap pattern."},{"line_number":69,"context_line":""},{"line_number":70,"context_line":"Alternatives"},{"line_number":71,"context_line":"------------"}],"source_content_type":"text/x-rst","patch_set":2,"id":"5a2ca52d_9702b63d","line":68,"in_reply_to":"5a2ca52d_26ba96e2","updated":"2015-10-13 15:29:26.000000000","message":"Agreed with your proposal since it\u0027s a data migration concept we agreed, just wanting to document it here, given it should have some data model impact that needs to be documented too.","commit_id":"3835b779286899a580b99f00122c9459c7183f77"},{"author":{"_account_id":782,"name":"John Garbutt","email":"john@johngarbutt.com","username":"johngarbutt"},"change_message_id":"53017ce1d5eb574bca88c933568922034213c85d","unresolved":false,"context_lines":[{"line_number":65,"context_line":"and then checking the usage against the quota.  If it was over at that point"},{"line_number":66,"context_line":"the check fails and the usage is removed, if it succeeds then resource creation"},{"line_number":67,"context_line":"can continue.  This check would need to be atomic using something like a"},{"line_number":68,"context_line":"compare-and-swap pattern."},{"line_number":69,"context_line":""},{"line_number":70,"context_line":"Alternatives"},{"line_number":71,"context_line":"------------"}],"source_content_type":"text/x-rst","patch_set":2,"id":"5a2ca52d_26ba96e2","line":68,"in_reply_to":"7a2fa921_c46a26ed","updated":"2015-10-13 14:43:31.000000000","message":"So, I think the transition can be simpler actually:\n\nWhen you add the quota, if the new quota is empty, but the old quota location has some info, re-generate new usage records, and delete the old quota data once thats complete, then continue as if its just new quota data.\n\nWe would also need a nova-manage command to force that, so we can eventually drop the old table, and only use the new table.","commit_id":"3835b779286899a580b99f00122c9459c7183f77"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"0340dcd9ea062760867e43c019084faea622fc63","unresolved":false,"context_lines":[{"line_number":77,"context_line":"Data model impact"},{"line_number":78,"context_line":"-----------------"},{"line_number":79,"context_line":""},{"line_number":80,"context_line":"Waiting for feedback before proceeding."},{"line_number":81,"context_line":""},{"line_number":82,"context_line":"REST API impact"},{"line_number":83,"context_line":"---------------"}],"source_content_type":"text/x-rst","patch_set":2,"id":"5a2ca52d_a602263d","line":80,"updated":"2015-10-13 14:40:56.000000000","message":"I\u0027m also considering how nested resources could match in a cells v2 world, where resources are either global or per cell. Does that mean we implicitely assess that eg. quotas about instances are per cell, or would we want to have quotas at the API level ? I guess operators would like to see global quotas...\n\nIf so, that would probably require a bit more logic about how we map between a quota object and a resource IMHO.","commit_id":"3835b779286899a580b99f00122c9459c7183f77"},{"author":{"_account_id":782,"name":"John Garbutt","email":"john@johngarbutt.com","username":"johngarbutt"},"change_message_id":"53017ce1d5eb574bca88c933568922034213c85d","unresolved":false,"context_lines":[{"line_number":77,"context_line":"Data model impact"},{"line_number":78,"context_line":"-----------------"},{"line_number":79,"context_line":""},{"line_number":80,"context_line":"Waiting for feedback before proceeding."},{"line_number":81,"context_line":""},{"line_number":82,"context_line":"REST API impact"},{"line_number":83,"context_line":"---------------"}],"source_content_type":"text/x-rst","patch_set":2,"id":"5a2ca52d_66c88e75","line":80,"range":{"start_line":80,"start_character":0,"end_line":80,"end_character":7},"updated":"2015-10-13 14:43:31.000000000","message":"Need to add the new object model in here.","commit_id":"3835b779286899a580b99f00122c9459c7183f77"},{"author":{"_account_id":4690,"name":"melanie witt","display_name":"melwitt","email":"melwittt@gmail.com","username":"melwitt"},"change_message_id":"c8f372b49ea22002549436c1685f577a110c17c4","unresolved":false,"context_lines":[{"line_number":77,"context_line":"Data model impact"},{"line_number":78,"context_line":"-----------------"},{"line_number":79,"context_line":""},{"line_number":80,"context_line":"Waiting for feedback before proceeding."},{"line_number":81,"context_line":""},{"line_number":82,"context_line":"REST API impact"},{"line_number":83,"context_line":"---------------"}],"source_content_type":"text/x-rst","patch_set":2,"id":"3a29b11f_5208fc27","line":80,"in_reply_to":"5a2ca52d_a602263d","updated":"2015-10-22 06:01:26.000000000","message":"On cells, I would think I\u0027d want to have quotas at the API level. If we do that, we won\u0027t be able to have FK constraint between quota objects and instances though.\n\nI\u0027m imagining if a request comes in from a user/project, to check if they have quota available, if quotas are not global, we would have to pull quota objects from all cells to decide whether to allow the request.","commit_id":"3835b779286899a580b99f00122c9459c7183f77"},{"author":{"_account_id":782,"name":"John Garbutt","email":"john@johngarbutt.com","username":"johngarbutt"},"change_message_id":"53017ce1d5eb574bca88c933568922034213c85d","unresolved":false,"context_lines":[{"line_number":105,"context_line":"There could be a slight performance impact incurred by needing to sum the"},{"line_number":106,"context_line":"usages each time a quota is checked.  But given a simple data model this should"},{"line_number":107,"context_line":"be fast in practice."},{"line_number":108,"context_line":""},{"line_number":109,"context_line":"Other deployer impact"},{"line_number":110,"context_line":"---------------------"},{"line_number":111,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"5a2ca52d_66aace1b","line":108,"updated":"2015-10-13 14:43:31.000000000","message":"We should discuss the case where there is lots of contention, and the back off approach above can cause you to fail lots of builds when you get very close to the quota limit.\n\nFor me this is an acceptable trade-off, but we should raise that explicitly, I feel.","commit_id":"3835b779286899a580b99f00122c9459c7183f77"},{"author":{"_account_id":782,"name":"John Garbutt","email":"john@johngarbutt.com","username":"johngarbutt"},"change_message_id":"53017ce1d5eb574bca88c933568922034213c85d","unresolved":false,"context_lines":[{"line_number":109,"context_line":"Other deployer impact"},{"line_number":110,"context_line":"---------------------"},{"line_number":111,"context_line":""},{"line_number":112,"context_line":"None"},{"line_number":113,"context_line":""},{"line_number":114,"context_line":"Developer impact"},{"line_number":115,"context_line":"----------------"}],"source_content_type":"text/x-rst","patch_set":2,"id":"5a2ca52d_264a9684","line":112,"updated":"2015-10-13 14:43:31.000000000","message":"Need to cover the upgrade impact, and the extra nova-manage command to force the data migration.","commit_id":"3835b779286899a580b99f00122c9459c7183f77"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"9b9ea47c5ee25d4536eaedfaaff08b36d9638e73","unresolved":false,"context_lines":[{"line_number":109,"context_line":"Other deployer impact"},{"line_number":110,"context_line":"---------------------"},{"line_number":111,"context_line":""},{"line_number":112,"context_line":"None"},{"line_number":113,"context_line":""},{"line_number":114,"context_line":"Developer impact"},{"line_number":115,"context_line":"----------------"}],"source_content_type":"text/x-rst","patch_set":2,"id":"5a2ca52d_978956cd","line":112,"in_reply_to":"5a2ca52d_264a9684","updated":"2015-10-13 15:29:26.000000000","message":"++","commit_id":"3835b779286899a580b99f00122c9459c7183f77"}],"specs/liberty/approved/quotas-reimagined.rst":[{"author":{"_account_id":1849,"name":"Joe Gordon","email":"joe.gordon0@gmail.com","username":"jogo"},"change_message_id":"6cab0227c9225ed2c4435b6883875b29aa5118bf","unresolved":false,"context_lines":[{"line_number":13,"context_line":"Quotas are notoriously inaccurate in Nova.  They can almost reliably be pushed"},{"line_number":14,"context_line":"out of sync by sequences of rapid builds and deletes.  This is due to how"},{"line_number":15,"context_line":"quotas are tracked currently, so we should consider other methods of tracking"},{"line_number":16,"context_line":"quotas."},{"line_number":17,"context_line":""},{"line_number":18,"context_line":""},{"line_number":19,"context_line":"Problem description"}],"source_content_type":"text/x-rst","patch_set":1,"id":"7a016987_8725045d","line":16,"updated":"2015-05-21 21:46:06.000000000","message":"Note: last time I tried this it didn\u0027t work. What reliably caused failure was doing rapid builds and deletes while restarting nova-compute etc.","commit_id":"7555e891fcd51b88e285c1a32a5233f735363afd"},{"author":{"_account_id":13997,"name":"SHIGEMATSU Mitsuhiro","email":"shigematsu.mitsuhiro@lab.ntt.co.jp","username":"pshige"},"change_message_id":"26eee5430aab4417e9f4f5178fe90d01d6489ea7","unresolved":false,"context_lines":[{"line_number":30,"context_line":"  only a specified amount of resources."},{"line_number":31,"context_line":""},{"line_number":32,"context_line":"* As a user I want quotas to be accurate so that I can fully utilize what has"},{"line_number":33,"context_line":"  been alloted to me."},{"line_number":34,"context_line":""},{"line_number":35,"context_line":"Project Priority"},{"line_number":36,"context_line":"-----------------"}],"source_content_type":"text/x-rst","patch_set":1,"id":"da20952f_ae7d666e","line":33,"updated":"2015-09-06 11:09:14.000000000","message":"s/alloted/allotted/","commit_id":"7555e891fcd51b88e285c1a32a5233f735363afd"},{"author":{"_account_id":1849,"name":"Joe Gordon","email":"joe.gordon0@gmail.com","username":"jogo"},"change_message_id":"6cab0227c9225ed2c4435b6883875b29aa5118bf","unresolved":false,"context_lines":[{"line_number":34,"context_line":""},{"line_number":35,"context_line":"Project Priority"},{"line_number":36,"context_line":"-----------------"},{"line_number":37,"context_line":""},{"line_number":38,"context_line":"Priorities have not been decided for Liberty."},{"line_number":39,"context_line":""},{"line_number":40,"context_line":"Proposed change"}],"source_content_type":"text/x-rst","patch_set":1,"id":"7a016987_a7c00051","line":37,"updated":"2015-05-21 21:46:06.000000000","message":"hopefully we decide this is a priority","commit_id":"7555e891fcd51b88e285c1a32a5233f735363afd"},{"author":{"_account_id":13997,"name":"SHIGEMATSU Mitsuhiro","email":"shigematsu.mitsuhiro@lab.ntt.co.jp","username":"pshige"},"change_message_id":"26eee5430aab4417e9f4f5178fe90d01d6489ea7","unresolved":false,"context_lines":[{"line_number":42,"context_line":""},{"line_number":43,"context_line":"Right now quotas are tracked as a single number, a hopefully up to date count"},{"line_number":44,"context_line":"of resources in use.  Updates are applied by creating deltas as part of a quota"},{"line_number":45,"context_line":"check when adding/removing resources and then commiting them at some later"},{"line_number":46,"context_line":"point.  This is problematic because quota updates are disconnected from the"},{"line_number":47,"context_line":"resource being created/deleted.  If an operation is interrupted there is no way"},{"line_number":48,"context_line":"to find the associated delta and ensure it is cleaned up.  There is an expire"}],"source_content_type":"text/x-rst","patch_set":1,"id":"da20952f_6e942e8a","line":45,"updated":"2015-09-06 11:09:14.000000000","message":"s/commiting/committing/","commit_id":"7555e891fcd51b88e285c1a32a5233f735363afd"},{"author":{"_account_id":1849,"name":"Joe Gordon","email":"joe.gordon0@gmail.com","username":"jogo"},"change_message_id":"6cab0227c9225ed2c4435b6883875b29aa5118bf","unresolved":false,"context_lines":[{"line_number":45,"context_line":"check when adding/removing resources and then commiting them at some later"},{"line_number":46,"context_line":"point.  This is problematic because quota updates are disconnected from the"},{"line_number":47,"context_line":"resource being created/deleted.  If an operation is interrupted there is no way"},{"line_number":48,"context_line":"to find the associated delta and ensure it is cleaned up.  There is an expire"},{"line_number":49,"context_line":"time which does provides some mechanism for eventual clean up."},{"line_number":50,"context_line":""},{"line_number":51,"context_line":"Rather than tracking a single number and relying on delta updates to keep it up"}],"source_content_type":"text/x-rst","patch_set":1,"id":"7a016987_b2a720bc","line":48,"updated":"2015-05-21 21:46:06.000000000","message":"Good summery of the issues","commit_id":"7555e891fcd51b88e285c1a32a5233f735363afd"},{"author":{"_account_id":1297,"name":"Joshua Harlow","email":"jxharlow@godaddy.com","username":"harlowja"},"change_message_id":"00777675f51fc3182e3311f46e1785023ab6f35e","unresolved":false,"context_lines":[{"line_number":46,"context_line":"point.  This is problematic because quota updates are disconnected from the"},{"line_number":47,"context_line":"resource being created/deleted.  If an operation is interrupted there is no way"},{"line_number":48,"context_line":"to find the associated delta and ensure it is cleaned up.  There is an expire"},{"line_number":49,"context_line":"time which does provides some mechanism for eventual clean up."},{"line_number":50,"context_line":""},{"line_number":51,"context_line":"Rather than tracking a single number and relying on delta updates to keep it up"},{"line_number":52,"context_line":"to date we should have a small quota object for each resource that uses quota."}],"source_content_type":"text/x-rst","patch_set":1,"id":"9a0c5dc1_16f89048","line":49,"updated":"2015-05-13 17:47:47.000000000","message":"Does anyone actually use the expiry time? How has that worked out for them? I\u0027d think that finding the right value to set for expiry is likely pretty difficult, u set it to high and you have wasted resources, you set it to low and you reclaim resources that are still \u0027being worked on\u0027...","commit_id":"7555e891fcd51b88e285c1a32a5233f735363afd"},{"author":{"_account_id":9060,"name":"Mike Dorman","email":"dorm@dorm.org","username":"gd-mdorman"},"change_message_id":"bbd9e41d55052f15ea1d22cdf8cada3e388a0dd1","unresolved":false,"context_lines":[{"line_number":46,"context_line":"point.  This is problematic because quota updates are disconnected from the"},{"line_number":47,"context_line":"resource being created/deleted.  If an operation is interrupted there is no way"},{"line_number":48,"context_line":"to find the associated delta and ensure it is cleaned up.  There is an expire"},{"line_number":49,"context_line":"time which does provides some mechanism for eventual clean up."},{"line_number":50,"context_line":""},{"line_number":51,"context_line":"Rather than tracking a single number and relying on delta updates to keep it up"},{"line_number":52,"context_line":"to date we should have a small quota object for each resource that uses quota."}],"source_content_type":"text/x-rst","patch_set":1,"id":"7a016987_a523ff04","line":49,"in_reply_to":"7a016987_09cc3734","updated":"2015-05-14 16:16:51.000000000","message":"We use the max_age/until_refresh combo with some success.  We still end up with quota problems, but not as often as before.  until_refresh makes fixing quota problems somewhat \"self service\", as after so many reservations the quota usages get updated, anyway.\n\nBut, agree that those are workarounds and it would be good if this were fixed at a more fundamental level.","commit_id":"7555e891fcd51b88e285c1a32a5233f735363afd"},{"author":{"_account_id":5441,"name":"Andrew Laski","email":"andrew@lascii.com","username":"alaski"},"change_message_id":"238708e7db3f5a3e1ac9c3d3bb4990dde3f9ef0c","unresolved":false,"context_lines":[{"line_number":46,"context_line":"point.  This is problematic because quota updates are disconnected from the"},{"line_number":47,"context_line":"resource being created/deleted.  If an operation is interrupted there is no way"},{"line_number":48,"context_line":"to find the associated delta and ensure it is cleaned up.  There is an expire"},{"line_number":49,"context_line":"time which does provides some mechanism for eventual clean up."},{"line_number":50,"context_line":""},{"line_number":51,"context_line":"Rather than tracking a single number and relying on delta updates to keep it up"},{"line_number":52,"context_line":"to date we should have a small quota object for each resource that uses quota."}],"source_content_type":"text/x-rst","patch_set":1,"id":"7a016987_09cc3734","line":49,"in_reply_to":"9a0c5dc1_16f89048","updated":"2015-05-14 14:10:47.000000000","message":"It\u0027s used, but I\u0027m not sure how it\u0027s worked out for anyone.","commit_id":"7555e891fcd51b88e285c1a32a5233f735363afd"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"4b38b4df5fc3ec1328ea68dd30d841fee6ffa835","unresolved":false,"context_lines":[{"line_number":47,"context_line":"resource being created/deleted.  If an operation is interrupted there is no way"},{"line_number":48,"context_line":"to find the associated delta and ensure it is cleaned up.  There is an expire"},{"line_number":49,"context_line":"time which does provides some mechanism for eventual clean up."},{"line_number":50,"context_line":""},{"line_number":51,"context_line":"Rather than tracking a single number and relying on delta updates to keep it up"},{"line_number":52,"context_line":"to date we should have a small quota object for each resource that uses quota."},{"line_number":53,"context_line":"This could just be a simple db row, similar to \u0027reservations\u0027, with the"}],"source_content_type":"text/x-rst","patch_set":1,"id":"7a016987_47573e58","line":50,"updated":"2015-05-19 17:33:37.000000000","message":"minor nit, I think the above paragraph is more related on decribing the problem so would be nice to be moved on the Problem description section","commit_id":"7555e891fcd51b88e285c1a32a5233f735363afd"},{"author":{"_account_id":1849,"name":"Joe Gordon","email":"joe.gordon0@gmail.com","username":"jogo"},"change_message_id":"6cab0227c9225ed2c4435b6883875b29aa5118bf","unresolved":false,"context_lines":[{"line_number":47,"context_line":"resource being created/deleted.  If an operation is interrupted there is no way"},{"line_number":48,"context_line":"to find the associated delta and ensure it is cleaned up.  There is an expire"},{"line_number":49,"context_line":"time which does provides some mechanism for eventual clean up."},{"line_number":50,"context_line":""},{"line_number":51,"context_line":"Rather than tracking a single number and relying on delta updates to keep it up"},{"line_number":52,"context_line":"to date we should have a small quota object for each resource that uses quota."},{"line_number":53,"context_line":"This could just be a simple db row, similar to \u0027reservations\u0027, with the"}],"source_content_type":"text/x-rst","patch_set":1,"id":"7a016987_7284080a","line":50,"in_reply_to":"7a016987_47573e58","updated":"2015-05-21 21:46:06.000000000","message":"agreed, it also might me nice to give some more context on why we have the current two phase approach etc.","commit_id":"7555e891fcd51b88e285c1a32a5233f735363afd"},{"author":{"_account_id":782,"name":"John Garbutt","email":"john@johngarbutt.com","username":"johngarbutt"},"change_message_id":"5435ebc99fb42935b806d7142d189ad3d6862e10","unresolved":false,"context_lines":[{"line_number":55,"context_line":"quota with the object using it we can at any point look up the usage"},{"line_number":56,"context_line":"information to update it.  So, for instance, anytime an instance is switched to"},{"line_number":57,"context_line":"the deleted state we can update the usage information.  And if there is an"},{"line_number":58,"context_line":"interruption during that we can easily audit usage records for all resources."},{"line_number":59,"context_line":""},{"line_number":60,"context_line":"Additionally we should get rid of the quota_usages table and base quota"},{"line_number":61,"context_line":"decisions on usage information gathered by summing all current usage objects."}],"source_content_type":"text/x-rst","patch_set":1,"id":"9a0c5dc1_0f088d90","line":58,"updated":"2015-05-13 09:49:26.000000000","message":"I think we need to thinking about what quota the usage relates to, the \"scope\" of the usage record, I guess.\n\nYou could imagine one instance adding several usage records.","commit_id":"7555e891fcd51b88e285c1a32a5233f735363afd"},{"author":{"_account_id":8688,"name":"Alexis Lee","email":"openstack@lxsli.co.uk","username":"lxsli"},"change_message_id":"35b48932e6ad1e80ae8d0bae42dcc0884b261a23","unresolved":false,"context_lines":[{"line_number":55,"context_line":"quota with the object using it we can at any point look up the usage"},{"line_number":56,"context_line":"information to update it.  So, for instance, anytime an instance is switched to"},{"line_number":57,"context_line":"the deleted state we can update the usage information.  And if there is an"},{"line_number":58,"context_line":"interruption during that we can easily audit usage records for all resources."},{"line_number":59,"context_line":""},{"line_number":60,"context_line":"Additionally we should get rid of the quota_usages table and base quota"},{"line_number":61,"context_line":"decisions on usage information gathered by summing all current usage objects."}],"source_content_type":"text/x-rst","patch_set":1,"id":"3a50d1a3_93ee7920","line":58,"in_reply_to":"7a016987_47285e9a","updated":"2015-07-29 10:14:15.000000000","message":"I agree, this should be synchronized with bp resource-objects and bp persistent-resource-claim. This way we can even have NUMA quotas, plus any other crazy resource that comes along.\n\nA quota naturally seems like an Inventory? Just tenant-associated rather than host-associated. It should be simple to reuse the inventory-checking machinery to test whether a quota permits a new request and to update the quota if a claim is made. Then the claim can be against both, so when it is freed, both the quota and host resources are freed.","commit_id":"7555e891fcd51b88e285c1a32a5233f735363afd"},{"author":{"_account_id":7166,"name":"Sylvain Bauza","email":"sbauza@redhat.com","username":"sbauza"},"change_message_id":"4b38b4df5fc3ec1328ea68dd30d841fee6ffa835","unresolved":false,"context_lines":[{"line_number":55,"context_line":"quota with the object using it we can at any point look up the usage"},{"line_number":56,"context_line":"information to update it.  So, for instance, anytime an instance is switched to"},{"line_number":57,"context_line":"the deleted state we can update the usage information.  And if there is an"},{"line_number":58,"context_line":"interruption during that we can easily audit usage records for all resources."},{"line_number":59,"context_line":""},{"line_number":60,"context_line":"Additionally we should get rid of the quota_usages table and base quota"},{"line_number":61,"context_line":"decisions on usage information gathered by summing all current usage objects."}],"source_content_type":"text/x-rst","patch_set":1,"id":"7a016987_47285e9a","line":58,"in_reply_to":"7a016987_c9d5ffb9","updated":"2015-05-19 17:33:37.000000000","message":"IMHO That\u0027s very closely connected to the notion of consumable resource that the scheduler has and where Claims are here for our help.\n\nMaybe a quick look against what we can claim for an instance should just help us thinking that a resource usage is more than just decrementing an integer but rather having a way to describe for each resource type how to increment its usage and how to decrement it.\n\nSince I think it\u0027s very correlated with the blueprint resource-objects I would lovely accept a base object called BaseUsage (or BaseQuota) that would provide the methods :\n\n - set_limit\n - get_limit\n - increment_usage\n - decrement_usage\n - get_usage\n\nI\u0027m not saying that scheduler usage of resources is the same as quotas (even if we have limits for our resources) so it doesn\u0027t maybe need to have the same BaseUsage to be persisted but maybe not :-)","commit_id":"7555e891fcd51b88e285c1a32a5233f735363afd"},{"author":{"_account_id":5441,"name":"Andrew Laski","email":"andrew@lascii.com","username":"alaski"},"change_message_id":"a84ce2d3ddf4021f8713ddfe1ef1488ad396963a","unresolved":false,"context_lines":[{"line_number":55,"context_line":"quota with the object using it we can at any point look up the usage"},{"line_number":56,"context_line":"information to update it.  So, for instance, anytime an instance is switched to"},{"line_number":57,"context_line":"the deleted state we can update the usage information.  And if there is an"},{"line_number":58,"context_line":"interruption during that we can easily audit usage records for all resources."},{"line_number":59,"context_line":""},{"line_number":60,"context_line":"Additionally we should get rid of the quota_usages table and base quota"},{"line_number":61,"context_line":"decisions on usage information gathered by summing all current usage objects."}],"source_content_type":"text/x-rst","patch_set":1,"id":"9a0c5dc1_2c36b252","line":58,"in_reply_to":"9a0c5dc1_0f088d90","updated":"2015-05-13 14:27:13.000000000","message":"Yeah, I should flesh that out.  My thinking is that anything which can be consumed, i.e. decremented or incremented and a quota checked on it, would have a usage record.  That means an instance boot would have multiple usage objects for things like RAM, cores, metadata, injected files, etc...  So the scope of a usage record is a single quota checked item.","commit_id":"7555e891fcd51b88e285c1a32a5233f735363afd"},{"author":{"_account_id":1297,"name":"Joshua Harlow","email":"jxharlow@godaddy.com","username":"harlowja"},"change_message_id":"00777675f51fc3182e3311f46e1785023ab6f35e","unresolved":false,"context_lines":[{"line_number":55,"context_line":"quota with the object using it we can at any point look up the usage"},{"line_number":56,"context_line":"information to update it.  So, for instance, anytime an instance is switched to"},{"line_number":57,"context_line":"the deleted state we can update the usage information.  And if there is an"},{"line_number":58,"context_line":"interruption during that we can easily audit usage records for all resources."},{"line_number":59,"context_line":""},{"line_number":60,"context_line":"Additionally we should get rid of the quota_usages table and base quota"},{"line_number":61,"context_line":"decisions on usage information gathered by summing all current usage objects."}],"source_content_type":"text/x-rst","patch_set":1,"id":"9a0c5dc1_ea41900c","line":58,"in_reply_to":"9a0c5dc1_2c36b252","updated":"2015-05-13 17:47:47.000000000","message":"Perhaps a data model/example data model table here will help people think about this, vs just text.","commit_id":"7555e891fcd51b88e285c1a32a5233f735363afd"},{"author":{"_account_id":5441,"name":"Andrew Laski","email":"andrew@lascii.com","username":"alaski"},"change_message_id":"238708e7db3f5a3e1ac9c3d3bb4990dde3f9ef0c","unresolved":false,"context_lines":[{"line_number":55,"context_line":"quota with the object using it we can at any point look up the usage"},{"line_number":56,"context_line":"information to update it.  So, for instance, anytime an instance is switched to"},{"line_number":57,"context_line":"the deleted state we can update the usage information.  And if there is an"},{"line_number":58,"context_line":"interruption during that we can easily audit usage records for all resources."},{"line_number":59,"context_line":""},{"line_number":60,"context_line":"Additionally we should get rid of the quota_usages table and base quota"},{"line_number":61,"context_line":"decisions on usage information gathered by summing all current usage objects."}],"source_content_type":"text/x-rst","patch_set":1,"id":"7a016987_c9d5ffb9","line":58,"in_reply_to":"9a0c5dc1_ea41900c","updated":"2015-05-14 14:10:47.000000000","message":"Sure.  Just wanted to get past a basic sanity check before taking the time to model it out.","commit_id":"7555e891fcd51b88e285c1a32a5233f735363afd"},{"author":{"_account_id":9060,"name":"Mike Dorman","email":"dorm@dorm.org","username":"gd-mdorman"},"change_message_id":"bbd9e41d55052f15ea1d22cdf8cada3e388a0dd1","unresolved":false,"context_lines":[{"line_number":59,"context_line":""},{"line_number":60,"context_line":"Additionally we should get rid of the quota_usages table and base quota"},{"line_number":61,"context_line":"decisions on usage information gathered by summing all current usage objects."},{"line_number":62,"context_line":"This eliminates one spot where data can drift from actual usage."},{"line_number":63,"context_line":""},{"line_number":64,"context_line":"Quota checks for adding resources would be modified to doing a usage addition"},{"line_number":65,"context_line":"and then checking the usage against the quota.  If it was over at that point"}],"source_content_type":"text/x-rst","patch_set":1,"id":"7a016987_45632b1b","line":62,"updated":"2015-05-14 16:16:51.000000000","message":"How is having the usage record for each quota \"consumer\" different than simply querying the instances table (as an example) to determine the usage? I think I am just missing something here.","commit_id":"7555e891fcd51b88e285c1a32a5233f735363afd"},{"author":{"_account_id":9060,"name":"Mike Dorman","email":"dorm@dorm.org","username":"gd-mdorman"},"change_message_id":"5089669b671f4f576ccde74b46c00a3a19196117","unresolved":false,"context_lines":[{"line_number":59,"context_line":""},{"line_number":60,"context_line":"Additionally we should get rid of the quota_usages table and base quota"},{"line_number":61,"context_line":"decisions on usage information gathered by summing all current usage objects."},{"line_number":62,"context_line":"This eliminates one spot where data can drift from actual usage."},{"line_number":63,"context_line":""},{"line_number":64,"context_line":"Quota checks for adding resources would be modified to doing a usage addition"},{"line_number":65,"context_line":"and then checking the usage against the quota.  If it was over at that point"}],"source_content_type":"text/x-rst","patch_set":1,"id":"7a016987_d8147a45","line":62,"in_reply_to":"7a016987_11679626","updated":"2015-05-15 15:16:39.000000000","message":"Makes sense.  Thanks.","commit_id":"7555e891fcd51b88e285c1a32a5233f735363afd"},{"author":{"_account_id":5441,"name":"Andrew Laski","email":"andrew@lascii.com","username":"alaski"},"change_message_id":"09020dc834b3aeda9f9994f39d373a0a83fc1ad8","unresolved":false,"context_lines":[{"line_number":59,"context_line":""},{"line_number":60,"context_line":"Additionally we should get rid of the quota_usages table and base quota"},{"line_number":61,"context_line":"decisions on usage information gathered by summing all current usage objects."},{"line_number":62,"context_line":"This eliminates one spot where data can drift from actual usage."},{"line_number":63,"context_line":""},{"line_number":64,"context_line":"Quota checks for adding resources would be modified to doing a usage addition"},{"line_number":65,"context_line":"and then checking the usage against the quota.  If it was over at that point"}],"source_content_type":"text/x-rst","patch_set":1,"id":"7a016987_11679626","line":62,"in_reply_to":"7a016987_45632b1b","updated":"2015-05-15 12:50:10.000000000","message":"Primarily a separation of concerns that would allow for optimizations if necessary.  But also storing a small quota usage object separately should allow for faster inserts/selects just based on size and number of indexes already on instance.  It\u0027s not fundamental to the spec though, just something I felt would alleviate performance concerns.","commit_id":"7555e891fcd51b88e285c1a32a5233f735363afd"},{"author":{"_account_id":782,"name":"John Garbutt","email":"john@johngarbutt.com","username":"johngarbutt"},"change_message_id":"5435ebc99fb42935b806d7142d189ad3d6862e10","unresolved":false,"context_lines":[{"line_number":65,"context_line":"and then checking the usage against the quota.  If it was over at that point"},{"line_number":66,"context_line":"the check fails and the usage is removed, if it succeeds then resource creation"},{"line_number":67,"context_line":"can continue.  This check would need to be atomic using something like a"},{"line_number":68,"context_line":"compare-and-swap pattern."},{"line_number":69,"context_line":""},{"line_number":70,"context_line":"Alternatives"},{"line_number":71,"context_line":"------------"}],"source_content_type":"text/x-rst","patch_set":1,"id":"9a0c5dc1_af274111","line":68,"updated":"2015-05-13 09:49:26.000000000","message":"I think a pattern of adding the record, then checking the count is not over the limit, then removing the record if it is over the limit, should help with the race conditions. Now in some cases we revert two requests, where one could have succeeded, but thats the \"safer\" failure. Its optimistic concurrency control, basically.","commit_id":"7555e891fcd51b88e285c1a32a5233f735363afd"},{"author":{"_account_id":9060,"name":"Mike Dorman","email":"dorm@dorm.org","username":"gd-mdorman"},"change_message_id":"bbd9e41d55052f15ea1d22cdf8cada3e388a0dd1","unresolved":false,"context_lines":[{"line_number":65,"context_line":"and then checking the usage against the quota.  If it was over at that point"},{"line_number":66,"context_line":"the check fails and the usage is removed, if it succeeds then resource creation"},{"line_number":67,"context_line":"can continue.  This check would need to be atomic using something like a"},{"line_number":68,"context_line":"compare-and-swap pattern."},{"line_number":69,"context_line":""},{"line_number":70,"context_line":"Alternatives"},{"line_number":71,"context_line":"------------"}],"source_content_type":"text/x-rst","patch_set":1,"id":"7a016987_652ce7ab","line":68,"in_reply_to":"9a0c5dc1_8c735e15","updated":"2015-05-14 16:16:51.000000000","message":"+1 for doing it in such a way that operations aren\u0027t blocked.","commit_id":"7555e891fcd51b88e285c1a32a5233f735363afd"},{"author":{"_account_id":5441,"name":"Andrew Laski","email":"andrew@lascii.com","username":"alaski"},"change_message_id":"a84ce2d3ddf4021f8713ddfe1ef1488ad396963a","unresolved":false,"context_lines":[{"line_number":65,"context_line":"and then checking the usage against the quota.  If it was over at that point"},{"line_number":66,"context_line":"the check fails and the usage is removed, if it succeeds then resource creation"},{"line_number":67,"context_line":"can continue.  This check would need to be atomic using something like a"},{"line_number":68,"context_line":"compare-and-swap pattern."},{"line_number":69,"context_line":""},{"line_number":70,"context_line":"Alternatives"},{"line_number":71,"context_line":"------------"}],"source_content_type":"text/x-rst","patch_set":1,"id":"9a0c5dc1_8c735e15","line":68,"in_reply_to":"9a0c5dc1_af274111","updated":"2015-05-13 14:27:13.000000000","message":"Yeah.  I also just had the idea of doing it as a two phase commit which should allow a second request which could succeed to not get blocked by a request in progress that will fail.","commit_id":"7555e891fcd51b88e285c1a32a5233f735363afd"},{"author":{"_account_id":782,"name":"John Garbutt","email":"john@johngarbutt.com","username":"johngarbutt"},"change_message_id":"5435ebc99fb42935b806d7142d189ad3d6862e10","unresolved":false,"context_lines":[{"line_number":66,"context_line":"the check fails and the usage is removed, if it succeeds then resource creation"},{"line_number":67,"context_line":"can continue.  This check would need to be atomic using something like a"},{"line_number":68,"context_line":"compare-and-swap pattern."},{"line_number":69,"context_line":""},{"line_number":70,"context_line":"Alternatives"},{"line_number":71,"context_line":"------------"},{"line_number":72,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"9a0c5dc1_0f4e4d96","line":69,"updated":"2015-05-13 09:49:26.000000000","message":"Another item here, I think is moving all quota updates to the API layer.\n\nAs there are no reservations, we simply update the quota assuming a success, in the API:\n* on create, always add to quota (or reject request)\n* on delete, always look for record to delete, after the DB is updated to deleting in the API (second delete request would not reduce the quota). Failed deletes should not happen, but no need to consume quota for those failed deletes.\n* on resize up, update quota usage at the first point of the resize\n* on resize down, update on resize_confirm request\n\nThe remaining bit would be some periodic system to check the list of reservations. You have to check the flavor matches the current flavor, and that the instance is not deleted.","commit_id":"7555e891fcd51b88e285c1a32a5233f735363afd"},{"author":{"_account_id":5441,"name":"Andrew Laski","email":"andrew@lascii.com","username":"alaski"},"change_message_id":"a84ce2d3ddf4021f8713ddfe1ef1488ad396963a","unresolved":false,"context_lines":[{"line_number":66,"context_line":"the check fails and the usage is removed, if it succeeds then resource creation"},{"line_number":67,"context_line":"can continue.  This check would need to be atomic using something like a"},{"line_number":68,"context_line":"compare-and-swap pattern."},{"line_number":69,"context_line":""},{"line_number":70,"context_line":"Alternatives"},{"line_number":71,"context_line":"------------"},{"line_number":72,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"9a0c5dc1_ccd716be","line":69,"in_reply_to":"9a0c5dc1_0f4e4d96","updated":"2015-05-13 14:27:13.000000000","message":"I mostly agree with this.  I think there will be times when we need to update quotas based on something happening on a compute, but not in general.","commit_id":"7555e891fcd51b88e285c1a32a5233f735363afd"},{"author":{"_account_id":13997,"name":"SHIGEMATSU Mitsuhiro","email":"shigematsu.mitsuhiro@lab.ntt.co.jp","username":"pshige"},"change_message_id":"26eee5430aab4417e9f4f5178fe90d01d6489ea7","unresolved":false,"context_lines":[{"line_number":71,"context_line":"------------"},{"line_number":72,"context_line":""},{"line_number":73,"context_line":"We could keep things as they are and continue to track down the edge cases"},{"line_number":74,"context_line":"where quotas are getting out of sync.  Or refresh quotas more continously,"},{"line_number":75,"context_line":"which starts to look like this proposal."},{"line_number":76,"context_line":""},{"line_number":77,"context_line":"Data model impact"}],"source_content_type":"text/x-rst","patch_set":1,"id":"da20952f_ced21a5c","line":74,"updated":"2015-09-06 11:09:14.000000000","message":"s/continously/continuously/","commit_id":"7555e891fcd51b88e285c1a32a5233f735363afd"},{"author":{"_account_id":782,"name":"John Garbutt","email":"john@johngarbutt.com","username":"johngarbutt"},"change_message_id":"5435ebc99fb42935b806d7142d189ad3d6862e10","unresolved":false,"context_lines":[{"line_number":77,"context_line":"Data model impact"},{"line_number":78,"context_line":"-----------------"},{"line_number":79,"context_line":""},{"line_number":80,"context_line":"Waiting for feedback before proceeding."},{"line_number":81,"context_line":""},{"line_number":82,"context_line":"REST API impact"},{"line_number":83,"context_line":"---------------"}],"source_content_type":"text/x-rst","patch_set":1,"id":"9a0c5dc1_8f211db7","line":80,"updated":"2015-05-13 09:49:26.000000000","message":"I guess its easier to add a new table for these usage records.\n\nWe should have a nice smooth transition between the systems for a release, then we can drop these older tables (and compat code) next release?","commit_id":"7555e891fcd51b88e285c1a32a5233f735363afd"},{"author":{"_account_id":782,"name":"John Garbutt","email":"john@johngarbutt.com","username":"johngarbutt"},"change_message_id":"5435ebc99fb42935b806d7142d189ad3d6862e10","unresolved":false,"context_lines":[{"line_number":105,"context_line":"There could be a slight performance impact incurred by needing to sum the"},{"line_number":106,"context_line":"usages each time a quota is checked.  But given a simple data model this should"},{"line_number":107,"context_line":"be fast in practice."},{"line_number":108,"context_line":""},{"line_number":109,"context_line":"Other deployer impact"},{"line_number":110,"context_line":"---------------------"},{"line_number":111,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"9a0c5dc1_32005601","line":108,"updated":"2015-05-13 09:49:26.000000000","message":"Not sure its worth pulling out here, but I think we could add a cache of the current list, that gets periodically updated, but that feels like a premature optimisation.\n\nAt least if we have this explicit list, such a cache would now be easier to sync, and we might what the cache stored in memcache rather than a DB, with some expiry on the key, etc, etc.","commit_id":"7555e891fcd51b88e285c1a32a5233f735363afd"},{"author":{"_account_id":1297,"name":"Joshua Harlow","email":"jxharlow@godaddy.com","username":"harlowja"},"change_message_id":"00777675f51fc3182e3311f46e1785023ab6f35e","unresolved":false,"context_lines":[{"line_number":105,"context_line":"There could be a slight performance impact incurred by needing to sum the"},{"line_number":106,"context_line":"usages each time a quota is checked.  But given a simple data model this should"},{"line_number":107,"context_line":"be fast in practice."},{"line_number":108,"context_line":""},{"line_number":109,"context_line":"Other deployer impact"},{"line_number":110,"context_line":"---------------------"},{"line_number":111,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"9a0c5dc1_8a6e9494","line":108,"in_reply_to":"9a0c5dc1_32005601","updated":"2015-05-13 17:47:47.000000000","message":"Imho, get the basics *very* solid here first, before starting to add caches all over the place.","commit_id":"7555e891fcd51b88e285c1a32a5233f735363afd"},{"author":{"_account_id":5441,"name":"Andrew Laski","email":"andrew@lascii.com","username":"alaski"},"change_message_id":"a84ce2d3ddf4021f8713ddfe1ef1488ad396963a","unresolved":false,"context_lines":[{"line_number":105,"context_line":"There could be a slight performance impact incurred by needing to sum the"},{"line_number":106,"context_line":"usages each time a quota is checked.  But given a simple data model this should"},{"line_number":107,"context_line":"be fast in practice."},{"line_number":108,"context_line":""},{"line_number":109,"context_line":"Other deployer impact"},{"line_number":110,"context_line":"---------------------"},{"line_number":111,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"9a0c5dc1_8cf8fe4a","line":108,"in_reply_to":"9a0c5dc1_32005601","updated":"2015-05-13 14:27:13.000000000","message":"Right.  I think a cache may be useful/needed down the road but I also think we can do without it for now.  And doing it outside of a db seems like the thing to do.","commit_id":"7555e891fcd51b88e285c1a32a5233f735363afd"},{"author":{"_account_id":782,"name":"John Garbutt","email":"john@johngarbutt.com","username":"johngarbutt"},"change_message_id":"13f1718a3e727dde981ae5cf28c888c1a9d81c15","unresolved":false,"context_lines":[{"line_number":124,"context_line":"-----------"},{"line_number":125,"context_line":""},{"line_number":126,"context_line":"Primary assignee:"},{"line_number":127,"context_line":"  None"},{"line_number":128,"context_line":""},{"line_number":129,"context_line":"Other contributors:"},{"line_number":130,"context_line":"  None"}],"source_content_type":"text/x-rst","patch_set":1,"id":"9a0c5dc1_f27cfe73","line":127,"updated":"2015-05-13 09:50:39.000000000","message":"we should probably move this to the backlog directory, as an example?","commit_id":"7555e891fcd51b88e285c1a32a5233f735363afd"},{"author":{"_account_id":5441,"name":"Andrew Laski","email":"andrew@lascii.com","username":"alaski"},"change_message_id":"a84ce2d3ddf4021f8713ddfe1ef1488ad396963a","unresolved":false,"context_lines":[{"line_number":124,"context_line":"-----------"},{"line_number":125,"context_line":""},{"line_number":126,"context_line":"Primary assignee:"},{"line_number":127,"context_line":"  None"},{"line_number":128,"context_line":""},{"line_number":129,"context_line":"Other contributors:"},{"line_number":130,"context_line":"  None"}],"source_content_type":"text/x-rst","patch_set":1,"id":"9a0c5dc1_ec789ac6","line":127,"in_reply_to":"9a0c5dc1_f27cfe73","updated":"2015-05-13 14:27:13.000000000","message":"If it gets to a point where we\u0027re ready to commit it and nobody has stepped up to do it, sure.  I would like to see if anyone wants to help out here when it\u0027s discussed at the summit.","commit_id":"7555e891fcd51b88e285c1a32a5233f735363afd"},{"author":{"_account_id":782,"name":"John Garbutt","email":"john@johngarbutt.com","username":"johngarbutt"},"change_message_id":"5435ebc99fb42935b806d7142d189ad3d6862e10","unresolved":false,"context_lines":[{"line_number":142,"context_line":"Testing"},{"line_number":143,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":144,"context_line":""},{"line_number":145,"context_line":"Good unit/integration tests would need to be added before and during this work."},{"line_number":146,"context_line":""},{"line_number":147,"context_line":"Documentation Impact"},{"line_number":148,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":1,"id":"9a0c5dc1_2f891127","line":145,"updated":"2015-05-13 09:49:26.000000000","message":"I think sdage had some ideas about functional tests we can do to try and detect/prevent race conditions.","commit_id":"7555e891fcd51b88e285c1a32a5233f735363afd"},{"author":{"_account_id":1297,"name":"Joshua Harlow","email":"jxharlow@godaddy.com","username":"harlowja"},"change_message_id":"6edfdf78f105c3bb14c852a4c5d3ea62b038652b","unresolved":false,"context_lines":[{"line_number":142,"context_line":"Testing"},{"line_number":143,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":144,"context_line":""},{"line_number":145,"context_line":"Good unit/integration tests would need to be added before and during this work."},{"line_number":146,"context_line":""},{"line_number":147,"context_line":"Documentation Impact"},{"line_number":148,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":1,"id":"7a016987_d3900298","line":145,"in_reply_to":"7a016987_bb48ca2e","updated":"2015-05-14 17:50:59.000000000","message":"Sounds good, let\u0027s see how this goes...","commit_id":"7555e891fcd51b88e285c1a32a5233f735363afd"},{"author":{"_account_id":1297,"name":"Joshua Harlow","email":"jxharlow@godaddy.com","username":"harlowja"},"change_message_id":"00777675f51fc3182e3311f46e1785023ab6f35e","unresolved":false,"context_lines":[{"line_number":142,"context_line":"Testing"},{"line_number":143,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":144,"context_line":""},{"line_number":145,"context_line":"Good unit/integration tests would need to be added before and during this work."},{"line_number":146,"context_line":""},{"line_number":147,"context_line":"Documentation Impact"},{"line_number":148,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":1,"id":"9a0c5dc1_ca124c09","line":145,"in_reply_to":"9a0c5dc1_2f891127","updated":"2015-05-13 17:47:47.000000000","message":"+100 we need to be able to test this as an isolated unit as much as possible (making sure it isn\u0027t racey, works, handles failures correctly...), which makes me think it should be as disconnected from the rest of nova code as possible, which makes me go back to wondering if this should just be something like I described in a big comment at the bottom of https://review.openstack.org/#/c/132127/","commit_id":"7555e891fcd51b88e285c1a32a5233f735363afd"},{"author":{"_account_id":5441,"name":"Andrew Laski","email":"andrew@lascii.com","username":"alaski"},"change_message_id":"238708e7db3f5a3e1ac9c3d3bb4990dde3f9ef0c","unresolved":false,"context_lines":[{"line_number":142,"context_line":"Testing"},{"line_number":143,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":144,"context_line":""},{"line_number":145,"context_line":"Good unit/integration tests would need to be added before and during this work."},{"line_number":146,"context_line":""},{"line_number":147,"context_line":"Documentation Impact"},{"line_number":148,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":1,"id":"7a016987_bb48ca2e","line":145,"in_reply_to":"9a0c5dc1_ca124c09","updated":"2015-05-14 14:10:47.000000000","message":"In the comment at the bottom of that review you mentioned Nova needing to refactor a clean interface that could take advantage of a library.  I think that as part of that we need to clean up the current implementation, and this is a suggestion on how to do that.\n\nAnd I would rather focus on getting quotas to be accurate than work towards being able to drop in a library that doesn\u0027t exist yet.","commit_id":"7555e891fcd51b88e285c1a32a5233f735363afd"}]}