)]}'
{"nova/db/sqlalchemy/migrate_repo/versions/313_add_bw_usage_cache_fk.py":[{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"ed611b5c381616724a4757245dbe15372112a66c","unresolved":false,"context_lines":[{"line_number":25,"context_line":"    bw_usage_cache \u003d Table(\u0027bw_usage_cache\u0027, meta, autoload\u003dTrue)"},{"line_number":26,"context_line":"    instances \u003d Table(\u0027instances\u0027, meta, autoload\u003dTrue)"},{"line_number":27,"context_line":""},{"line_number":28,"context_line":"    fkey \u003d ForeignKeyConstraint(columns\u003d[bw_usage_cache.c.uuid],"},{"line_number":29,"context_line":"                                refcolumns\u003d[instances.c.uuid])"},{"line_number":30,"context_line":"    fkey.create()"}],"source_content_type":"text/x-python","patch_set":1,"id":"9a8ffd7b_27d06053","line":28,"updated":"2015-11-24 15:42:31.000000000","message":"Actually, before we create the fkey, we should probably check for bw_usage_cache entries that have a uuid that refer to an instances entry that no longer exists, and then archive (move to shadow tables) or purge them. I\u0027m actually surprised the turbo-hipster data set doesn\u0027t have a scenario like that.","commit_id":"76d0f97f2c3a358934483689c3ea8687a2a4aafc"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"aebaef1e0e6d2a88df9d709edfba0f445ea34a8c","unresolved":false,"context_lines":[{"line_number":30,"context_line":"    bw_usage_cache \u003d Table(\u0027bw_usage_cache\u0027, meta, autoload\u003dTrue)"},{"line_number":31,"context_line":"    instances \u003d Table(\u0027instances\u0027, meta, autoload\u003dTrue)"},{"line_number":32,"context_line":""},{"line_number":33,"context_line":"    result \u003d (bw_usage_cache.delete()."},{"line_number":34,"context_line":"         where(~bw_usage_cache.c.uuid.in_("},{"line_number":35,"context_line":"             sql.select([instances.c.uuid])))).execute()"},{"line_number":36,"context_line":"    LOG.info(_LI(\"Deleted %d orphaned bw_usage_cache rows\"), result.rowcount)"}],"source_content_type":"text/x-python","patch_set":2,"id":"9a8ffd7b_9477e101","line":33,"updated":"2015-11-24 22:29:00.000000000","message":"We would need a test for this, you\u0027d have to insert some orphaned rows in the pre migration method in the tests, then run the migration and verify those are gone and the fkey was created.\n\nSee _pre_upgrade_293 for example.","commit_id":"3a8c40e89f1aa3823e66726dbb06144302227d12"},{"author":{"_account_id":782,"name":"John Garbutt","email":"john@johngarbutt.com","username":"johngarbutt"},"change_message_id":"45c6f14bae2eea6c97306480c149189dfbc17112","unresolved":false,"context_lines":[{"line_number":26,"context_line":""},{"line_number":27,"context_line":"@contextmanager"},{"line_number":28,"context_line":"def disable_fks(engine):"},{"line_number":29,"context_line":"    active \u003d engine.name in (\u0027mysql\u0027)"},{"line_number":30,"context_line":"    if active:"},{"line_number":31,"context_line":"        engine.execute(\"SET FOREIGN_KEY_CHECKS\u003d0\")"},{"line_number":32,"context_line":"    try:"}],"source_content_type":"text/x-python","patch_set":5,"id":"9a8ffd7b_850f4487","line":29,"updated":"2015-11-26 14:56:15.000000000","message":"does this include mariadb?","commit_id":"424bcca9dcce43bf2f4f18f78459cd75bdc046b7"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"3419e1307ada5e23546e496532955d8643d9d337","unresolved":false,"context_lines":[{"line_number":26,"context_line":""},{"line_number":27,"context_line":"@contextmanager"},{"line_number":28,"context_line":"def disable_fks(engine):"},{"line_number":29,"context_line":"    active \u003d engine.name in (\u0027mysql\u0027)"},{"line_number":30,"context_line":"    if active:"},{"line_number":31,"context_line":"        engine.execute(\"SET FOREIGN_KEY_CHECKS\u003d0\")"},{"line_number":32,"context_line":"    try:"}],"source_content_type":"text/x-python","patch_set":5,"id":"7a740942_e211eeb0","line":29,"in_reply_to":"7a740942_42d8021e","updated":"2015-12-03 15:54:37.000000000","message":"https://github.com/openstack/nova/blob/master/nova/db/sqlalchemy/migrate_repo/versions/298_mysql_extra_specs_binary_collation.py#L17","commit_id":"424bcca9dcce43bf2f4f18f78459cd75bdc046b7"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"4ba933fa234c6a3c73c75a1bfe6387579d2b9469","unresolved":false,"context_lines":[{"line_number":26,"context_line":""},{"line_number":27,"context_line":"@contextmanager"},{"line_number":28,"context_line":"def disable_fks(engine):"},{"line_number":29,"context_line":"    active \u003d engine.name in (\u0027mysql\u0027)"},{"line_number":30,"context_line":"    if active:"},{"line_number":31,"context_line":"        engine.execute(\"SET FOREIGN_KEY_CHECKS\u003d0\")"},{"line_number":32,"context_line":"    try:"}],"source_content_type":"text/x-python","patch_set":5,"id":"7a740942_42d8021e","line":29,"in_reply_to":"9a8ffd7b_850f4487","updated":"2015-12-03 15:53:35.000000000","message":"I think so, this is how it\u0027s always been checked in the DB migrations.","commit_id":"424bcca9dcce43bf2f4f18f78459cd75bdc046b7"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"cb00abe4eba83815769d34ca5cfe8ddee0e0316d","unresolved":false,"context_lines":[{"line_number":60,"context_line":""},{"line_number":61,"context_line":"    rows \u003d kill_orphans(bw_usage_cache, instances)"},{"line_number":62,"context_line":"    if rows \u003e 0:"},{"line_number":63,"context_line":"        LOG.warn(_LI("},{"line_number":64,"context_line":"            \"Post-scan: deleted %d orphaned bw_usage_cache rows!\"), rows)"}],"source_content_type":"text/x-python","patch_set":7,"id":"5a710552_d15d2416","line":63,"updated":"2015-12-16 16:53:25.000000000","message":"This would be _LW.","commit_id":"d265926710832fe1238e3948c32cab7932615c5d"}],"nova/db/sqlalchemy/migrate_repo/versions/331_add_bw_usage_cache_fk.py":[{"author":{"_account_id":6849,"name":"Roman Podoliaka","email":"roman.podoliaka@gmail.com","username":"rpodolyaka"},"change_message_id":"c395d486c91e72f8632c8e809a6d08a04311bb5c","unresolved":false,"context_lines":[{"line_number":25,"context_line":""},{"line_number":26,"context_line":""},{"line_number":27,"context_line":"@contextmanager"},{"line_number":28,"context_line":"def disable_fks(engine):"},{"line_number":29,"context_line":"    active \u003d engine.name in (\u0027mysql\u0027)"},{"line_number":30,"context_line":"    if active:"},{"line_number":31,"context_line":"        engine.execute(\"SET FOREIGN_KEY_CHECKS\u003d0\")"}],"source_content_type":"text/x-python","patch_set":11,"id":"1a122d0e_79c6a468","line":28,"updated":"2016-04-29 20:26:54.000000000","message":"There is a potential problem with using of an Engine instance here, as it merely represents a connection pool: engine.execute() will take *one* connection from a pool and execute the passed SQL query. \n\nIn this particular case we set a SESSION variable, which has *connection* scope. The next call of execute() *can* take another connection from the pool.\n\nWhen applying migration scripts we *should not* have more than one connection in the pool, so this *should* work fine. Still, it\u0027s probably better to re-write this to use connections explicitly. E.g. get a connection in advance:\n\n  with engine.begin() as conn:\n      # use the connection instance here and pass it to disable_fks()\n      ...\n\n\nin that case code below would need to be changed slightly:\n\n   conn.engine.name\n\ninstead of\n\n   engine.name","commit_id":"2e0455514be4fc35614b6455585392b511c8effa"},{"author":{"_account_id":8688,"name":"Alexis Lee","email":"openstack@lxsli.co.uk","username":"lxsli"},"change_message_id":"5ddc3cb463ce7a6fdc33950e42b4547d0beb3caa","unresolved":false,"context_lines":[{"line_number":25,"context_line":""},{"line_number":26,"context_line":""},{"line_number":27,"context_line":"@contextmanager"},{"line_number":28,"context_line":"def disable_fks(engine):"},{"line_number":29,"context_line":"    active \u003d engine.name in (\u0027mysql\u0027)"},{"line_number":30,"context_line":"    if active:"},{"line_number":31,"context_line":"        engine.execute(\"SET FOREIGN_KEY_CHECKS\u003d0\")"}],"source_content_type":"text/x-python","patch_set":11,"id":"1a122d0e_e2b3bc08","line":28,"in_reply_to":"1a122d0e_79c6a468","updated":"2016-05-04 10:00:32.000000000","message":"Done","commit_id":"2e0455514be4fc35614b6455585392b511c8effa"},{"author":{"_account_id":6849,"name":"Roman Podoliaka","email":"roman.podoliaka@gmail.com","username":"rpodolyaka"},"change_message_id":"c395d486c91e72f8632c8e809a6d08a04311bb5c","unresolved":false,"context_lines":[{"line_number":26,"context_line":""},{"line_number":27,"context_line":"@contextmanager"},{"line_number":28,"context_line":"def disable_fks(engine):"},{"line_number":29,"context_line":"    active \u003d engine.name in (\u0027mysql\u0027)"},{"line_number":30,"context_line":"    if active:"},{"line_number":31,"context_line":"        engine.execute(\"SET FOREIGN_KEY_CHECKS\u003d0\")"},{"line_number":32,"context_line":"    try:"}],"source_content_type":"text/x-python","patch_set":11,"id":"1a122d0e_b9003c02","line":29,"updated":"2016-04-29 20:26:54.000000000","message":"So this works, but maybe by chance, as it\u0027s actually a search in a string rather than in a tuple: \n\n(Pdb) eng.name\n \u0027mysql\u0027\n (Pdb) eng.name in (\u0027mysql\u0027)\n True\n (Pdb) (\u0027mysql\u0027)\n \u0027mysql\u0027\n (Pdb) (\u0027mysql\u0027, )\n (\u0027mysql\u0027,)\n (Pdb) eng.name in (\u0027mysql\u0027, )\n True\n\nwe may want to change this to:\n\n  (\u0027mysql\u0027, )\n\nto be explicit","commit_id":"2e0455514be4fc35614b6455585392b511c8effa"},{"author":{"_account_id":8688,"name":"Alexis Lee","email":"openstack@lxsli.co.uk","username":"lxsli"},"change_message_id":"5ddc3cb463ce7a6fdc33950e42b4547d0beb3caa","unresolved":false,"context_lines":[{"line_number":26,"context_line":""},{"line_number":27,"context_line":"@contextmanager"},{"line_number":28,"context_line":"def disable_fks(engine):"},{"line_number":29,"context_line":"    active \u003d engine.name in (\u0027mysql\u0027)"},{"line_number":30,"context_line":"    if active:"},{"line_number":31,"context_line":"        engine.execute(\"SET FOREIGN_KEY_CHECKS\u003d0\")"},{"line_number":32,"context_line":"    try:"}],"source_content_type":"text/x-python","patch_set":11,"id":"1a122d0e_22ce5483","line":29,"in_reply_to":"1a122d0e_b9003c02","updated":"2016-05-04 10:00:32.000000000","message":"Done","commit_id":"2e0455514be4fc35614b6455585392b511c8effa"},{"author":{"_account_id":6849,"name":"Roman Podoliaka","email":"roman.podoliaka@gmail.com","username":"rpodolyaka"},"change_message_id":"c395d486c91e72f8632c8e809a6d08a04311bb5c","unresolved":false,"context_lines":[{"line_number":28,"context_line":"def disable_fks(engine):"},{"line_number":29,"context_line":"    active \u003d engine.name in (\u0027mysql\u0027)"},{"line_number":30,"context_line":"    if active:"},{"line_number":31,"context_line":"        engine.execute(\"SET FOREIGN_KEY_CHECKS\u003d0\")"},{"line_number":32,"context_line":"    try:"},{"line_number":33,"context_line":"        yield"},{"line_number":34,"context_line":"    finally:"}],"source_content_type":"text/x-python","patch_set":11,"id":"1a122d0e_39cc2c69","line":31,"updated":"2016-04-29 20:26:54.000000000","message":"This one and below can potentially be executed for different connections in the pool\n\ne.g. http://xsnippet.org/361666/","commit_id":"2e0455514be4fc35614b6455585392b511c8effa"},{"author":{"_account_id":6849,"name":"Roman Podoliaka","email":"roman.podoliaka@gmail.com","username":"rpodolyaka"},"change_message_id":"c395d486c91e72f8632c8e809a6d08a04311bb5c","unresolved":false,"context_lines":[{"line_number":50,"context_line":"    bw_usage_cache \u003d Table(\u0027bw_usage_cache\u0027, meta, autoload\u003dTrue)"},{"line_number":51,"context_line":"    instances \u003d Table(\u0027instances\u0027, meta, autoload\u003dTrue)"},{"line_number":52,"context_line":""},{"line_number":53,"context_line":"    rows \u003d kill_orphans(bw_usage_cache, instances)"},{"line_number":54,"context_line":"    LOG.info(_LI(\"Deleted %d orphaned bw_usage_cache rows\"), rows)"},{"line_number":55,"context_line":""},{"line_number":56,"context_line":"    with disable_fks(migrate_engine):"}],"source_content_type":"text/x-python","patch_set":11,"id":"1a122d0e_39dd4ce0","line":53,"updated":"2016-04-29 20:26:54.000000000","message":"This potentially touches a large number of rows. We are now doing online schema migrations and separate them from data migrations, so that we don\u0027t lock the database tables while data is migrated. \n\nProbably, we should not call kill_orphans() in the migration scripts at all and introduce an online data migration instead, e.g. similar to https://review.openstack.org/#/c/289548/  , which would delete rows in batches by passing a LIMIT value to DELETE (as it\u0027s recommended in http://dev.mysql.com/doc/refman/5.7/en/delete.html)","commit_id":"2e0455514be4fc35614b6455585392b511c8effa"},{"author":{"_account_id":8688,"name":"Alexis Lee","email":"openstack@lxsli.co.uk","username":"lxsli"},"change_message_id":"5ddc3cb463ce7a6fdc33950e42b4547d0beb3caa","unresolved":false,"context_lines":[{"line_number":50,"context_line":"    bw_usage_cache \u003d Table(\u0027bw_usage_cache\u0027, meta, autoload\u003dTrue)"},{"line_number":51,"context_line":"    instances \u003d Table(\u0027instances\u0027, meta, autoload\u003dTrue)"},{"line_number":52,"context_line":""},{"line_number":53,"context_line":"    rows \u003d kill_orphans(bw_usage_cache, instances)"},{"line_number":54,"context_line":"    LOG.info(_LI(\"Deleted %d orphaned bw_usage_cache rows\"), rows)"},{"line_number":55,"context_line":""},{"line_number":56,"context_line":"    with disable_fks(migrate_engine):"}],"source_content_type":"text/x-python","patch_set":11,"id":"1a122d0e_6277ac02","line":53,"in_reply_to":"1a122d0e_39dd4ce0","updated":"2016-05-04 10:00:32.000000000","message":"wish I\u0027d read this comment first... seems pretty reasonable, I\u0027ll work on that","commit_id":"2e0455514be4fc35614b6455585392b511c8effa"}],"nova/objects/bandwidth_usage.py":[{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"d2bd0d9a8cb416477091ccd4edefe808c4218066","unresolved":false,"context_lines":[{"line_number":104,"context_line":""},{"line_number":105,"context_line":""},{"line_number":106,"context_line":"@db_api.main_context_manager.writer"},{"line_number":107,"context_line":"def instance_uuid_online_data_migration(context, max_count):"},{"line_number":108,"context_line":"    # NOTE(lxsli): We don\u0027t need this imported at runtime, so"},{"line_number":109,"context_line":"    # keep it separate here"},{"line_number":110,"context_line":"    from nova.db.sqlalchemy import models"}],"source_content_type":"text/x-python","patch_set":12,"id":"1a122d0e_2641ff39","line":107,"updated":"2016-05-05 15:09:16.000000000","message":"The problem I have with this is, we don\u0027t enforce that the operator runs nova-manage db online_data_migrations before nova-manage db sync. So I think your migration would at least have to check (do a simple query) that they\u0027ve run the online data migration before running the migration script to change the schema, otherwise it\u0027s going to fail.","commit_id":"bbf7766a144c3990807ab41ff5d30ef05b4f36c0"},{"author":{"_account_id":4393,"name":"Dan Smith","email":"dms@danplanet.com","username":"danms"},"change_message_id":"e97ec09663709251e01ce82c8ed8c468b299e91d","unresolved":false,"context_lines":[{"line_number":104,"context_line":""},{"line_number":105,"context_line":""},{"line_number":106,"context_line":"@db_api.main_context_manager.writer"},{"line_number":107,"context_line":"def instance_uuid_online_data_migration(context, max_count):"},{"line_number":108,"context_line":"    # NOTE(lxsli): We don\u0027t need this imported at runtime, so"},{"line_number":109,"context_line":"    # keep it separate here"},{"line_number":110,"context_line":"    from nova.db.sqlalchemy import models"}],"source_content_type":"text/x-python","patch_set":12,"id":"1a122d0e_c6a5ab03","line":107,"in_reply_to":"1a122d0e_2641ff39","updated":"2016-05-05 15:10:59.000000000","message":"This can\u0027t be. The online migrations for a given release _have_ to go after the db_sync, else they can\u0027t do their thing.\n\nThat probably means we need to do the online migration and enforcement in code in Newton and then add the FK in Ocata.","commit_id":"bbf7766a144c3990807ab41ff5d30ef05b4f36c0"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"04400b2a97ee4a79768e5da48b1adbaa728cd6fe","unresolved":false,"context_lines":[{"line_number":112,"context_line":"    bw_usage \u003d models.BandwidthUsage"},{"line_number":113,"context_line":"    instance \u003d models.Instance"},{"line_number":114,"context_line":""},{"line_number":115,"context_line":"    engine \u003d db_api.get_api_engine()"},{"line_number":116,"context_line":"    count_all \u003d engine.execute(sql.select([func.count(bw_usage.id)])"},{"line_number":117,"context_line":"         .where(~bw_usage.uuid.in_("},{"line_number":118,"context_line":"             sql.select([instance.uuid])))"}],"source_content_type":"text/x-python","patch_set":12,"id":"1a122d0e_866293ab","line":115,"range":{"start_line":115,"start_character":4,"end_line":115,"end_character":36},"updated":"2016-05-05 15:13:03.000000000","message":"This is the API DB, not the nova cell DB, which is probably why you\u0027re getting table not found for bw_usage_cache.","commit_id":"bbf7766a144c3990807ab41ff5d30ef05b4f36c0"}],"nova/tests/unit/db/test_migrations.py":[{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"cb00abe4eba83815769d34ca5cfe8ddee0e0316d","unresolved":false,"context_lines":[{"line_number":810,"context_line":"        bw_usage_cache \u003d oslodbutils.get_table(engine, \u0027bw_usage_cache\u0027)"},{"line_number":811,"context_line":"        fake_bw_usage \u003d {\u0027start_period\u0027: timeutils.utcnow(),"},{"line_number":812,"context_line":"                         \u0027uuid\u0027: \u0027meow\u0027}"},{"line_number":813,"context_line":"        bw_usage_cache.insert().execute(fake_bw_usage)"},{"line_number":814,"context_line":""},{"line_number":815,"context_line":"    def _check_313(self, engine, data):"},{"line_number":816,"context_line":"        bw_usage_cache \u003d oslodbutils.get_table(engine, \u0027bw_usage_cache\u0027)"}],"source_content_type":"text/x-python","patch_set":7,"id":"5a710552_714f906b","line":813,"updated":"2015-12-16 16:53:25.000000000","message":"Would it be too much to ask that we have a bw_usage_cache entry with a valid uuid reference back to an instance that exists?  Then after running the migration we could make sure we only have one result and it\u0027s not the \u0027meow\u0027 instance.","commit_id":"d265926710832fe1238e3948c32cab7932615c5d"}]}