)]}'
{"specs/utf8mb4-support-for-mysql.rst":[{"author":{"_account_id":16051,"name":"Ronald Bradford","email":"ronald.bradford@gmail.com","username":"ronaldbradford"},"change_message_id":"9a49d021d66a9978203ce254de8759e5409d99f0","unresolved":false,"context_lines":[{"line_number":2,"context_line":" Support for 4 byte unicode characters"},{"line_number":3,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":4,"context_line":""},{"line_number":5,"context_line":"Support for 4-byte unicode characters in mysql database."},{"line_number":6,"context_line":""},{"line_number":7,"context_line":"MySQL database uses utf8 character set currently in different"},{"line_number":8,"context_line":"components of openstack."}],"source_content_type":"text/x-rst","patch_set":1,"id":"dae33548_670e623c","line":5,"updated":"2016-02-17 17:06:41.000000000","message":"As mentioned below, not to nit but MySQL is the correct way to describe the product. Always good to use the correct name throughout the document.  I am one that prefers to see the product appropriately named.","commit_id":"86cb2ecd117354b63dfe7b6a57d57c4a041b0e81"},{"author":{"_account_id":17405,"name":"Jacob Gregor","email":"jgregor@us.ibm.com","username":"jgregor"},"change_message_id":"b019e77e1f1c9d86131ac172787fb3f5ad6eeac2","unresolved":false,"context_lines":[{"line_number":2,"context_line":" Support for 4 byte unicode characters"},{"line_number":3,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":4,"context_line":""},{"line_number":5,"context_line":"Support for 4-byte unicode characters in mysql database."},{"line_number":6,"context_line":""},{"line_number":7,"context_line":"MySQL database uses utf8 character set currently in different"},{"line_number":8,"context_line":"components of openstack."}],"source_content_type":"text/x-rst","patch_set":1,"id":"3afc51ec_baf92450","line":5,"in_reply_to":"dae33548_670e623c","updated":"2016-03-11 21:44:46.000000000","message":"Done","commit_id":"86cb2ecd117354b63dfe7b6a57d57c4a041b0e81"},{"author":{"_account_id":16051,"name":"Ronald Bradford","email":"ronald.bradford@gmail.com","username":"ronaldbradford"},"change_message_id":"9a49d021d66a9978203ce254de8759e5409d99f0","unresolved":false,"context_lines":[{"line_number":5,"context_line":"Support for 4-byte unicode characters in mysql database."},{"line_number":6,"context_line":""},{"line_number":7,"context_line":"MySQL database uses utf8 character set currently in different"},{"line_number":8,"context_line":"components of openstack."},{"line_number":9,"context_line":"But utf8 does not support 4 byte characters which limits user to"},{"line_number":10,"context_line":"use 4 byte unicode characters in different entity names."},{"line_number":11,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"dae33548_4a287ee2","line":8,"updated":"2016-02-17 17:06:41.000000000","message":"Another small nit. If we were wanting to do right by product names, the correct capitalization is OpenStack.","commit_id":"86cb2ecd117354b63dfe7b6a57d57c4a041b0e81"},{"author":{"_account_id":17405,"name":"Jacob Gregor","email":"jgregor@us.ibm.com","username":"jgregor"},"change_message_id":"b019e77e1f1c9d86131ac172787fb3f5ad6eeac2","unresolved":false,"context_lines":[{"line_number":5,"context_line":"Support for 4-byte unicode characters in mysql database."},{"line_number":6,"context_line":""},{"line_number":7,"context_line":"MySQL database uses utf8 character set currently in different"},{"line_number":8,"context_line":"components of openstack."},{"line_number":9,"context_line":"But utf8 does not support 4 byte characters which limits user to"},{"line_number":10,"context_line":"use 4 byte unicode characters in different entity names."},{"line_number":11,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"3afc51ec_5a9f4003","line":8,"in_reply_to":"dae33548_4a287ee2","updated":"2016-03-11 21:44:46.000000000","message":"Done","commit_id":"86cb2ecd117354b63dfe7b6a57d57c4a041b0e81"},{"author":{"_account_id":16051,"name":"Ronald Bradford","email":"ronald.bradford@gmail.com","username":"ronaldbradford"},"change_message_id":"9a49d021d66a9978203ce254de8759e5409d99f0","unresolved":false,"context_lines":[{"line_number":6,"context_line":""},{"line_number":7,"context_line":"MySQL database uses utf8 character set currently in different"},{"line_number":8,"context_line":"components of openstack."},{"line_number":9,"context_line":"But utf8 does not support 4 byte characters which limits user to"},{"line_number":10,"context_line":"use 4 byte unicode characters in different entity names."},{"line_number":11,"context_line":""},{"line_number":12,"context_line":"Problem description"}],"source_content_type":"text/x-rst","patch_set":1,"id":"dae33548_76f0d591","line":9,"updated":"2016-02-17 17:06:41.000000000","message":"Perhaps a more succinct introduction to this could be worded.\n \nThe default utf8 character set in MySQL is 3 bytes and does not support all possible language characters.  Historically, it was necessary to use a 4 byte unicode datatype prior to the introduction of a true 4 byte utf8 datatype for all higher order characters with MySQL 5.5 in 2010.","commit_id":"86cb2ecd117354b63dfe7b6a57d57c4a041b0e81"},{"author":{"_account_id":17405,"name":"Jacob Gregor","email":"jgregor@us.ibm.com","username":"jgregor"},"change_message_id":"b019e77e1f1c9d86131ac172787fb3f5ad6eeac2","unresolved":false,"context_lines":[{"line_number":6,"context_line":""},{"line_number":7,"context_line":"MySQL database uses utf8 character set currently in different"},{"line_number":8,"context_line":"components of openstack."},{"line_number":9,"context_line":"But utf8 does not support 4 byte characters which limits user to"},{"line_number":10,"context_line":"use 4 byte unicode characters in different entity names."},{"line_number":11,"context_line":""},{"line_number":12,"context_line":"Problem description"}],"source_content_type":"text/x-rst","patch_set":1,"id":"3afc51ec_7a867cba","line":9,"in_reply_to":"dae33548_76f0d591","updated":"2016-03-11 21:44:46.000000000","message":"Done","commit_id":"86cb2ecd117354b63dfe7b6a57d57c4a041b0e81"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"ae5c5c9eff0c12eb675994be1ec3c29ba9b1461a","unresolved":false,"context_lines":[{"line_number":12,"context_line":"Problem description"},{"line_number":13,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":14,"context_line":""},{"line_number":15,"context_line":"Different components in openstack use mySql database as default"},{"line_number":16,"context_line":"database to store different information."},{"line_number":17,"context_line":"Currently mysql supports utf8 character-set but this is not pure"},{"line_number":18,"context_line":"utf8 support as mysql lacks support for 4 byte unicode characters"}],"source_content_type":"text/x-rst","patch_set":1,"id":"dae33548_f797ff8d","line":15,"range":{"start_line":15,"start_character":38,"end_line":15,"end_character":43},"updated":"2016-02-16 17:49:35.000000000","message":"Someone in nitpicking mood would say MySQL is the correct spelling. ;)","commit_id":"86cb2ecd117354b63dfe7b6a57d57c4a041b0e81"},{"author":{"_account_id":17405,"name":"Jacob Gregor","email":"jgregor@us.ibm.com","username":"jgregor"},"change_message_id":"b019e77e1f1c9d86131ac172787fb3f5ad6eeac2","unresolved":false,"context_lines":[{"line_number":12,"context_line":"Problem description"},{"line_number":13,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":14,"context_line":""},{"line_number":15,"context_line":"Different components in openstack use mySql database as default"},{"line_number":16,"context_line":"database to store different information."},{"line_number":17,"context_line":"Currently mysql supports utf8 character-set but this is not pure"},{"line_number":18,"context_line":"utf8 support as mysql lacks support for 4 byte unicode characters"}],"source_content_type":"text/x-rst","patch_set":1,"id":"3afc51ec_da9350f9","line":15,"range":{"start_line":15,"start_character":38,"end_line":15,"end_character":43},"in_reply_to":"dae33548_f797ff8d","updated":"2016-03-11 21:44:46.000000000","message":"Done","commit_id":"86cb2ecd117354b63dfe7b6a57d57c4a041b0e81"},{"author":{"_account_id":11564,"name":"Chris Dent","email":"cdent@anticdent.org","username":"chdent"},"change_message_id":"afd97f218208902da5dc5c3032dfb12248980cc8","unresolved":false,"context_lines":[{"line_number":18,"context_line":"utf8 support as mysql lacks support for 4 byte unicode characters"},{"line_number":19,"context_line":"under utf8 character set."},{"line_number":20,"context_line":""},{"line_number":21,"context_line":"This restricts user to use 4 byte unicode characters for entities"},{"line_number":22,"context_line":"which needs to be stored in database."},{"line_number":23,"context_line":""},{"line_number":24,"context_line":"For ex,"}],"source_content_type":"text/x-rst","patch_set":1,"id":"dae33548_e123d0a5","line":21,"updated":"2016-02-16 11:49:27.000000000","message":"I think you want something like \"This makes it impossible for users to use 4 byte unicode...\". There\u0027s a similar problem at line 9. In both cases you\u0027re reversing your actual intent.","commit_id":"86cb2ecd117354b63dfe7b6a57d57c4a041b0e81"},{"author":{"_account_id":17405,"name":"Jacob Gregor","email":"jgregor@us.ibm.com","username":"jgregor"},"change_message_id":"b019e77e1f1c9d86131ac172787fb3f5ad6eeac2","unresolved":false,"context_lines":[{"line_number":18,"context_line":"utf8 support as mysql lacks support for 4 byte unicode characters"},{"line_number":19,"context_line":"under utf8 character set."},{"line_number":20,"context_line":""},{"line_number":21,"context_line":"This restricts user to use 4 byte unicode characters for entities"},{"line_number":22,"context_line":"which needs to be stored in database."},{"line_number":23,"context_line":""},{"line_number":24,"context_line":"For ex,"}],"source_content_type":"text/x-rst","patch_set":1,"id":"3afc51ec_ba9004ee","line":21,"in_reply_to":"dae33548_e123d0a5","updated":"2016-03-11 21:44:46.000000000","message":"Done","commit_id":"86cb2ecd117354b63dfe7b6a57d57c4a041b0e81"},{"author":{"_account_id":17405,"name":"Jacob Gregor","email":"jgregor@us.ibm.com","username":"jgregor"},"change_message_id":"45d9f3f654574c07186bbd9b43e15ec6f91aa623","unresolved":false,"context_lines":[{"line_number":23,"context_line":""},{"line_number":24,"context_line":"For ex,"},{"line_number":25,"context_line":"In cinder, an user can not use 4 byte unicode characters in name and"},{"line_number":26,"context_line":"description of different entities like volume, snapshot, snapshot,"},{"line_number":27,"context_line":"QoS etc."},{"line_number":28,"context_line":"Same is the case with nova, glance, Heat and keystone for instance"},{"line_number":29,"context_line":"name, image name, stack name and role name respectively."}],"source_content_type":"text/x-rst","patch_set":1,"id":"dae33548_083b90d2","line":26,"range":{"start_line":26,"start_character":56,"end_line":26,"end_character":65},"updated":"2016-02-15 22:09:58.000000000","message":"Listed twice.","commit_id":"86cb2ecd117354b63dfe7b6a57d57c4a041b0e81"},{"author":{"_account_id":16308,"name":"Sheel Rana","email":"ranasheel2000@gmail.com","username":"sheel.rana"},"change_message_id":"fda13c55c8774ed91b1b52333bb3e32203bd8fc8","unresolved":false,"context_lines":[{"line_number":23,"context_line":""},{"line_number":24,"context_line":"For ex,"},{"line_number":25,"context_line":"In cinder, an user can not use 4 byte unicode characters in name and"},{"line_number":26,"context_line":"description of different entities like volume, snapshot, snapshot,"},{"line_number":27,"context_line":"QoS etc."},{"line_number":28,"context_line":"Same is the case with nova, glance, Heat and keystone for instance"},{"line_number":29,"context_line":"name, image name, stack name and role name respectively."}],"source_content_type":"text/x-rst","patch_set":1,"id":"dae33548_8ed2fffe","line":26,"range":{"start_line":26,"start_character":56,"end_line":26,"end_character":65},"in_reply_to":"dae33548_083b90d2","updated":"2016-02-16 07:06:32.000000000","message":"right, will update.","commit_id":"86cb2ecd117354b63dfe7b6a57d57c4a041b0e81"},{"author":{"_account_id":17405,"name":"Jacob Gregor","email":"jgregor@us.ibm.com","username":"jgregor"},"change_message_id":"b019e77e1f1c9d86131ac172787fb3f5ad6eeac2","unresolved":false,"context_lines":[{"line_number":23,"context_line":""},{"line_number":24,"context_line":"For ex,"},{"line_number":25,"context_line":"In cinder, an user can not use 4 byte unicode characters in name and"},{"line_number":26,"context_line":"description of different entities like volume, snapshot, snapshot,"},{"line_number":27,"context_line":"QoS etc."},{"line_number":28,"context_line":"Same is the case with nova, glance, Heat and keystone for instance"},{"line_number":29,"context_line":"name, image name, stack name and role name respectively."}],"source_content_type":"text/x-rst","patch_set":1,"id":"3afc51ec_1abc986b","line":26,"range":{"start_line":26,"start_character":56,"end_line":26,"end_character":65},"in_reply_to":"dae33548_8ed2fffe","updated":"2016-03-11 21:44:46.000000000","message":"Done","commit_id":"86cb2ecd117354b63dfe7b6a57d57c4a041b0e81"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"ae5c5c9eff0c12eb675994be1ec3c29ba9b1461a","unresolved":false,"context_lines":[{"line_number":28,"context_line":"Same is the case with nova, glance, Heat and keystone for instance"},{"line_number":29,"context_line":"name, image name, stack name and role name respectively."},{"line_number":30,"context_line":""},{"line_number":31,"context_line":"In case user uses 4byte unicode characters in name field, mySql raises"},{"line_number":32,"context_line":"exception for handling 4 byte unicode characters which leads to"},{"line_number":33,"context_line":"operation failure."},{"line_number":34,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"dae33548_b759f700","line":31,"updated":"2016-02-16 17:49:35.000000000","message":"missing space","commit_id":"86cb2ecd117354b63dfe7b6a57d57c4a041b0e81"},{"author":{"_account_id":17405,"name":"Jacob Gregor","email":"jgregor@us.ibm.com","username":"jgregor"},"change_message_id":"b019e77e1f1c9d86131ac172787fb3f5ad6eeac2","unresolved":false,"context_lines":[{"line_number":28,"context_line":"Same is the case with nova, glance, Heat and keystone for instance"},{"line_number":29,"context_line":"name, image name, stack name and role name respectively."},{"line_number":30,"context_line":""},{"line_number":31,"context_line":"In case user uses 4byte unicode characters in name field, mySql raises"},{"line_number":32,"context_line":"exception for handling 4 byte unicode characters which leads to"},{"line_number":33,"context_line":"operation failure."},{"line_number":34,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"3afc51ec_fac0cce2","line":31,"in_reply_to":"dae33548_b759f700","updated":"2016-03-11 21:44:46.000000000","message":"Done","commit_id":"86cb2ecd117354b63dfe7b6a57d57c4a041b0e81"},{"author":{"_account_id":16051,"name":"Ronald Bradford","email":"ronald.bradford@gmail.com","username":"ronaldbradford"},"change_message_id":"9a49d021d66a9978203ce254de8759e5409d99f0","unresolved":false,"context_lines":[{"line_number":46,"context_line":"To support 4 byte unicode characters, we need to"},{"line_number":47,"context_line":""},{"line_number":48,"context_line":"1. change the character set and collation properties of the databases,"},{"line_number":49,"context_line":"   tables, and columns to use utf8mb4 instead of utf8."},{"line_number":50,"context_line":"   This can simply be done using Alter Tables."},{"line_number":51,"context_line":""},{"line_number":52,"context_line":"2. convert Text, Char, Varchar to utf8mb4 charset."}],"source_content_type":"text/x-rst","patch_set":1,"id":"dae33548_59a86277","line":49,"updated":"2016-02-17 17:06:41.000000000","message":"May pay to add a reference to https://dev.mysql.com/doc/refman/5.5/en/charset-unicode-utf8mb4.html here to describe this to people not familiar with MySQL.","commit_id":"86cb2ecd117354b63dfe7b6a57d57c4a041b0e81"},{"author":{"_account_id":17405,"name":"Jacob Gregor","email":"jgregor@us.ibm.com","username":"jgregor"},"change_message_id":"b019e77e1f1c9d86131ac172787fb3f5ad6eeac2","unresolved":false,"context_lines":[{"line_number":46,"context_line":"To support 4 byte unicode characters, we need to"},{"line_number":47,"context_line":""},{"line_number":48,"context_line":"1. change the character set and collation properties of the databases,"},{"line_number":49,"context_line":"   tables, and columns to use utf8mb4 instead of utf8."},{"line_number":50,"context_line":"   This can simply be done using Alter Tables."},{"line_number":51,"context_line":""},{"line_number":52,"context_line":"2. convert Text, Char, Varchar to utf8mb4 charset."}],"source_content_type":"text/x-rst","patch_set":1,"id":"3afc51ec_3abb5474","line":49,"in_reply_to":"dae33548_59a86277","updated":"2016-03-11 21:44:46.000000000","message":"Done","commit_id":"86cb2ecd117354b63dfe7b6a57d57c4a041b0e81"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"c48a3f280f8b29120c0eea7956f9fbd7c3c724bf","unresolved":false,"context_lines":[{"line_number":46,"context_line":"To support 4 byte unicode characters, we need to"},{"line_number":47,"context_line":""},{"line_number":48,"context_line":"1. change the character set and collation properties of the databases,"},{"line_number":49,"context_line":"   tables, and columns to use utf8mb4 instead of utf8."},{"line_number":50,"context_line":"   This can simply be done using Alter Tables."},{"line_number":51,"context_line":""},{"line_number":52,"context_line":"2. convert Text, Char, Varchar to utf8mb4 charset."}],"source_content_type":"text/x-rst","patch_set":1,"id":"bae84128_59b580cb","line":49,"in_reply_to":"dae33548_59a86277","updated":"2016-02-19 21:33:32.000000000","message":"Uhm, yeah, no. I don\u0027t have a problem setting specific \"name\" columns to use utf8mb4, but under no circumstances do I want to see any defaults (for the schema or the tables) set to utf8mb4.\n\nIt\u0027s already a performance problem that we use three *times* as much storage than we actually need for columns that store UUIDs because we set the default database and table column charset to utf8 (3-byte default in MySQL).\n\nThis 3X storage requirements means index scans and searches are 3 times slower, require 3 times more resources to satisfy in the query plan, and generally just slow things down for no reason at all.","commit_id":"86cb2ecd117354b63dfe7b6a57d57c4a041b0e81"},{"author":{"_account_id":11564,"name":"Chris Dent","email":"cdent@anticdent.org","username":"chdent"},"change_message_id":"afd97f218208902da5dc5c3032dfb12248980cc8","unresolved":false,"context_lines":[{"line_number":47,"context_line":""},{"line_number":48,"context_line":"1. change the character set and collation properties of the databases,"},{"line_number":49,"context_line":"   tables, and columns to use utf8mb4 instead of utf8."},{"line_number":50,"context_line":"   This can simply be done using Alter Tables."},{"line_number":51,"context_line":""},{"line_number":52,"context_line":"2. convert Text, Char, Varchar to utf8mb4 charset."},{"line_number":53,"context_line":"   Below is the list of tables for which needs conversion:"}],"source_content_type":"text/x-rst","patch_set":1,"id":"dae33548_413b9cd1","line":50,"updated":"2016-02-16 11:49:27.000000000","message":"There was some discussion in the nova irc channel yesterday that altering the charset of a column probably violates the constraints on what a migration may do. Altering the charset of a column may require that the table be rebuilt, which takes time and thus would cause downtime. I suspect this is a concern that will need to be researched and confirmed or denied before we can go much further on this idea.","commit_id":"86cb2ecd117354b63dfe7b6a57d57c4a041b0e81"},{"author":{"_account_id":17405,"name":"Jacob Gregor","email":"jgregor@us.ibm.com","username":"jgregor"},"change_message_id":"45d9f3f654574c07186bbd9b43e15ec6f91aa623","unresolved":false,"context_lines":[{"line_number":47,"context_line":""},{"line_number":48,"context_line":"1. change the character set and collation properties of the databases,"},{"line_number":49,"context_line":"   tables, and columns to use utf8mb4 instead of utf8."},{"line_number":50,"context_line":"   This can simply be done using Alter Tables."},{"line_number":51,"context_line":""},{"line_number":52,"context_line":"2. convert Text, Char, Varchar to utf8mb4 charset."},{"line_number":53,"context_line":"   Below is the list of tables for which needs conversion:"}],"source_content_type":"text/x-rst","patch_set":1,"id":"dae33548_88ba00e0","line":50,"range":{"start_line":50,"start_character":33,"end_line":50,"end_character":45},"updated":"2016-02-15 22:09:58.000000000","message":"Will we eventually also need to Alter Database?","commit_id":"86cb2ecd117354b63dfe7b6a57d57c4a041b0e81"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"ae5c5c9eff0c12eb675994be1ec3c29ba9b1461a","unresolved":false,"context_lines":[{"line_number":47,"context_line":""},{"line_number":48,"context_line":"1. change the character set and collation properties of the databases,"},{"line_number":49,"context_line":"   tables, and columns to use utf8mb4 instead of utf8."},{"line_number":50,"context_line":"   This can simply be done using Alter Tables."},{"line_number":51,"context_line":""},{"line_number":52,"context_line":"2. convert Text, Char, Varchar to utf8mb4 charset."},{"line_number":53,"context_line":"   Below is the list of tables for which needs conversion:"}],"source_content_type":"text/x-rst","patch_set":1,"id":"dae33548_b7d297d0","line":50,"in_reply_to":"dae33548_413b9cd1","updated":"2016-02-16 17:49:35.000000000","message":"The IRC discussion was actually triggered by this idea - in Cinder we\u0027re having almost the same set of constraints for DB migrations.\n\nIn general that would mean that in case of Nova it would require creating new columns and moving data there as a live-data-update.\n\nIn Cinder we would need to stretch similar effort through 3 releases. This would mean if we start work in X release, we get the functionality working in X+1 and finish all the cleanups in X+2. It may be possible to shrink that to 2 if we would be smart enough, but I\u0027m 100% sure such approach is actually feasible.","commit_id":"86cb2ecd117354b63dfe7b6a57d57c4a041b0e81"},{"author":{"_account_id":16308,"name":"Sheel Rana","email":"ranasheel2000@gmail.com","username":"sheel.rana"},"change_message_id":"fda13c55c8774ed91b1b52333bb3e32203bd8fc8","unresolved":false,"context_lines":[{"line_number":47,"context_line":""},{"line_number":48,"context_line":"1. change the character set and collation properties of the databases,"},{"line_number":49,"context_line":"   tables, and columns to use utf8mb4 instead of utf8."},{"line_number":50,"context_line":"   This can simply be done using Alter Tables."},{"line_number":51,"context_line":""},{"line_number":52,"context_line":"2. convert Text, Char, Varchar to utf8mb4 charset."},{"line_number":53,"context_line":"   Below is the list of tables for which needs conversion:"}],"source_content_type":"text/x-rst","patch_set":1,"id":"dae33548_8e6e7f2d","line":50,"range":{"start_line":50,"start_character":33,"end_line":50,"end_character":45},"in_reply_to":"dae33548_88ba00e0","updated":"2016-02-16 07:06:32.000000000","message":"right. \nThough mentioned under point 1 but will add here as well.\n\"1. change the character set and collation properties of the databases,\n   tables, and columns to use utf8mb4 instead of utf8.\"","commit_id":"86cb2ecd117354b63dfe7b6a57d57c4a041b0e81"},{"author":{"_account_id":16051,"name":"Ronald Bradford","email":"ronald.bradford@gmail.com","username":"ronaldbradford"},"change_message_id":"9a49d021d66a9978203ce254de8759e5409d99f0","unresolved":false,"context_lines":[{"line_number":47,"context_line":""},{"line_number":48,"context_line":"1. change the character set and collation properties of the databases,"},{"line_number":49,"context_line":"   tables, and columns to use utf8mb4 instead of utf8."},{"line_number":50,"context_line":"   This can simply be done using Alter Tables."},{"line_number":51,"context_line":""},{"line_number":52,"context_line":"2. convert Text, Char, Varchar to utf8mb4 charset."},{"line_number":53,"context_line":"   Below is the list of tables for which needs conversion:"}],"source_content_type":"text/x-rst","patch_set":1,"id":"dae33548_1967fae9","line":50,"range":{"start_line":50,"start_character":33,"end_line":50,"end_character":45},"in_reply_to":"dae33548_8e6e7f2d","updated":"2016-02-17 17:06:41.000000000","message":"You are right, all schemas (aka databases) are utf8. You can verify this with. \n\nselect * from information_schema.schemata;","commit_id":"86cb2ecd117354b63dfe7b6a57d57c4a041b0e81"},{"author":{"_account_id":16051,"name":"Ronald Bradford","email":"ronald.bradford@gmail.com","username":"ronaldbradford"},"change_message_id":"9a49d021d66a9978203ce254de8759e5409d99f0","unresolved":false,"context_lines":[{"line_number":47,"context_line":""},{"line_number":48,"context_line":"1. change the character set and collation properties of the databases,"},{"line_number":49,"context_line":"   tables, and columns to use utf8mb4 instead of utf8."},{"line_number":50,"context_line":"   This can simply be done using Alter Tables."},{"line_number":51,"context_line":""},{"line_number":52,"context_line":"2. convert Text, Char, Varchar to utf8mb4 charset."},{"line_number":53,"context_line":"   Below is the list of tables for which needs conversion:"}],"source_content_type":"text/x-rst","patch_set":1,"id":"dae33548_9932eaa5","line":50,"in_reply_to":"dae33548_b7d297d0","updated":"2016-02-17 17:06:41.000000000","message":"While the process of database migration is complex in a highly available system (and a separate dicussion), adding new columns and moving data inplace has several critical disadvantages that make it impractical in a system of any volume.\n\n1. It takes more diskspace.\n2. For columns that require indexes it takes even more diskspace.\n3. Removing the columns at the end does not reclaim that diskspace.\n4. For a MySQL environment that does NOT use innodb_file_per_table, you cannot get this space back with an entire dump, destroy and reload of ALL databases.","commit_id":"86cb2ecd117354b63dfe7b6a57d57c4a041b0e81"},{"author":{"_account_id":17405,"name":"Jacob Gregor","email":"jgregor@us.ibm.com","username":"jgregor"},"change_message_id":"45d9f3f654574c07186bbd9b43e15ec6f91aa623","unresolved":false,"context_lines":[{"line_number":53,"context_line":"   Below is the list of tables for which needs conversion:"},{"line_number":54,"context_line":""},{"line_number":55,"context_line":"   a. Cinder:"},{"line_number":56,"context_line":"   volume_glance_metadata"},{"line_number":57,"context_line":""},{"line_number":58,"context_line":"   b. Glance:"},{"line_number":59,"context_line":"   image_properties, image_locations, task_info"}],"source_content_type":"text/x-rst","patch_set":1,"id":"dae33548_a8038472","line":56,"range":{"start_line":56,"start_character":3,"end_line":56,"end_character":25},"updated":"2016-02-15 22:09:58.000000000","message":"Shouldn\u0027t there be more tables here? Ex: migrations, services,volume_types, etc ?","commit_id":"86cb2ecd117354b63dfe7b6a57d57c4a041b0e81"},{"author":{"_account_id":16308,"name":"Sheel Rana","email":"ranasheel2000@gmail.com","username":"sheel.rana"},"change_message_id":"fda13c55c8774ed91b1b52333bb3e32203bd8fc8","unresolved":false,"context_lines":[{"line_number":53,"context_line":"   Below is the list of tables for which needs conversion:"},{"line_number":54,"context_line":""},{"line_number":55,"context_line":"   a. Cinder:"},{"line_number":56,"context_line":"   volume_glance_metadata"},{"line_number":57,"context_line":""},{"line_number":58,"context_line":"   b. Glance:"},{"line_number":59,"context_line":"   image_properties, image_locations, task_info"}],"source_content_type":"text/x-rst","patch_set":1,"id":"dae33548_ce9407e4","line":56,"range":{"start_line":56,"start_character":3,"end_line":56,"end_character":25},"in_reply_to":"dae33548_a8038472","updated":"2016-02-16 07:06:32.000000000","message":"I could find only one table with column type as {TEXT|VARCHAR|CHAR} which we need to take care during conversion.","commit_id":"86cb2ecd117354b63dfe7b6a57d57c4a041b0e81"},{"author":{"_account_id":16051,"name":"Ronald Bradford","email":"ronald.bradford@gmail.com","username":"ronaldbradford"},"change_message_id":"9a49d021d66a9978203ce254de8759e5409d99f0","unresolved":false,"context_lines":[{"line_number":71,"context_line":""},{"line_number":72,"context_line":"   For ex,"},{"line_number":73,"context_line":"   ALTER TABLE volume_glance_metadata DEFAULT CHARACTER SET utf8mb4,"},{"line_number":74,"context_line":"   MODIFY value TEXT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;"},{"line_number":75,"context_line":""},{"line_number":76,"context_line":"3. repair and optimize all tables"},{"line_number":77,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"dae33548_ea6fd2f9","line":74,"updated":"2016-02-17 17:06:41.000000000","message":"There is a more applicable syntax to alter ALL columns\n\nALTER TABLE \u003ctable\u003e CONVERT TO CHARACTER SET utf8mb4;\n\nThis will also automatically fix the long indexes, indicated by warnings.\nQuery OK, 0 rows affected, 2 warnings (0.09 sec)\nRecords: 0  Duplicates: 0  Warnings: 2\n\nmysql\u003e show warnings;\n+---------+------+---------------------------------------------------------+\n| Level   | Code | Message                                                 |\n+---------+------+---------------------------------------------------------+\n| Warning | 1071 | Specified key was too long; max key length is 767 bytes |\n| Warning | 1071 | Specified key was too long; max key length is 767 bytes |\n+---------+------+---------------------------------------------------------+\n2 rows in set (0.00 sec)\n\nBefore\n  KEY `owner_image_idx` (`owner`),\nAfter\n  KEY `owner_image_idx` (`owner`(191)),\n\n\nHowever I should point out that owner should NEVER be VARCHAR(255) in the first place. It should be CHAR(32).\n\nEven if you left the table column as VARCHAR(255) (poor design), the index should be altered to like VARCHAR(8). \nThis is a much bigger problem that should be tackled across the system.","commit_id":"86cb2ecd117354b63dfe7b6a57d57c4a041b0e81"},{"author":{"_account_id":17405,"name":"Jacob Gregor","email":"jgregor@us.ibm.com","username":"jgregor"},"change_message_id":"b019e77e1f1c9d86131ac172787fb3f5ad6eeac2","unresolved":false,"context_lines":[{"line_number":71,"context_line":""},{"line_number":72,"context_line":"   For ex,"},{"line_number":73,"context_line":"   ALTER TABLE volume_glance_metadata DEFAULT CHARACTER SET utf8mb4,"},{"line_number":74,"context_line":"   MODIFY value TEXT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;"},{"line_number":75,"context_line":""},{"line_number":76,"context_line":"3. repair and optimize all tables"},{"line_number":77,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"3afc51ec_5a3bc0eb","line":74,"in_reply_to":"dae33548_ea6fd2f9","updated":"2016-03-11 21:44:46.000000000","message":"Made a note of this.","commit_id":"86cb2ecd117354b63dfe7b6a57d57c4a041b0e81"},{"author":{"_account_id":16051,"name":"Ronald Bradford","email":"ronald.bradford@gmail.com","username":"ronaldbradford"},"change_message_id":"9a49d021d66a9978203ce254de8759e5409d99f0","unresolved":false,"context_lines":[{"line_number":73,"context_line":"   ALTER TABLE volume_glance_metadata DEFAULT CHARACTER SET utf8mb4,"},{"line_number":74,"context_line":"   MODIFY value TEXT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;"},{"line_number":75,"context_line":""},{"line_number":76,"context_line":"3. repair and optimize all tables"},{"line_number":77,"context_line":""},{"line_number":78,"context_line":"**Challenges:**"},{"line_number":79,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"dae33548_cade4e10","line":76,"updated":"2016-02-17 17:06:41.000000000","message":"You do not repair InnoDB tables. This is only for crashed MyISAM tables.\n\nYou also do NOT need to optimize tables. This is only if you are altering the volume of rows significantly.","commit_id":"86cb2ecd117354b63dfe7b6a57d57c4a041b0e81"},{"author":{"_account_id":17405,"name":"Jacob Gregor","email":"jgregor@us.ibm.com","username":"jgregor"},"change_message_id":"b019e77e1f1c9d86131ac172787fb3f5ad6eeac2","unresolved":false,"context_lines":[{"line_number":73,"context_line":"   ALTER TABLE volume_glance_metadata DEFAULT CHARACTER SET utf8mb4,"},{"line_number":74,"context_line":"   MODIFY value TEXT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;"},{"line_number":75,"context_line":""},{"line_number":76,"context_line":"3. repair and optimize all tables"},{"line_number":77,"context_line":""},{"line_number":78,"context_line":"**Challenges:**"},{"line_number":79,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"3afc51ec_5a6d8003","line":76,"in_reply_to":"dae33548_cade4e10","updated":"2016-03-11 21:44:46.000000000","message":"Removed this step","commit_id":"86cb2ecd117354b63dfe7b6a57d57c4a041b0e81"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"ae5c5c9eff0c12eb675994be1ec3c29ba9b1461a","unresolved":false,"context_lines":[{"line_number":77,"context_line":""},{"line_number":78,"context_line":"**Challenges:**"},{"line_number":79,"context_line":""},{"line_number":80,"context_line":"1. Only MySQL 5.5.3 or later version supports utf8mb4."},{"line_number":81,"context_line":"   So, need to confirm database version is 5.5.3 or greater."},{"line_number":82,"context_line":""},{"line_number":83,"context_line":"2. InnoDB supports a maximum index length of 767 bytes."},{"line_number":84,"context_line":"   So, you can index a maximum of 255 or 191 characters for utf8"}],"source_content_type":"text/x-rst","patch_set":1,"id":"dae33548_5292399f","line":81,"range":{"start_line":80,"start_character":0,"end_line":81,"end_character":60},"updated":"2016-02-16 17:49:35.000000000","message":"Quick search tells me that this is met for Ubuntu 12.04 and CentOS 7, so this constraint seems feasible.","commit_id":"86cb2ecd117354b63dfe7b6a57d57c4a041b0e81"},{"author":{"_account_id":16051,"name":"Ronald Bradford","email":"ronald.bradford@gmail.com","username":"ronaldbradford"},"change_message_id":"9a49d021d66a9978203ce254de8759e5409d99f0","unresolved":false,"context_lines":[{"line_number":77,"context_line":""},{"line_number":78,"context_line":"**Challenges:**"},{"line_number":79,"context_line":""},{"line_number":80,"context_line":"1. Only MySQL 5.5.3 or later version supports utf8mb4."},{"line_number":81,"context_line":"   So, need to confirm database version is 5.5.3 or greater."},{"line_number":82,"context_line":""},{"line_number":83,"context_line":"2. InnoDB supports a maximum index length of 767 bytes."},{"line_number":84,"context_line":"   So, you can index a maximum of 255 or 191 characters for utf8"}],"source_content_type":"text/x-rst","patch_set":1,"id":"dae33548_16a22999","line":81,"range":{"start_line":80,"start_character":0,"end_line":81,"end_character":60},"in_reply_to":"dae33548_5292399f","updated":"2016-02-17 17:06:41.000000000","message":"MySQL 5.5.8 GA was released in 12-2010. Infact 5.5 is now reached EOL as there are two subsequent GA releases.\n\nTechnically as OpenStack really keeps current, we should deprecated MySQL 5.5 required syntax and usage and recommend the next OpenStack release uses a minimum of 5.6.  This however is a discussion for a different spec.","commit_id":"86cb2ecd117354b63dfe7b6a57d57c4a041b0e81"},{"author":{"_account_id":16051,"name":"Ronald Bradford","email":"ronald.bradford@gmail.com","username":"ronaldbradford"},"change_message_id":"9a49d021d66a9978203ce254de8759e5409d99f0","unresolved":false,"context_lines":[{"line_number":82,"context_line":""},{"line_number":83,"context_line":"2. InnoDB supports a maximum index length of 767 bytes."},{"line_number":84,"context_line":"   So, you can index a maximum of 255 or 191 characters for utf8"},{"line_number":85,"context_line":"   and utf8mb4 columns respectively."},{"line_number":86,"context_line":"   If existing utf8 columns have indexes longer than 191 characters,"},{"line_number":87,"context_line":"   then need to index a smaller number of characters when using"},{"line_number":88,"context_line":"   utf8mb4."}],"source_content_type":"text/x-rst","patch_set":1,"id":"dae33548_8a560628","line":85,"updated":"2016-02-17 17:06:41.000000000","message":"See above point.","commit_id":"86cb2ecd117354b63dfe7b6a57d57c4a041b0e81"},{"author":{"_account_id":16051,"name":"Ronald Bradford","email":"ronald.bradford@gmail.com","username":"ronaldbradford"},"change_message_id":"9a49d021d66a9978203ce254de8759e5409d99f0","unresolved":false,"context_lines":[{"line_number":86,"context_line":"   If existing utf8 columns have indexes longer than 191 characters,"},{"line_number":87,"context_line":"   then need to index a smaller number of characters when using"},{"line_number":88,"context_line":"   utf8mb4."},{"line_number":89,"context_line":"   So, need to change indexed VARCHAR(255) columns to VARCHAR(191)"},{"line_number":90,"context_line":"   for utf8mb4."},{"line_number":91,"context_line":""},{"line_number":92,"context_line":"   It seems indexing on varchar could be present but will not be"}],"source_content_type":"text/x-rst","patch_set":1,"id":"dae33548_ec1d3263","line":89,"updated":"2016-02-17 17:06:41.000000000","message":"Not applicable to this spec, but I should point out that VARCHAR(255) is a very poor schema design. It means generally that no thought was actually put into the architectural design of the software. There is an impact to performance using VARCHAR(255). See below.","commit_id":"86cb2ecd117354b63dfe7b6a57d57c4a041b0e81"},{"author":{"_account_id":2472,"name":"Doug Hellmann","email":"dhellmann@redhat.com","username":"doug-hellmann"},"change_message_id":"e9f7c72d5b1dfc04c72607100885d945c6b0b56a","unresolved":false,"context_lines":[{"line_number":87,"context_line":"   then need to index a smaller number of characters when using"},{"line_number":88,"context_line":"   utf8mb4."},{"line_number":89,"context_line":"   So, need to change indexed VARCHAR(255) columns to VARCHAR(191)"},{"line_number":90,"context_line":"   for utf8mb4."},{"line_number":91,"context_line":""},{"line_number":92,"context_line":"   It seems indexing on varchar could be present but will not be"},{"line_number":93,"context_line":"   having such length for indexing and having 191 length should be"}],"source_content_type":"text/x-rst","patch_set":1,"id":"dae33548_78391341","line":90,"updated":"2016-02-16 21:40:24.000000000","message":"Is that going to mean truncating existing data?","commit_id":"86cb2ecd117354b63dfe7b6a57d57c4a041b0e81"},{"author":{"_account_id":16051,"name":"Ronald Bradford","email":"ronald.bradford@gmail.com","username":"ronaldbradford"},"change_message_id":"9a49d021d66a9978203ce254de8759e5409d99f0","unresolved":false,"context_lines":[{"line_number":87,"context_line":"   then need to index a smaller number of characters when using"},{"line_number":88,"context_line":"   utf8mb4."},{"line_number":89,"context_line":"   So, need to change indexed VARCHAR(255) columns to VARCHAR(191)"},{"line_number":90,"context_line":"   for utf8mb4."},{"line_number":91,"context_line":""},{"line_number":92,"context_line":"   It seems indexing on varchar could be present but will not be"},{"line_number":93,"context_line":"   having such length for indexing and having 191 length should be"}],"source_content_type":"text/x-rst","patch_set":1,"id":"dae33548_999b0a18","line":90,"in_reply_to":"dae33548_58324f11","updated":"2016-02-17 17:06:41.000000000","message":"No, this does not truncate the underlying data. It only truncates the value of the data that is stored in the btree leaf nodes. Infact it can be hugely beneficial to limit index length to a number that reasonably identifies a small (e.g. 1-5 rows) of data. This can reduce the actual on-disk size of an index by up to 50%. This saving, translates to more information in memory, and fastest query execution.\n\nHowever, this technique does not work for unique indexes.","commit_id":"86cb2ecd117354b63dfe7b6a57d57c4a041b0e81"},{"author":{"_account_id":2472,"name":"Doug Hellmann","email":"dhellmann@redhat.com","username":"doug-hellmann"},"change_message_id":"5b3e7506fa71f8b3b1fc603181d920a66113e5f3","unresolved":false,"context_lines":[{"line_number":87,"context_line":"   then need to index a smaller number of characters when using"},{"line_number":88,"context_line":"   utf8mb4."},{"line_number":89,"context_line":"   So, need to change indexed VARCHAR(255) columns to VARCHAR(191)"},{"line_number":90,"context_line":"   for utf8mb4."},{"line_number":91,"context_line":""},{"line_number":92,"context_line":"   It seems indexing on varchar could be present but will not be"},{"line_number":93,"context_line":"   having such length for indexing and having 191 length should be"}],"source_content_type":"text/x-rst","patch_set":1,"id":"dae33548_58324f11","line":90,"in_reply_to":"dae33548_78391341","updated":"2016-02-16 21:44:03.000000000","message":"This also seems to be assuming that the column is stored with each character as a 4 byte value. Is that how MySQL stores the data? Or does it use the least number of bytes necessary for each character?","commit_id":"86cb2ecd117354b63dfe7b6a57d57c4a041b0e81"},{"author":{"_account_id":11564,"name":"Chris Dent","email":"cdent@anticdent.org","username":"chdent"},"change_message_id":"afd97f218208902da5dc5c3032dfb12248980cc8","unresolved":false,"context_lines":[{"line_number":95,"context_line":"   (I could see some variable length text in nova specifically)."},{"line_number":96,"context_line":""},{"line_number":97,"context_line":"3. This kind of database change seems to create problem for online"},{"line_number":98,"context_line":"   database migration. Need to discuss on same."},{"line_number":99,"context_line":""},{"line_number":100,"context_line":"Alternatives"},{"line_number":101,"context_line":"------------"}],"source_content_type":"text/x-rst","patch_set":1,"id":"dae33548_213658e7","line":98,"updated":"2016-02-16 11:49:27.000000000","message":"Yeah, see my comments above.","commit_id":"86cb2ecd117354b63dfe7b6a57d57c4a041b0e81"},{"author":{"_account_id":16051,"name":"Ronald Bradford","email":"ronald.bradford@gmail.com","username":"ronaldbradford"},"change_message_id":"9a49d021d66a9978203ce254de8759e5409d99f0","unresolved":false,"context_lines":[{"line_number":95,"context_line":"   (I could see some variable length text in nova specifically)."},{"line_number":96,"context_line":""},{"line_number":97,"context_line":"3. This kind of database change seems to create problem for online"},{"line_number":98,"context_line":"   database migration. Need to discuss on same."},{"line_number":99,"context_line":""},{"line_number":100,"context_line":"Alternatives"},{"line_number":101,"context_line":"------------"}],"source_content_type":"text/x-rst","patch_set":1,"id":"dae33548_4fbff0a9","line":98,"in_reply_to":"dae33548_213658e7","updated":"2016-02-17 17:06:41.000000000","message":"InnoDB in MySQL 5.5 supports that concept of an online ALTER statement (see https://dev.mysql.com/doc/refman/5.5/en/innodb-online-ddl.html).\n\nHowever while in theory it\u0027s great, in practices it comes with limitations (which get better in 5.6 and 5.7) and with issues depending on how you got to MySQL 5.5 (e.g. did you upgrade) etc.","commit_id":"86cb2ecd117354b63dfe7b6a57d57c4a041b0e81"},{"author":{"_account_id":17405,"name":"Jacob Gregor","email":"jgregor@us.ibm.com","username":"jgregor"},"change_message_id":"b019e77e1f1c9d86131ac172787fb3f5ad6eeac2","unresolved":false,"context_lines":[{"line_number":95,"context_line":"   (I could see some variable length text in nova specifically)."},{"line_number":96,"context_line":""},{"line_number":97,"context_line":"3. This kind of database change seems to create problem for online"},{"line_number":98,"context_line":"   database migration. Need to discuss on same."},{"line_number":99,"context_line":""},{"line_number":100,"context_line":"Alternatives"},{"line_number":101,"context_line":"------------"}],"source_content_type":"text/x-rst","patch_set":1,"id":"3afc51ec_3a49948c","line":98,"in_reply_to":"dae33548_4fbff0a9","updated":"2016-03-11 21:44:46.000000000","message":"Added some info here.","commit_id":"86cb2ecd117354b63dfe7b6a57d57c4a041b0e81"},{"author":{"_account_id":11564,"name":"Chris Dent","email":"cdent@anticdent.org","username":"chdent"},"change_message_id":"afd97f218208902da5dc5c3032dfb12248980cc8","unresolved":false,"context_lines":[{"line_number":102,"context_line":""},{"line_number":103,"context_line":"- Restrict user for not using 4 byte unicode characters for different"},{"line_number":104,"context_line":"  entity names and handle exception gracefully with proper error"},{"line_number":105,"context_line":"  message generation."},{"line_number":106,"context_line":""},{"line_number":107,"context_line":"Currently cinder supports same."},{"line_number":108,"context_line":"It generates 400 error with message \"invalid value\"."}],"source_content_type":"text/x-rst","patch_set":1,"id":"dae33548_8135a4db","line":105,"updated":"2016-02-16 11:49:27.000000000","message":"We should do this now, as a bug fix, and then consider a longer term plan.","commit_id":"86cb2ecd117354b63dfe7b6a57d57c4a041b0e81"},{"author":{"_account_id":2472,"name":"Doug Hellmann","email":"dhellmann@redhat.com","username":"doug-hellmann"},"change_message_id":"e9f7c72d5b1dfc04c72607100885d945c6b0b56a","unresolved":false,"context_lines":[{"line_number":102,"context_line":""},{"line_number":103,"context_line":"- Restrict user for not using 4 byte unicode characters for different"},{"line_number":104,"context_line":"  entity names and handle exception gracefully with proper error"},{"line_number":105,"context_line":"  message generation."},{"line_number":106,"context_line":""},{"line_number":107,"context_line":"Currently cinder supports same."},{"line_number":108,"context_line":"It generates 400 error with message \"invalid value\"."}],"source_content_type":"text/x-rst","patch_set":1,"id":"dae33548_387dcbe7","line":105,"in_reply_to":"dae33548_8135a4db","updated":"2016-02-16 21:40:24.000000000","message":"+1","commit_id":"86cb2ecd117354b63dfe7b6a57d57c4a041b0e81"},{"author":{"_account_id":16051,"name":"Ronald Bradford","email":"ronald.bradford@gmail.com","username":"ronaldbradford"},"change_message_id":"9a49d021d66a9978203ce254de8759e5409d99f0","unresolved":false,"context_lines":[{"line_number":106,"context_line":""},{"line_number":107,"context_line":"Currently cinder supports same."},{"line_number":108,"context_line":"It generates 400 error with message \"invalid value\"."},{"line_number":109,"context_line":""},{"line_number":110,"context_line":"Implementation"},{"line_number":111,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":112,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"dae33548_d984720c","line":109,"updated":"2016-02-17 17:06:41.000000000","message":"There is one critical architectural design that is missing from the OpenStack schemas (at least when looking at a devstack environment).\n\nNot all columns need to be utf8, and therefore do not need to be modified to utf8mb4 in the first place.  \n\nThe following information is why.  I was not involved in OpenStack database design, nor are these points considering that OpenStack uses an SQL abstraction layer and an oslo.db library.  These may be limitations of capacity technically, or procedural y (e.g. a rule to be consistent for the sake of consistency).\n\nI share this information because to design an optimized and performant large MySQL system, these points can have a large impact across sizing, backups, performance and the are therefore a big deal in a larger more concurrent system.\n\nFor this example I will use the glance.images.checksum column (because it\u0027s a table in devstack with some data).\n\nThis column is defined as \n\n`checksum` varchar(32) DEFAULT NULL\n\nthe column in indexed (important for this discussion)\n\nKEY `checksum_image_idx` (`checksum`),\n\nand the table is\n\nENGINE\u003dInnoDB DEFAULT CHARSET\u003dutf8\n\nA sample value is 8a40c862b5735975d82605c1dd395796\n\nchecksum is a MD5 string of 32 bytes.  MD5\u0027s as we know ONLY store hexidecimal characters, (it can never be utf8).  \n\nOn disk, MySQL stores this optimally at 32-35 bytes, however in memory this is 99 bytes long.  That\u0027s 32 bytes x 3 bytes per character (utf8 datatype) + 2 bytes because it\u0027s VARCHAR, and 1 more byte because is is NULLABLE.\n\nThe optimal datatype for this column in CHAR(32) CHARACTER SET latin1 NOT NULL \n\nBecause, quite simply this is 32 bytes of memory for the indexed column, not 99 bytes.  You may ask, big deal a few bytes. However when across dozens to hundreds of indexed columns in a database with millions of rows in tables, on a machine with a finite amount of RAM, the impact can mean queries running 10x slower.\n\nAgain, I caveat this comment with how to ideally design a MySQL database schema. In life however we make compromises, and some that may be applicable to OpenStack include, the lack of a data architect, an SQL abstraction layer, rules to enforce consistency and complexity of upgrading an existing product.  Many of these can be overcome with the right investment.","commit_id":"86cb2ecd117354b63dfe7b6a57d57c4a041b0e81"},{"author":{"_account_id":4523,"name":"Eric Harney","email":"eharney@redhat.com","username":"eharney"},"change_message_id":"adfe841143dd7be040f1d521ffa31ad3847e7860","unresolved":false,"context_lines":[{"line_number":2,"context_line":" Support for 4 byte unicode characters"},{"line_number":3,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":4,"context_line":""},{"line_number":5,"context_line":"Support for 4-byte unicode characters in MySQL database."},{"line_number":6,"context_line":""},{"line_number":7,"context_line":"MySQL database uses utf8 character set currently in different"},{"line_number":8,"context_line":"components of OpenStack."}],"source_content_type":"text/x-rst","patch_set":4,"id":"ba0121b8_9eac4f8d","line":5,"updated":"2016-04-01 13:51:08.000000000","message":"Why is this spec only about MySQL?  What about postgres?\n\nI don\u0027t see how this can reach a complete solution when only looking at one database backend.","commit_id":"4b8c83188e1e431e9c43dbd622777581d5ab31c3"},{"author":{"_account_id":4523,"name":"Eric Harney","email":"eharney@redhat.com","username":"eharney"},"change_message_id":"8d8a4b250f4bd9312074b483428a52952e2f14a1","unresolved":false,"context_lines":[{"line_number":2,"context_line":" Support for 4 byte unicode characters"},{"line_number":3,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":4,"context_line":""},{"line_number":5,"context_line":"Support for 4-byte unicode characters in MySQL database."},{"line_number":6,"context_line":""},{"line_number":7,"context_line":"MySQL database uses utf8 character set currently in different"},{"line_number":8,"context_line":"components of OpenStack."}],"source_content_type":"text/x-rst","patch_set":4,"id":"dab17558_7616aa56","line":5,"in_reply_to":"5a18252c_3e88e9c4","updated":"2016-05-12 17:26:06.000000000","message":"I was misreading and read Sheel\u0027s comment as saying that Postgres doesn\u0027t support 4-byte UTF8.\n\nWe\u0027re all on the same page then, it does.  So why is this spec just talking about MySQL?  It needs to handle Postgres, too.","commit_id":"4b8c83188e1e431e9c43dbd622777581d5ab31c3"},{"author":{"_account_id":7198,"name":"Jay Bryant","email":"jungleboyj@electronicjungle.net","username":"jsbryant"},"change_message_id":"e71e92707d9b57d87979d6165a612762e869dc67","unresolved":false,"context_lines":[{"line_number":2,"context_line":" Support for 4 byte unicode characters"},{"line_number":3,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":4,"context_line":""},{"line_number":5,"context_line":"Support for 4-byte unicode characters in MySQL database."},{"line_number":6,"context_line":""},{"line_number":7,"context_line":"MySQL database uses utf8 character set currently in different"},{"line_number":8,"context_line":"components of OpenStack."}],"source_content_type":"text/x-rst","patch_set":4,"id":"5a18252c_3e88e9c4","line":5,"in_reply_to":"9a061dce_6bca32a3","updated":"2016-04-14 14:43:21.000000000","message":"Eric, I am confused.  For postgresql it says:\nUTF8 \tUnicode, 8-bit \tall \tYes \t1-4 \tUnicode\n\nThat seems to suggest to me that it will support 4 byte unicode characters.  This page also indicates full utf8 support:  https://www.drupal.org/node/1314214","commit_id":"4b8c83188e1e431e9c43dbd622777581d5ab31c3"},{"author":{"_account_id":4523,"name":"Eric Harney","email":"eharney@redhat.com","username":"eharney"},"change_message_id":"008ff1b1e5a8ca1c62255f3372b4a3284bd8d507","unresolved":false,"context_lines":[{"line_number":2,"context_line":" Support for 4 byte unicode characters"},{"line_number":3,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":4,"context_line":""},{"line_number":5,"context_line":"Support for 4-byte unicode characters in MySQL database."},{"line_number":6,"context_line":""},{"line_number":7,"context_line":"MySQL database uses utf8 character set currently in different"},{"line_number":8,"context_line":"components of OpenStack."}],"source_content_type":"text/x-rst","patch_set":4,"id":"9a061dce_6bca32a3","line":5,"in_reply_to":"ba0121b8_1b479f1e","updated":"2016-04-04 15:41:50.000000000","message":"\u003eIt looks like postgres does support 4 byte utf. \u003ehttp://www.postgresql.org/docs/8.4/static/multibyte.html\n\nI\u0027m not sure how you drew this conclusion from this page.  It says it supports 1-4 bytes per char.","commit_id":"4b8c83188e1e431e9c43dbd622777581d5ab31c3"},{"author":{"_account_id":16308,"name":"Sheel Rana","email":"ranasheel2000@gmail.com","username":"sheel.rana"},"change_message_id":"66ae4f1f861a1f118111563d8f379532615c7ef6","unresolved":false,"context_lines":[{"line_number":2,"context_line":" Support for 4 byte unicode characters"},{"line_number":3,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":4,"context_line":""},{"line_number":5,"context_line":"Support for 4-byte unicode characters in MySQL database."},{"line_number":6,"context_line":""},{"line_number":7,"context_line":"MySQL database uses utf8 character set currently in different"},{"line_number":8,"context_line":"components of OpenStack."}],"source_content_type":"text/x-rst","patch_set":4,"id":"ba0121b8_1b479f1e","line":5,"in_reply_to":"ba0121b8_9eac4f8d","updated":"2016-04-01 19:01:34.000000000","message":"Some answered from old comments\n\u003eDB2 can support 4 byte characters but I would not say it is widely used as far as OpenStack is \u003econcerned.\n\n\u003eIt looks like postgres does support 4 byte utf.\n\u003ehttp://www.postgresql.org/docs/8.4/static/multibyte.html","commit_id":"4b8c83188e1e431e9c43dbd622777581d5ab31c3"},{"author":{"_account_id":17860,"name":"Samuel de Medeiros Queiroz","email":"samueldmq@gmail.com","username":"samueldmq"},"change_message_id":"b03a0b57c976f252c5a7a487726c0a2df6e1e626","unresolved":false,"context_lines":[{"line_number":2,"context_line":" Support for 4 byte unicode characters"},{"line_number":3,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":4,"context_line":""},{"line_number":5,"context_line":"Support for 4-byte unicode characters in MySQL database."},{"line_number":6,"context_line":""},{"line_number":7,"context_line":"MySQL database uses utf8 character set currently in different"},{"line_number":8,"context_line":"components of OpenStack."}],"source_content_type":"text/x-rst","patch_set":4,"id":"da7fb54a_95efd4af","line":5,"in_reply_to":"da7fb54a_9548b45d","updated":"2016-09-02 16:24:37.000000000","message":"Also, what are the supported databases ? Does this proposal apply to all of them? That way we may ensure interoperability.","commit_id":"4b8c83188e1e431e9c43dbd622777581d5ab31c3"},{"author":{"_account_id":17860,"name":"Samuel de Medeiros Queiroz","email":"samueldmq@gmail.com","username":"samueldmq"},"change_message_id":"63506af3a780787749a26e95f92d35f1e45595c8","unresolved":false,"context_lines":[{"line_number":2,"context_line":" Support for 4 byte unicode characters"},{"line_number":3,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":4,"context_line":""},{"line_number":5,"context_line":"Support for 4-byte unicode characters in MySQL database."},{"line_number":6,"context_line":""},{"line_number":7,"context_line":"MySQL database uses utf8 character set currently in different"},{"line_number":8,"context_line":"components of OpenStack."}],"source_content_type":"text/x-rst","patch_set":4,"id":"da7fb54a_9548b45d","line":5,"in_reply_to":"dab17558_7616aa56","updated":"2016-09-02 16:22:36.000000000","message":"++","commit_id":"4b8c83188e1e431e9c43dbd622777581d5ab31c3"},{"author":{"_account_id":17860,"name":"Samuel de Medeiros Queiroz","email":"samueldmq@gmail.com","username":"samueldmq"},"change_message_id":"63506af3a780787749a26e95f92d35f1e45595c8","unresolved":false,"context_lines":[{"line_number":6,"context_line":""},{"line_number":7,"context_line":"MySQL database uses utf8 character set currently in different"},{"line_number":8,"context_line":"components of OpenStack."},{"line_number":9,"context_line":"The default utf8 character set in MySQL is 3 bytes and does not"},{"line_number":10,"context_line":"support all possible language characters. Historically, it was"},{"line_number":11,"context_line":"neccessary to use a 4 byte unicode datatype prior to the introduction"},{"line_number":12,"context_line":"of a true 4 byte utf8 datatype for all higher order characters with"},{"line_number":13,"context_line":"MySQL 5.5 in 2010."},{"line_number":14,"context_line":""},{"line_number":15,"context_line":"Problem description"},{"line_number":16,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":4,"id":"da7fb54a_d586bc64","line":13,"range":{"start_line":9,"start_character":0,"end_line":13,"end_character":18},"updated":"2016-09-02 16:22:36.000000000","message":"Is there a reference for this?","commit_id":"4b8c83188e1e431e9c43dbd622777581d5ab31c3"},{"author":{"_account_id":4523,"name":"Eric Harney","email":"eharney@redhat.com","username":"eharney"},"change_message_id":"adfe841143dd7be040f1d521ffa31ad3847e7860","unresolved":false,"context_lines":[{"line_number":39,"context_line":"ERROR (ClientException): Unexpected API Error."},{"line_number":40,"context_line":"Please report this at http://bugs.launchpad.net/nova/ and attach"},{"line_number":41,"context_line":"the Nova API log if possible."},{"line_number":42,"context_line":"\u003cclass \u0027oslo_db.exception.DBError\u0027\u003e (HTTP 500)"},{"line_number":43,"context_line":"Request-ID: req-a4eef1d6-11fa-4188-b116-ffdf728e04f4)"},{"line_number":44,"context_line":""},{"line_number":45,"context_line":"Proposed change"}],"source_content_type":"text/x-rst","patch_set":4,"id":"ba0121b8_fe42ab0d","line":42,"updated":"2016-04-01 13:51:08.000000000","message":"This is a bug that should be handled and return 400 regardless of the work proposed here.","commit_id":"4b8c83188e1e431e9c43dbd622777581d5ab31c3"},{"author":{"_account_id":16308,"name":"Sheel Rana","email":"ranasheel2000@gmail.com","username":"sheel.rana"},"change_message_id":"66ae4f1f861a1f118111563d8f379532615c7ef6","unresolved":false,"context_lines":[{"line_number":39,"context_line":"ERROR (ClientException): Unexpected API Error."},{"line_number":40,"context_line":"Please report this at http://bugs.launchpad.net/nova/ and attach"},{"line_number":41,"context_line":"the Nova API log if possible."},{"line_number":42,"context_line":"\u003cclass \u0027oslo_db.exception.DBError\u0027\u003e (HTTP 500)"},{"line_number":43,"context_line":"Request-ID: req-a4eef1d6-11fa-4188-b116-ffdf728e04f4)"},{"line_number":44,"context_line":""},{"line_number":45,"context_line":"Proposed change"}],"source_content_type":"text/x-rst","patch_set":4,"id":"ba0121b8_db124709","line":42,"in_reply_to":"ba0121b8_fe42ab0d","updated":"2016-04-01 19:01:34.000000000","message":"This is already in progress.. I am working on it..\nThanks!!","commit_id":"4b8c83188e1e431e9c43dbd622777581d5ab31c3"},{"author":{"_account_id":6486,"name":"Brant Knudson","email":"blk@acm.org","username":"blk-u"},"change_message_id":"a964287e675cf1fe8535ef2564d9baf76a64d3d6","unresolved":false,"context_lines":[{"line_number":47,"context_line":""},{"line_number":48,"context_line":"To support 4 byte unicode characters, we need to"},{"line_number":49,"context_line":""},{"line_number":50,"context_line":"1. Change the character set and collation properties of the databases,"},{"line_number":51,"context_line":"   tables, and columns to use utf8mb4 instead of utf8. There will need"},{"line_number":52,"context_line":"   to be more discussion on what should use utf8mb4 and what should not."},{"line_number":53,"context_line":"   This can simply be done using Alter Tables."}],"source_content_type":"text/x-rst","patch_set":4,"id":"1af94dfe_445bff54","line":50,"updated":"2016-03-15 13:47:00.000000000","message":"How about instead of forcing the deployment to use whatever we think is right for them, we change openstack to work with whatever the deployment has configured. So if the deployer wants text columns to be utf8mb4 they can set that as the default for their database (or migrate existing tables) and openstack will work. We could have the db commands (like keystone-manage db_sync) fail with an error and require a --force parameter if they have a crazy setup like myisam or a small charset (latin1 or whatever).","commit_id":"4b8c83188e1e431e9c43dbd622777581d5ab31c3"},{"author":{"_account_id":16308,"name":"Sheel Rana","email":"ranasheel2000@gmail.com","username":"sheel.rana"},"change_message_id":"358a76f73e22192f242160e350de395f01c936cb","unresolved":false,"context_lines":[{"line_number":47,"context_line":""},{"line_number":48,"context_line":"To support 4 byte unicode characters, we need to"},{"line_number":49,"context_line":""},{"line_number":50,"context_line":"1. Change the character set and collation properties of the databases,"},{"line_number":51,"context_line":"   tables, and columns to use utf8mb4 instead of utf8. There will need"},{"line_number":52,"context_line":"   to be more discussion on what should use utf8mb4 and what should not."},{"line_number":53,"context_line":"   This can simply be done using Alter Tables."}],"source_content_type":"text/x-rst","patch_set":4,"id":"1af94dfe_da5fe473","line":50,"in_reply_to":"1af94dfe_445bff54","updated":"2016-03-15 14:16:03.000000000","message":"Though I think this approach is Ok.\nBut we have some attached complexities to this approach.\n\nIn different components we have schema validation to restrict 4 byte unicode charset.\nIf we provide \"On demand\" migration to deployer, we have to think about schema validation as well in same way.\n\nFor ex, if deployer decided to go with utf8mb4, then we have to remove schema validation for utfmb4 and if deployer decides to go without utf8mb4, then we have to keep schema validation for 4byte unicodes.\n\nThis is one example, there would be some other cases as well.\nSo, it seems good to go in predecided approach instead of pluggin type support to deployer.","commit_id":"4b8c83188e1e431e9c43dbd622777581d5ab31c3"},{"author":{"_account_id":11564,"name":"Chris Dent","email":"cdent@anticdent.org","username":"chdent"},"change_message_id":"bfef9858910d7705e2e1cc560454a5ef69ddb574","unresolved":false,"context_lines":[{"line_number":47,"context_line":""},{"line_number":48,"context_line":"To support 4 byte unicode characters, we need to"},{"line_number":49,"context_line":""},{"line_number":50,"context_line":"1. Change the character set and collation properties of the databases,"},{"line_number":51,"context_line":"   tables, and columns to use utf8mb4 instead of utf8. There will need"},{"line_number":52,"context_line":"   to be more discussion on what should use utf8mb4 and what should not."},{"line_number":53,"context_line":"   This can simply be done using Alter Tables."}],"source_content_type":"text/x-rst","patch_set":4,"id":"1af94dfe_5d958e3e","line":50,"in_reply_to":"1af94dfe_da5fe473","updated":"2016-03-15 14:29:59.000000000","message":"One issue with letting deployments decide for themselves whether they support utf8 or utf8mb4 or just latin1 or whatever else is that it means that the interfaces presented by different clouds will behave differently. On the one hand this is okay: you try to do something get back a 400 with the right message, fix what you\u0027re doing, go from there, no biggie.\n\nOn the other hand: \u0027But I want to name my thingie \"☃man\" at all my public clouds\u0027.\n\nSo to a certain extent what we\u0027re talking about here is consistency in the exposed interfaces and how important that is.","commit_id":"4b8c83188e1e431e9c43dbd622777581d5ab31c3"},{"author":{"_account_id":17860,"name":"Samuel de Medeiros Queiroz","email":"samueldmq@gmail.com","username":"samueldmq"},"change_message_id":"63506af3a780787749a26e95f92d35f1e45595c8","unresolved":false,"context_lines":[{"line_number":146,"context_line":""},{"line_number":147,"context_line":"* Add support to Keystone database schema."},{"line_number":148,"context_line":""},{"line_number":149,"context_line":"  https://bugs.launchpad.net/keystone/+bug/1545736"},{"line_number":150,"context_line":""},{"line_number":151,"context_line":"* Add support to Glance database schema."},{"line_number":152,"context_line":""}],"source_content_type":"text/x-rst","patch_set":4,"id":"da7fb54a_d5e15cd2","line":149,"updated":"2016-09-02 16:22:36.000000000","message":"what the approach for adding such support ? will you propose a new database migration to add it and remove schema checks that do not allow it today ?\n\nI think it is interesting to state how the issue will be solved here in the work items","commit_id":"4b8c83188e1e431e9c43dbd622777581d5ab31c3"}]}