)]}'
{"specs/juno/switch-to-mysql-connector.rst":[{"author":{"_account_id":2472,"name":"Doug Hellmann","email":"dhellmann@redhat.com","username":"doug-hellmann"},"change_message_id":"d5a33b8d921dd834564d40a0ebacfe67a6ee3456","unresolved":false,"context_lines":[{"line_number":6,"context_line":""},{"line_number":7,"context_line":"Introduction paragraph -- why are we doing anything?"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"Switch from MySQLdb to MySQL Connector which provides multiple benefits,"},{"line_number":10,"context_line":"including resilience of code and potential performance advantages."},{"line_number":11,"context_line":""},{"line_number":12,"context_line":"Problem description"}],"source_content_type":"text/x-rst","patch_set":1,"id":"baada198_c8475b79","line":9,"updated":"2014-07-22 17:39:47.000000000","message":"Is https://pypi.python.org/pypi/mysql-connector-python the package you mean?","commit_id":"9f68be5fce158030971d363fb71d30762a234754"},{"author":{"_account_id":11279,"name":"Angus Lees","email":"gus@inodes.org","username":"gus"},"change_message_id":"fca189fadbee8bc29f51fb2fec87480549481493","unresolved":false,"context_lines":[{"line_number":6,"context_line":""},{"line_number":7,"context_line":"Introduction paragraph -- why are we doing anything?"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"Switch from MySQLdb to MySQL Connector which provides multiple benefits,"},{"line_number":10,"context_line":"including resilience of code and potential performance advantages."},{"line_number":11,"context_line":""},{"line_number":12,"context_line":"Problem description"}],"source_content_type":"text/x-rst","patch_set":1,"id":"baada198_ac714de0","line":9,"in_reply_to":"baada198_c8475b79","updated":"2014-07-23 00:39:52.000000000","message":"Yes.  Aka mysql+mysqlconnector:// in sqlalchemy.","commit_id":"9f68be5fce158030971d363fb71d30762a234754"},{"author":{"_account_id":2472,"name":"Doug Hellmann","email":"dhellmann@redhat.com","username":"doug-hellmann"},"change_message_id":"d5a33b8d921dd834564d40a0ebacfe67a6ee3456","unresolved":false,"context_lines":[{"line_number":153,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":154,"context_line":""},{"line_number":155,"context_line":"MySQL Connector should be posted to PyPI to be able to introduce it as part of"},{"line_number":156,"context_line":"global requirements list."},{"line_number":157,"context_line":""},{"line_number":158,"context_line":"References"},{"line_number":159,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":1,"id":"baada198_633bd0c8","line":156,"updated":"2014-07-22 17:39:47.000000000","message":"Having the code published on PyPI will be a requirement to accepting this, I think.\n\nCan you also comment on the license for the library, and how their exception might be seen to affect commercial distributions of OpenStack that include other customizations (like drivers)? You might need to get the legal-discuss mailing list involved in the answer to that question.","commit_id":"9f68be5fce158030971d363fb71d30762a234754"},{"author":{"_account_id":11279,"name":"Angus Lees","email":"gus@inodes.org","username":"gus"},"change_message_id":"fca189fadbee8bc29f51fb2fec87480549481493","unresolved":false,"context_lines":[{"line_number":153,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":154,"context_line":""},{"line_number":155,"context_line":"MySQL Connector should be posted to PyPI to be able to introduce it as part of"},{"line_number":156,"context_line":"global requirements list."},{"line_number":157,"context_line":""},{"line_number":158,"context_line":"References"},{"line_number":159,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":1,"id":"baada198_ef8a5763","line":156,"in_reply_to":"baada198_633bd0c8","updated":"2014-07-23 00:39:52.000000000","message":"Re license - as far as I can work out, both mysqlconnector and mysqldb are GPL2, so there\u0027s no change here.\n\nAs far as I can see the \"mysql FOSS exception\" (http://www.mysql.com/about/legal/licensing/foss-exception/) grants additional alternatives to vanilla GPL2, so is strictly more liberal than the current mysqldb license.","commit_id":"9f68be5fce158030971d363fb71d30762a234754"},{"author":{"_account_id":1030,"name":"Chris Behrens","email":"cbehrens@codestud.com","username":"cbehrens"},"change_message_id":"ffafb9b4ed626155ba508ae804f8372ad68c86c8","unresolved":false,"context_lines":[{"line_number":13,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":14,"context_line":""},{"line_number":15,"context_line":"The current MySQL client library we use (MySQLdb) plays badly with eventlet and"},{"line_number":16,"context_line":"may result in db deadlocks (see [#dead_locks]_). It also blocks execution of"},{"line_number":17,"context_line":"other green threads while we\u0027re deep in the library code (see [#one_thread]_)."},{"line_number":18,"context_line":"To avoid those issues, we need to switch to a pure python library with better"},{"line_number":19,"context_line":"eventlet support. MySQL Connector [#official_page]_ is an official MySQL Python"}],"source_content_type":"text/x-rst","patch_set":3,"id":"baada198_1e4c5475","line":16,"updated":"2014-08-05 06:19:58.000000000","message":"It really does not play badly. You generally should not be doing anything within a transaction that causes eventlet to switch greenthreads. Can you show me a case where eventlet would switch greenthreads within one our dbapi calls? (I see below the notification example. I\u0027ll comment on that below.) I think the only case is logging when using syslog, and I would think that a greenthread switch there would be rare.\n\nIf you\u0027re getting a fair amount of deadlocks, you\u0027re likely doing something else wrong. We have a lot of queries that can cause deadlocks. Deadlocks should be retried in all cases (see mysql docs)... which we\u0027ve been very bad about in general. A 2nd try is likely to succeed whether it\u0027s caused by our own queries or by eventlet.\n\nEither way, the real solution to the problem is to make it so that DB calls can run in real Threads. We have a bug filed for this (it\u0027s, surprise, an eventlet bug).  See: https://bugs.launchpad.net/nova/+bug/1171601\n\nEventlet has a preliminary patch (if you follow the PR for the eventlet issue) to fix this that we need to try. This should be our fix.","commit_id":"78cd0d14e86408fb82d74568ba88f8067b519805"},{"author":{"_account_id":9656,"name":"Ihar Hrachyshka","email":"ihrachys@redhat.com","username":"ihrachys","status":"Red Hat Networking Systems Engineer"},"change_message_id":"da976a5a39865cb3d5df94871a177b612f8f6daa","unresolved":false,"context_lines":[{"line_number":13,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":14,"context_line":""},{"line_number":15,"context_line":"The current MySQL client library we use (MySQLdb) plays badly with eventlet and"},{"line_number":16,"context_line":"may result in db deadlocks (see [#dead_locks]_). It also blocks execution of"},{"line_number":17,"context_line":"other green threads while we\u0027re deep in the library code (see [#one_thread]_)."},{"line_number":18,"context_line":"To avoid those issues, we need to switch to a pure python library with better"},{"line_number":19,"context_line":"eventlet support. MySQL Connector [#official_page]_ is an official MySQL Python"}],"source_content_type":"text/x-rst","patch_set":3,"id":"baada198_b880d99d","line":16,"in_reply_to":"baada198_1e4c5475","updated":"2014-08-05 09:22:07.000000000","message":"\u003e You generally should not be doing anything within a transaction that causes eventlet to switch greenthreads\n\u003e and I would think that a greenthread switch there would be rare\n\nRare meaning \u0027not worth fixing\u0027? I disagree. This is what I call \u0027play badly\u0027. You make developer control all the code below the stack on whether it yields. It\u0027s unrealistic. If anything, logging under transaction *is* reasonable, it yields, and results in deadlock. What could be more wrong than forbidding logging to large pieces of code?\n\n\u003e If you\u0027re getting a fair amount of deadlocks, you\u0027re likely doing something else wrong\nYes, in some cases we may try to isolate yielding code from transactions, though I wouldn\u0027t bet on any code that it doesn\u0027t yield.\n\n\u003e A 2nd try is likely to succeed whether it\u0027s caused by our own queries or by eventlet.\n\u0027Likely\u0027? This is not terms we should reason the issue in. Also, retry will be issued 50 secs after the first attempt (after mysql timeout will be reached). This introduces significant performance drop itself.\n\n\u003e Either way, the real solution to the problem is to make it so that DB calls can run in real Threads.\nI agree somewhat. Though most projects are far from that. In Neutron, we only considered that idea briefly in IRC, no progress till Kilo or even L-release expected. We must not leave projects failing with deadlocks till that time.\n\nAs for using eventlet tpool, that\u0027s fine, though I don\u0027t know whether it will play nice in face of expected asyncio adoption. Also, I wonder why tpool work is done on nova side instead of oslo.db.","commit_id":"78cd0d14e86408fb82d74568ba88f8067b519805"},{"author":{"_account_id":1030,"name":"Chris Behrens","email":"cbehrens@codestud.com","username":"cbehrens"},"change_message_id":"ffafb9b4ed626155ba508ae804f8372ad68c86c8","unresolved":false,"context_lines":[{"line_number":37,"context_line":"------------"},{"line_number":38,"context_line":""},{"line_number":39,"context_line":"Some database lock timeouts can be mitigated by refactoring code"},{"line_number":40,"context_line":"[#hack_around_notifications]_. We may also introduce special mechanisms that"},{"line_number":41,"context_line":"would reduce the chances for the current thread to yield under transaction that"},{"line_number":42,"context_line":"are one of the main causes of db deadlocks. It is not realistic to expect that"},{"line_number":43,"context_line":"we\u0027re able to nail down and fix all occurances of possible yields under"}],"source_content_type":"text/x-rst","patch_set":3,"id":"baada198_5e696cbe","line":40,"updated":"2014-08-05 06:19:58.000000000","message":"I don\u0027t view this fix as a hack. Why would you want to do any IO or anything that can potentially block while you\u0027re inside a DB transaction? That\u0027s a horrible idea to begin with, IMO, and should be avoided.","commit_id":"78cd0d14e86408fb82d74568ba88f8067b519805"},{"author":{"_account_id":9656,"name":"Ihar Hrachyshka","email":"ihrachys@redhat.com","username":"ihrachys","status":"Red Hat Networking Systems Engineer"},"change_message_id":"da976a5a39865cb3d5df94871a177b612f8f6daa","unresolved":false,"context_lines":[{"line_number":37,"context_line":"------------"},{"line_number":38,"context_line":""},{"line_number":39,"context_line":"Some database lock timeouts can be mitigated by refactoring code"},{"line_number":40,"context_line":"[#hack_around_notifications]_. We may also introduce special mechanisms that"},{"line_number":41,"context_line":"would reduce the chances for the current thread to yield under transaction that"},{"line_number":42,"context_line":"are one of the main causes of db deadlocks. It is not realistic to expect that"},{"line_number":43,"context_line":"we\u0027re able to nail down and fix all occurances of possible yields under"}],"source_content_type":"text/x-rst","patch_set":3,"id":"baada198_f87891c3","line":40,"in_reply_to":"baada198_5e696cbe","updated":"2014-08-05 09:22:07.000000000","message":"I agree it should be avoided. I disagree that it\u0027s realistic to expect that you can achieve 100% of non-db IO removed from transactions. Leaving code working in 98% of cases is wrong.","commit_id":"78cd0d14e86408fb82d74568ba88f8067b519805"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"f712ef1a929647a33fecbc10c2c99abdf1d95ad3","unresolved":false,"context_lines":[{"line_number":76,"context_line":""},{"line_number":77,"context_line":"This expectation is proved by the following measurements [#measurements]_."},{"line_number":78,"context_line":""},{"line_number":79,"context_line":"A simple performance test for Neutron was run locally on devstack to check"},{"line_number":80,"context_line":"impact of the change. Four scenarios were checked:"},{"line_number":81,"context_line":""},{"line_number":82,"context_line":"* create 200 ports in serial"}],"source_content_type":"text/x-rst","patch_set":3,"id":"baada198_606ce66b","line":79,"updated":"2014-08-04 16:30:26.000000000","message":"This is the problem with this: it\u0027s too simple of a performance test. We need a better test that looks at realistic work and payloads across the full stack of integrated services. I think you will find, as Vishy did, that changing the driver to mysql-connector won\u0027t yield the type of results below.","commit_id":"78cd0d14e86408fb82d74568ba88f8067b519805"},{"author":{"_account_id":9656,"name":"Ihar Hrachyshka","email":"ihrachys@redhat.com","username":"ihrachys","status":"Red Hat Networking Systems Engineer"},"change_message_id":"da976a5a39865cb3d5df94871a177b612f8f6daa","unresolved":false,"context_lines":[{"line_number":76,"context_line":""},{"line_number":77,"context_line":"This expectation is proved by the following measurements [#measurements]_."},{"line_number":78,"context_line":""},{"line_number":79,"context_line":"A simple performance test for Neutron was run locally on devstack to check"},{"line_number":80,"context_line":"impact of the change. Four scenarios were checked:"},{"line_number":81,"context_line":""},{"line_number":82,"context_line":"* create 200 ports in serial"}],"source_content_type":"text/x-rst","patch_set":3,"id":"baada198_bbd4534c","line":79,"in_reply_to":"baada198_606ce66b","updated":"2014-08-05 09:22:07.000000000","message":"I\u0027ve asked Red Hat performance team to look into this internally. Hopefully, they will provide some results based on real deployments in our lab.\n\nThe problem with Vishy\u0027s claims is that they are general and not reproducible. No tests or numbers or description of environment used were provided.\n\nTo begin discussion on this, I\u0027d like to hear from you what you consider a good performance test in that case. The problem with that argument for better benchmarking is, partly, that no one really mentions what should be done. There are no tools for this in the wild. Rally tests that I ran and provided results in mailing list [1][2] also don\u0027t look too deep. So we should define which test results we would consider authoritative, and then we\u0027ll make experimentation and base our decision on its results.\n\n[1]: http://lists.openstack.org/pipermail/openstack-dev/2014-July/040477.html\n[2]: http://lists.openstack.org/pipermail/openstack-dev/2014-July/040363.html","commit_id":"78cd0d14e86408fb82d74568ba88f8067b519805"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"89af907bf8a4b81a62a8293bdbc36cf380c46f61","unresolved":false,"context_lines":[{"line_number":76,"context_line":""},{"line_number":77,"context_line":"This expectation is proved by the following measurements [#measurements]_."},{"line_number":78,"context_line":""},{"line_number":79,"context_line":"A simple performance test for Neutron was run locally on devstack to check"},{"line_number":80,"context_line":"impact of the change. Four scenarios were checked:"},{"line_number":81,"context_line":""},{"line_number":82,"context_line":"* create 200 ports in serial"}],"source_content_type":"text/x-rst","patch_set":3,"id":"baada198_02494bb8","line":79,"in_reply_to":"baada198_bbd4534c","updated":"2014-08-05 13:43:02.000000000","message":"Hi Ihar, Angus,\n\nYes, I don\u0027t disagree with you that the Rally tests are not deep tests at this time. And I agree that we need to specify what performance tests need to look like. I can work on that with you and Angus, certainly.\n\nMy point above is that making a change like this should not be done lightly and should not be done based solely on micro-benchmarks that do not represent a realistic workload. That\u0027s all I was trying to say.\n\nI think it would be very much appropriate to start off this work by creating a new check job in Jenkins that would install the mysql python connector, configure the sql_connection config options to use it, and then run the same tempest tests as the normal gate job. That would give us some pretty good data on a) whether the deadlock issues that can plague the gate job with MySQLdb driver go away, and b) some rough overall performance differences of the check job times.\n\nBest,\n\n-jay","commit_id":"78cd0d14e86408fb82d74568ba88f8067b519805"},{"author":{"_account_id":11816,"name":"mike bayer","display_name":"Mike Bayer","email":"mike_mp@zzzcomputing.com","username":"zzzeek","status":"Red Hat"},"change_message_id":"226e10b8296fb910672af07d85492667665d7d58","unresolved":false,"context_lines":[{"line_number":86,"context_line":""},{"line_number":87,"context_line":"Each scenario was run three times."},{"line_number":88,"context_line":""},{"line_number":89,"context_line":"For serial scenarios, the average result is pretty the same:"},{"line_number":90,"context_line":""},{"line_number":91,"context_line":"* mysqldb (ports): 59.43 sec"},{"line_number":92,"context_line":"* mysql connector (ports): 62.81 sec"}],"source_content_type":"text/x-rst","patch_set":3,"id":"baada198_45e694e9","line":89,"updated":"2014-07-23 20:55:32.000000000","message":"this is great news for serial cases.","commit_id":"78cd0d14e86408fb82d74568ba88f8067b519805"},{"author":{"_account_id":6928,"name":"Ben Nemec","email":"openstack@nemebean.com","username":"bnemec"},"change_message_id":"e4c3ebb490b8050d66c67ffa403e72ba0bc2c29a","unresolved":false,"context_lines":[{"line_number":101,"context_line":"* mysqldb (ports): 58.37 sec"},{"line_number":102,"context_line":"* mysql connector (ports): 35.32 sec"},{"line_number":103,"context_line":"* mysqldb (networks): 215.81 sec"},{"line_number":104,"context_line":"* mysql connector (networks): 88.66 sec"},{"line_number":105,"context_line":""},{"line_number":106,"context_line":"More detailed benchmarking and scalability testing will be done in the second"},{"line_number":107,"context_line":"phase which belongs to Kilo cycle. Details on what constitutes a proper"}],"source_content_type":"text/x-rst","patch_set":4,"id":"baada198_b4746737","line":104,"updated":"2014-08-12 16:43:56.000000000","message":"Nice!","commit_id":"aa6e05af2b98ae58d6f474e40d23ba236c48aaf7"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"b880620eed610f5f68166e2d8ea25bca9c49ad03","unresolved":false,"context_lines":[{"line_number":150,"context_line":"  non-default MySQL drivers (development tracked by Angus Lees)."},{"line_number":151,"context_line":"* update devstack to enable optional support for running it against MySQL"},{"line_number":152,"context_line":"  Connector."},{"line_number":153,"context_line":"* create a separate gate check to run tempest against the new driver."},{"line_number":154,"context_line":""},{"line_number":155,"context_line":"Incubation"},{"line_number":156,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":4,"id":"baada198_e431bfef","line":153,"updated":"2014-08-05 15:37:44.000000000","message":"++ I support these work items.","commit_id":"aa6e05af2b98ae58d6f474e40d23ba236c48aaf7"},{"author":{"_account_id":8788,"name":"Miguel Angel Ajo","email":"mangelajo@redhat.com","username":"mangelajo"},"change_message_id":"e99d8ec2ee2b09339c545ca148a04f9e81f26cd4","unresolved":false,"context_lines":[{"line_number":150,"context_line":"  non-default MySQL drivers (development tracked by Angus Lees)."},{"line_number":151,"context_line":"* update devstack to enable optional support for running it against MySQL"},{"line_number":152,"context_line":"  Connector."},{"line_number":153,"context_line":"* create a separate gate check to run tempest against the new driver."},{"line_number":154,"context_line":""},{"line_number":155,"context_line":"Incubation"},{"line_number":156,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":4,"id":"baada198_7eb0c240","line":153,"in_reply_to":"baada198_e431bfef","updated":"2014-08-13 12:04:04.000000000","message":"+1","commit_id":"aa6e05af2b98ae58d6f474e40d23ba236c48aaf7"}]}