)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":6537,"name":"gordon chung","email":"gord@live.ca","username":"chungg"},"change_message_id":"858135d569792bdbba13fbe571b366226cf74dce","unresolved":false,"context_lines":[{"line_number":4,"context_line":"Commit:     Gordon Chung \u003cchungg@ca.ibm.com\u003e"},{"line_number":5,"context_line":"CommitDate: 2014-03-11 18:35:14 -0400"},{"line_number":6,"context_line":""},{"line_number":7,"context_line":"ceilometer service is working on a single thread"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"the collector only ever grabs one connection to the database at a"},{"line_number":10,"context_line":"time. this causes a massive backlog on the message queue."}],"source_content_type":"text/x-gerrit-commit-message","patch_set":1,"id":"AAAAVn%2F%2B0oM%3D","line":7,"updated":"2014-03-11 23:11:38.000000000","message":"this line?","commit_id":"d1db2c29806d1cace240cd64b1b6ad4f47df2dc6"},{"author":{"_account_id":6537,"name":"gordon chung","email":"gord@live.ca","username":"chungg"},"change_message_id":"858135d569792bdbba13fbe571b366226cf74dce","unresolved":false,"context_lines":[{"line_number":7,"context_line":"ceilometer service is working on a single thread"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"the collector only ever grabs one connection to the database at a"},{"line_number":10,"context_line":"time. this causes a massive backlog on the message queue."},{"line_number":11,"context_line":""},{"line_number":12,"context_line":"this patch reuses Change-Id: I5ccdc141c441db32fab361783a31a96015516878"},{"line_number":13,"context_line":""}],"source_content_type":"text/x-gerrit-commit-message","patch_set":1,"id":"AAAAVn%2F%2B0oI%3D","line":10,"updated":"2014-03-11 23:11:38.000000000","message":"or this line?","commit_id":"d1db2c29806d1cace240cd64b1b6ad4f47df2dc6"}],"ceilometer/service.py":[{"author":{"_account_id":6849,"name":"Roman Podoliaka","email":"roman.podoliaka@gmail.com","username":"rpodolyaka"},"change_message_id":"3a27ebfb6dfc5430ff1d4e712cda80ef427e6b5a","unresolved":false,"context_lines":[{"line_number":111,"context_line":"    # NOTE(jd) We need to monkey patch the socket and select module for,"},{"line_number":112,"context_line":"    # at least, oslo.rpc, otherwise everything\u0027s blocked on its first read()"},{"line_number":113,"context_line":"    # or select()"},{"line_number":114,"context_line":"    eventlet.monkey_patch(socket\u003dTrue, select\u003dTrue, thread\u003dTrue, time\u003dTrue)"},{"line_number":115,"context_line":"    gettextutils.install(\u0027ceilometer\u0027, lazy\u003dTrue)"},{"line_number":116,"context_line":"    rpc.set_defaults(control_exchange\u003d\u0027ceilometer\u0027)"},{"line_number":117,"context_line":"    cfg.set_defaults(log.log_opts,"}],"source_content_type":"text/x-python","patch_set":2,"id":"AAAAVn%2F%2Bzes%3D","line":114,"updated":"2014-03-12 02:33:19.000000000","message":"Patching of threading module is very important here.\n\ntl;dr: if eventlet is used, but threading isn\u0027t patched, a very small amount of trivial SQL queries processed concurrently can lead to TimeoutError and blocking of the whole process for 30s.\n\nWe\u0027ve seen a very interesting issue in Glance, which might be the same issue you are seeing here.\n\nAs I understand, you are using eventlet here, which means every request is processed by a separate green thread running within one process. SQLAlchemy uses QueuePool of size 5 by default for MySQL and PostgreSQL. Imagine that 6 simple INSERT statements are executed concurrently while processing 6 separate requests. For each of those a corresponding green thread will obtain a connection from the db connections pool, execute the statement, receive the result and retrieve the connection to the pool. What is important here, is that in case of python-mysqldb no context switch is done on IO, so DB queries block the whole process. In order to do context switches, oslo.db yields execution to another green thread right before a DB connection is returned to the pool. This basically means, that 5 concurrent INSERT requests will consume 5 available connections from the pool and all yield executing context to the 6th green thread, without actually returning a connection to the pool. \n\nAt this moment, the 6th thread will try to grab a connection from pool, but QueuePool implementation will block. SQLAlchemy uses threading.Condition implementation in order to do blocking here. And when threading isn\u0027t explicitly patched by eventlet, green thread will be blocked. The default timeout is 30s. After that the 6th thread will raise an TimeoutError exception, and yield the execution context to the 1st green thread, which will return the connection to the pool and then yield the execution context to the 2nd thread, which also will return the connection to the pool and so on. And this wouldn\u0027t happen if the 6th thread didn\u0027t block the execution of the whole process while it was waiting for db connections to be returned to the pool.\n\nSo much for using green threads...","commit_id":"834e62a35e5f118e223463c8aa62b50fd2b9b002"},{"author":{"_account_id":9562,"name":"Ildiko Vancsa","email":"ildiko.vancsa@gmail.com","username":"ildikov"},"change_message_id":"9b3dfe569890b3e180a2f1b30c2bdd15c9945d1b","unresolved":false,"context_lines":[{"line_number":111,"context_line":"    # NOTE(jd) We need to monkey patch the socket and select module for,"},{"line_number":112,"context_line":"    # at least, oslo.rpc, otherwise everything\u0027s blocked on its first read()"},{"line_number":113,"context_line":"    # or select()"},{"line_number":114,"context_line":"    eventlet.monkey_patch(socket\u003dTrue, select\u003dTrue, thread\u003dTrue, time\u003dTrue)"},{"line_number":115,"context_line":"    gettextutils.install(\u0027ceilometer\u0027, lazy\u003dTrue)"},{"line_number":116,"context_line":"    rpc.set_defaults(control_exchange\u003d\u0027ceilometer\u0027)"},{"line_number":117,"context_line":"    cfg.set_defaults(log.log_opts,"}],"source_content_type":"text/x-python","patch_set":2,"id":"AAAAVn%2F%2BxyE%3D","line":114,"in_reply_to":"AAAAVn%2F%2BysY%3D","updated":"2014-03-12 08:36:50.000000000","message":"There is already a patch from Mehdi for monkey_patching: https://review.openstack.org/#/c/79628 , it affects your modification here too. It would be good to synchronize these two patch sets.","commit_id":"834e62a35e5f118e223463c8aa62b50fd2b9b002"},{"author":{"_account_id":6849,"name":"Roman Podoliaka","email":"roman.podoliaka@gmail.com","username":"rpodolyaka"},"change_message_id":"ea1d6007eb7ea9698c4b8200af32705d45d581ca","unresolved":false,"context_lines":[{"line_number":111,"context_line":"    # NOTE(jd) We need to monkey patch the socket and select module for,"},{"line_number":112,"context_line":"    # at least, oslo.rpc, otherwise everything\u0027s blocked on its first read()"},{"line_number":113,"context_line":"    # or select()"},{"line_number":114,"context_line":"    eventlet.monkey_patch(socket\u003dTrue, select\u003dTrue, thread\u003dTrue, time\u003dTrue)"},{"line_number":115,"context_line":"    gettextutils.install(\u0027ceilometer\u0027, lazy\u003dTrue)"},{"line_number":116,"context_line":"    rpc.set_defaults(control_exchange\u003d\u0027ceilometer\u0027)"},{"line_number":117,"context_line":"    cfg.set_defaults(log.log_opts,"}],"source_content_type":"text/x-python","patch_set":2,"id":"AAAAVn%2F%2BysY%3D","line":114,"in_reply_to":"AAAAVn%2F%2Bzao%3D","updated":"2014-03-12 05:42:39.000000000","message":"1. Ensure threading is patched.\n\n2. Use multiple worker processes to run SQL queries concurrently in case of python-mysqldb (psycopg2 would work great even with one worker)","commit_id":"834e62a35e5f118e223463c8aa62b50fd2b9b002"},{"author":{"_account_id":6537,"name":"gordon chung","email":"gord@live.ca","username":"chungg"},"change_message_id":"1e277929749413a88cf08fb287218535b938d91e","unresolved":false,"context_lines":[{"line_number":111,"context_line":"    # NOTE(jd) We need to monkey patch the socket and select module for,"},{"line_number":112,"context_line":"    # at least, oslo.rpc, otherwise everything\u0027s blocked on its first read()"},{"line_number":113,"context_line":"    # or select()"},{"line_number":114,"context_line":"    eventlet.monkey_patch(socket\u003dTrue, select\u003dTrue, thread\u003dTrue, time\u003dTrue)"},{"line_number":115,"context_line":"    gettextutils.install(\u0027ceilometer\u0027, lazy\u003dTrue)"},{"line_number":116,"context_line":"    rpc.set_defaults(control_exchange\u003d\u0027ceilometer\u0027)"},{"line_number":117,"context_line":"    cfg.set_defaults(log.log_opts,"}],"source_content_type":"text/x-python","patch_set":2,"id":"AAAAVn%2F%2Bzao%3D","line":114,"in_reply_to":"AAAAVn%2F%2Bzes%3D","updated":"2014-03-12 02:41:44.000000000","message":"hmm.. interesting. i think i may have seen that happening... how was this solved in Glance?","commit_id":"834e62a35e5f118e223463c8aa62b50fd2b9b002"}],"ceilometer/storage/impl_sqlalchemy.py":[{"author":{"_account_id":6849,"name":"Roman Podoliaka","email":"roman.podoliaka@gmail.com","username":"rpodolyaka"},"change_message_id":"3a27ebfb6dfc5430ff1d4e712cda80ef427e6b5a","unresolved":false,"context_lines":[{"line_number":225,"context_line":"        self._maker \u003d sqlalchemy_session.get_maker(self._engine)"},{"line_number":226,"context_line":"        sqlalchemy_session._ENGINE \u003d None"},{"line_number":227,"context_line":"        sqlalchemy_session._MAKER \u003d None"},{"line_number":228,"context_line":"        if isinstance(self._engine.pool, pool.QueuePool):"},{"line_number":229,"context_line":"            poolsize \u003d (self._engine.pool.size() +"},{"line_number":230,"context_line":"                        self._engine.pool._max_overflow)"},{"line_number":231,"context_line":"            self.pool \u003d eventlet.GreenPool(poolsize)"}],"source_content_type":"text/x-python","patch_set":2,"id":"AAAAVn%2F%2BzeE%3D","line":228,"updated":"2014-03-12 02:33:19.000000000","message":"I don\u0027t see much value in limiting the amount of green threads here: we have 5 threads in pool (+ 5 overflow), which should be more than enough, if we don\u0027t have a lot of queries executing for more than 30s and the context switch is done properly.","commit_id":"834e62a35e5f118e223463c8aa62b50fd2b9b002"},{"author":{"_account_id":6849,"name":"Roman Podoliaka","email":"roman.podoliaka@gmail.com","username":"rpodolyaka"},"change_message_id":"ea1d6007eb7ea9698c4b8200af32705d45d581ca","unresolved":false,"context_lines":[{"line_number":225,"context_line":"        self._maker \u003d sqlalchemy_session.get_maker(self._engine)"},{"line_number":226,"context_line":"        sqlalchemy_session._ENGINE \u003d None"},{"line_number":227,"context_line":"        sqlalchemy_session._MAKER \u003d None"},{"line_number":228,"context_line":"        if isinstance(self._engine.pool, pool.QueuePool):"},{"line_number":229,"context_line":"            poolsize \u003d (self._engine.pool.size() +"},{"line_number":230,"context_line":"                        self._engine.pool._max_overflow)"},{"line_number":231,"context_line":"            self.pool \u003d eventlet.GreenPool(poolsize)"}],"source_content_type":"text/x-python","patch_set":2,"id":"AAAAVn%2F%2Bys0%3D","line":228,"in_reply_to":"AAAAVn%2F%2BzbI%3D","updated":"2014-03-12 05:42:39.000000000","message":"\u003e\u003e\u003e are you saying if we patch eventlet properly, this is a non-issue?\n\nYes, 5 + 5 overflow connections must be enough, if you don\u0027t have long running queries (\u003e30s). Especially if you use multiple workers (and this is something you want to do in case of python-mysqldb, otherwise sql queries will be run sequentially)","commit_id":"834e62a35e5f118e223463c8aa62b50fd2b9b002"},{"author":{"_account_id":6537,"name":"gordon chung","email":"gord@live.ca","username":"chungg"},"change_message_id":"1e277929749413a88cf08fb287218535b938d91e","unresolved":false,"context_lines":[{"line_number":225,"context_line":"        self._maker \u003d sqlalchemy_session.get_maker(self._engine)"},{"line_number":226,"context_line":"        sqlalchemy_session._ENGINE \u003d None"},{"line_number":227,"context_line":"        sqlalchemy_session._MAKER \u003d None"},{"line_number":228,"context_line":"        if isinstance(self._engine.pool, pool.QueuePool):"},{"line_number":229,"context_line":"            poolsize \u003d (self._engine.pool.size() +"},{"line_number":230,"context_line":"                        self._engine.pool._max_overflow)"},{"line_number":231,"context_line":"            self.pool \u003d eventlet.GreenPool(poolsize)"}],"source_content_type":"text/x-python","patch_set":2,"id":"AAAAVn%2F%2BzbI%3D","line":228,"in_reply_to":"AAAAVn%2F%2BzeE%3D","updated":"2014-03-12 02:41:44.000000000","message":"hmm.. so we originally added this because of record_metering_data(). it\u0027d perfectly normal to average about 10 records per second. because of that, we were getting queuepool errors. see https://bugs.launchpad.net/ceilometer/+bug/1243251\n\nare you saying if we patch eventlet properly, this is a non-issue?","commit_id":"834e62a35e5f118e223463c8aa62b50fd2b9b002"}]}