)]}'
{"mistral/db/sqlalchemy/migration/alembic_migrations/versions/018_increate_task_execution_unique_key_size.py":[{"author":{"_account_id":9408,"name":"Michal Gershenzon","email":"michal.gershenzon@gmail.com","username":"Mgershen"},"change_message_id":"33b60d39e74d20b7b5353e0e3de077018079df1a","unresolved":false,"context_lines":[{"line_number":30,"context_line":""},{"line_number":31,"context_line":""},{"line_number":32,"context_line":"def upgrade():"},{"line_number":33,"context_line":"    op.alter_column(\u0027task_executions_v2\u0027, \u0027unique_key\u0027, type_\u003dsa.String(250))"}],"source_content_type":"text/x-python","patch_set":13,"id":"9a89bdaa_7a406657","line":33,"updated":"2016-09-04 16:26:44.000000000","message":"might be worth mentioning this in the commit message, so no one will confuse this with dropping scheduler \u0027unique_key\u0027 column.","commit_id":"4925075da72ea71c0fe5b51921668fec4315fb49"},{"author":{"_account_id":8731,"name":"Renat Akhmerov","email":"renat.akhmerov@gmail.com","username":"rakhmerov"},"change_message_id":"f55c6cbab990a80fa3f791dc458a05f0da301291","unresolved":false,"context_lines":[{"line_number":30,"context_line":""},{"line_number":31,"context_line":""},{"line_number":32,"context_line":"def upgrade():"},{"line_number":33,"context_line":"    op.alter_column(\u0027task_executions_v2\u0027, \u0027unique_key\u0027, type_\u003dsa.String(250))"}],"source_content_type":"text/x-python","patch_set":13,"id":"9a89bdaa_a9ce2a58","line":33,"in_reply_to":"9a89bdaa_7a406657","updated":"2016-09-05 02:59:50.000000000","message":"Ok","commit_id":"4925075da72ea71c0fe5b51921668fec4315fb49"}],"mistral/db/v2/sqlalchemy/api.py":[{"author":{"_account_id":9408,"name":"Michal Gershenzon","email":"michal.gershenzon@gmail.com","username":"Mgershen"},"change_message_id":"33b60d39e74d20b7b5353e0e3de077018079df1a","unresolved":false,"context_lines":[{"line_number":1406,"context_line":"    # session may not immediately issue an SQL query to a database"},{"line_number":1407,"context_line":"    # and instead just schedule it whereas we need to make sure to"},{"line_number":1408,"context_line":"    # issue a query immediately."},{"line_number":1409,"context_line":"    session.flush()"},{"line_number":1410,"context_line":""},{"line_number":1411,"context_line":"    insert \u003d models.NamedLock.__table__.insert()"},{"line_number":1412,"context_line":""}],"source_content_type":"text/x-python","patch_set":13,"id":"9a89bdaa_ba090e34","line":1409,"updated":"2016-09-04 16:26:44.000000000","message":"can session be None? should we use try/except?","commit_id":"4925075da72ea71c0fe5b51921668fec4315fb49"},{"author":{"_account_id":8731,"name":"Renat Akhmerov","email":"renat.akhmerov@gmail.com","username":"rakhmerov"},"change_message_id":"f55c6cbab990a80fa3f791dc458a05f0da301291","unresolved":false,"context_lines":[{"line_number":1406,"context_line":"    # session may not immediately issue an SQL query to a database"},{"line_number":1407,"context_line":"    # and instead just schedule it whereas we need to make sure to"},{"line_number":1408,"context_line":"    # issue a query immediately."},{"line_number":1409,"context_line":"    session.flush()"},{"line_number":1410,"context_line":""},{"line_number":1411,"context_line":"    insert \u003d models.NamedLock.__table__.insert()"},{"line_number":1412,"context_line":""}],"source_content_type":"text/x-python","patch_set":13,"id":"9a89bdaa_e9c11230","line":1409,"in_reply_to":"9a89bdaa_ba090e34","updated":"2016-09-05 02:59:50.000000000","message":"No, in fact, it can\u0027t be None because of the decorator. It always passes some value. None is needed because we should be able to ignore this parameter when we make a call explicitly.","commit_id":"4925075da72ea71c0fe5b51921668fec4315fb49"},{"author":{"_account_id":18238,"name":"Hardik Parekh","email":"hardik.parekh047@gmail.com","username":"hparekh"},"change_message_id":"70c00e72fa2132c8387e8e445336792d270a274f","unresolved":false,"context_lines":[{"line_number":830,"context_line":"    return task_ex"},{"line_number":831,"context_line":""},{"line_number":832,"context_line":""},{"line_number":833,"context_line":"def insert_or_ignore_task_execution(values):"},{"line_number":834,"context_line":"    insert_or_ignore(models.TaskExecution, values.copy())"},{"line_number":835,"context_line":""},{"line_number":836,"context_line":""}],"source_content_type":"text/x-python","patch_set":14,"id":"9a89bdaa_bb01956c","line":833,"range":{"start_line":833,"start_character":4,"end_line":833,"end_character":35},"updated":"2016-09-07 09:27:18.000000000","message":"You have removed use of this function. I think we can remove it here.","commit_id":"aa71431546f83837c01b1eed95deda68673f3218"},{"author":{"_account_id":8731,"name":"Renat Akhmerov","email":"renat.akhmerov@gmail.com","username":"rakhmerov"},"change_message_id":"77cc1f6119878b9864ca1bc37af8064f7f1f1953","unresolved":false,"context_lines":[{"line_number":830,"context_line":"    return task_ex"},{"line_number":831,"context_line":""},{"line_number":832,"context_line":""},{"line_number":833,"context_line":"def insert_or_ignore_task_execution(values):"},{"line_number":834,"context_line":"    insert_or_ignore(models.TaskExecution, values.copy())"},{"line_number":835,"context_line":""},{"line_number":836,"context_line":""}],"source_content_type":"text/x-python","patch_set":14,"id":"9a89bdaa_145ecb43","line":833,"range":{"start_line":833,"start_character":4,"end_line":833,"end_character":35},"in_reply_to":"9a89bdaa_bb01956c","updated":"2016-09-07 11:04:03.000000000","message":"Ok","commit_id":"aa71431546f83837c01b1eed95deda68673f3218"},{"author":{"_account_id":18238,"name":"Hardik Parekh","email":"hardik.parekh047@gmail.com","username":"hparekh"},"change_message_id":"70c00e72fa2132c8387e8e445336792d270a274f","unresolved":false,"context_lines":[{"line_number":884,"context_line":"    return delayed_call"},{"line_number":885,"context_line":""},{"line_number":886,"context_line":""},{"line_number":887,"context_line":"def insert_or_ignore_delayed_call(values):"},{"line_number":888,"context_line":"    insert_or_ignore(models.DelayedCall, values.copy())"},{"line_number":889,"context_line":""},{"line_number":890,"context_line":""}],"source_content_type":"text/x-python","patch_set":14,"id":"9a89bdaa_763804f0","line":887,"range":{"start_line":887,"start_character":4,"end_line":887,"end_character":33},"updated":"2016-09-07 09:27:18.000000000","message":"You have removed use of this function. I think we can remove it here.","commit_id":"aa71431546f83837c01b1eed95deda68673f3218"},{"author":{"_account_id":8731,"name":"Renat Akhmerov","email":"renat.akhmerov@gmail.com","username":"rakhmerov"},"change_message_id":"77cc1f6119878b9864ca1bc37af8064f7f1f1953","unresolved":false,"context_lines":[{"line_number":884,"context_line":"    return delayed_call"},{"line_number":885,"context_line":""},{"line_number":886,"context_line":""},{"line_number":887,"context_line":"def insert_or_ignore_delayed_call(values):"},{"line_number":888,"context_line":"    insert_or_ignore(models.DelayedCall, values.copy())"},{"line_number":889,"context_line":""},{"line_number":890,"context_line":""}],"source_content_type":"text/x-python","patch_set":14,"id":"9a89bdaa_b46b5ff0","line":887,"range":{"start_line":887,"start_character":4,"end_line":887,"end_character":33},"in_reply_to":"9a89bdaa_763804f0","updated":"2016-09-07 11:04:03.000000000","message":"Ok","commit_id":"aa71431546f83837c01b1eed95deda68673f3218"}],"mistral/engine/dispatcher.py":[{"author":{"_account_id":18238,"name":"Hardik Parekh","email":"hardik.parekh047@gmail.com","username":"hparekh"},"change_message_id":"70c00e72fa2132c8387e8e445336792d270a274f","unresolved":false,"context_lines":[{"line_number":74,"context_line":""},{"line_number":75,"context_line":"    res.sort(key\u003dfunctools.cmp_to_key(_compare_task_commands))"},{"line_number":76,"context_line":""},{"line_number":77,"context_line":"    return res + [state_cmd]"},{"line_number":78,"context_line":""},{"line_number":79,"context_line":""},{"line_number":80,"context_line":"@profiler.trace(\u0027dispatcher-dispatch-commands\u0027)"}],"source_content_type":"text/x-python","patch_set":14,"id":"9a89bdaa_156b3286","line":77,"range":{"start_line":77,"start_character":11,"end_line":77,"end_character":28},"updated":"2016-09-07 09:27:18.000000000","message":"How about using res.append(state_cmd) instead of extensing list.","commit_id":"aa71431546f83837c01b1eed95deda68673f3218"},{"author":{"_account_id":8731,"name":"Renat Akhmerov","email":"renat.akhmerov@gmail.com","username":"rakhmerov"},"change_message_id":"77cc1f6119878b9864ca1bc37af8064f7f1f1953","unresolved":false,"context_lines":[{"line_number":74,"context_line":""},{"line_number":75,"context_line":"    res.sort(key\u003dfunctools.cmp_to_key(_compare_task_commands))"},{"line_number":76,"context_line":""},{"line_number":77,"context_line":"    return res + [state_cmd]"},{"line_number":78,"context_line":""},{"line_number":79,"context_line":""},{"line_number":80,"context_line":"@profiler.trace(\u0027dispatcher-dispatch-commands\u0027)"}],"source_content_type":"text/x-python","patch_set":14,"id":"9a89bdaa_0f13165c","line":77,"range":{"start_line":77,"start_character":11,"end_line":77,"end_character":28},"in_reply_to":"9a89bdaa_156b3286","updated":"2016-09-07 11:04:03.000000000","message":"Yeah, sure.","commit_id":"aa71431546f83837c01b1eed95deda68673f3218"}],"mistral/engine/rpc_backend/oslo/oslo_server.py":[{"author":{"_account_id":8731,"name":"Renat Akhmerov","email":"renat.akhmerov@gmail.com","username":"rakhmerov"},"change_message_id":"b5358ec4c81c9cb05632e10e584b374b54a5794f","unresolved":false,"context_lines":[{"line_number":48,"context_line":"            rpc.get_transport(),"},{"line_number":49,"context_line":"            target,"},{"line_number":50,"context_line":"            self.endpoints,"},{"line_number":51,"context_line":"            executor\u003d\u0027blocking\u0027,"},{"line_number":52,"context_line":"            serializer\u003dctx.RpcContextSerializer(ctx.JsonPayloadSerializer())"},{"line_number":53,"context_line":"        )"},{"line_number":54,"context_line":""}],"source_content_type":"text/x-python","patch_set":12,"id":"fa7ab95a_a6f49587","line":51,"range":{"start_line":51,"start_character":22,"end_line":51,"end_character":30},"updated":"2016-08-31 10:55:31.000000000","message":"My explanation why I switched to \u0027blocking\u0027.\n\nWe have to switch to \u0027blocking\u0027 executor for now because \u0027eventlet\u0027 doesn\u0027t work for us in case of heavy load. Whenever we use some external locking (like db_api.named_lock()) we can\u0027t really use eventlet\u0027s green threads because it doesn\u0027t switch between greenlets in all needed cases like external calls to DB. And that in turn leads to a situation when TX A acquired a lock L, then greenlet switched to TX B and it tries to acquire lock L and hangs. Greenlet, however, never switch threads back to TX A so that it releases the lock. And therefore all threads are just stuck until an external call times out.\n\nI easily reproduced this behavior with a simple example that you can see at http://paste.openstack.org/show/564749/. This code deadlocks on iteration #2.\n\nThe real reason is the lack of true parallelism (with real parallel threads TX A would just continue and release the lock) or maybe we have some misconfiguration of eventlet which prevents it from correct threads switching.\n\nIt still needs to be further investigated but once we can use more than 1 engine this problem won\u0027t be severe because we\u0027ll be able to easily scale the system. Now there\u0027s a couple of known issues in Scheduler and because of that we can\u0027t use multiple engines under load.\n\nAny thoughts and ideas would be very helpful.","commit_id":"53042cb4010036b3807e9841354e0b98429b9c5d"},{"author":{"_account_id":18238,"name":"Hardik Parekh","email":"hardik.parekh047@gmail.com","username":"hparekh"},"change_message_id":"70c00e72fa2132c8387e8e445336792d270a274f","unresolved":false,"context_lines":[{"line_number":48,"context_line":"            rpc.get_transport(),"},{"line_number":49,"context_line":"            target,"},{"line_number":50,"context_line":"            self.endpoints,"},{"line_number":51,"context_line":"            executor\u003d\u0027blocking\u0027,"},{"line_number":52,"context_line":"            serializer\u003dctx.RpcContextSerializer(ctx.JsonPayloadSerializer())"},{"line_number":53,"context_line":"        )"},{"line_number":54,"context_line":""}],"source_content_type":"text/x-python","patch_set":14,"id":"9a89bdaa_701f2c6c","line":51,"range":{"start_line":51,"start_character":22,"end_line":51,"end_character":30},"updated":"2016-09-07 09:27:18.000000000","message":"Why are we use blocking executor ?","commit_id":"aa71431546f83837c01b1eed95deda68673f3218"},{"author":{"_account_id":8731,"name":"Renat Akhmerov","email":"renat.akhmerov@gmail.com","username":"rakhmerov"},"change_message_id":"77cc1f6119878b9864ca1bc37af8064f7f1f1953","unresolved":false,"context_lines":[{"line_number":48,"context_line":"            rpc.get_transport(),"},{"line_number":49,"context_line":"            target,"},{"line_number":50,"context_line":"            self.endpoints,"},{"line_number":51,"context_line":"            executor\u003d\u0027blocking\u0027,"},{"line_number":52,"context_line":"            serializer\u003dctx.RpcContextSerializer(ctx.JsonPayloadSerializer())"},{"line_number":53,"context_line":"        )"},{"line_number":54,"context_line":""}],"source_content_type":"text/x-python","patch_set":14,"id":"9a89bdaa_0f6876b8","line":51,"range":{"start_line":51,"start_character":22,"end_line":51,"end_character":30},"in_reply_to":"9a89bdaa_701f2c6c","updated":"2016-09-07 11:04:03.000000000","message":"Yes, valid question. Please take a look at my explanation provided in patchset 12: https://review.openstack.org/#/c/356343/12/mistral/engine/rpc_backend/oslo/oslo_server.py","commit_id":"aa71431546f83837c01b1eed95deda68673f3218"}]}