)]}'
{"zaqar/queues/storage/redis/claims.py":[{"author":{"_account_id":6159,"name":"Flavio Percoco Premoli","display_name":"flaper87","email":"flavio.percoco@flyrlabs.com","username":"flaper87"},"change_message_id":"832416a0487348613ae677b32944b43c6083003b","unresolved":false,"context_lines":[{"line_number":94,"context_line":"        return self.driver.queue_controller"},{"line_number":95,"context_line":""},{"line_number":96,"context_line":"    @decorators.lazy_property(write\u003dFalse)"},{"line_number":97,"context_line":"    def _scripts(self):"},{"line_number":98,"context_line":"        # NOTE(kgriffs): A watch on a pipe could also be used, but that"},{"line_number":99,"context_line":"        # is less efficient and predictable, based on our experience in having"},{"line_number":100,"context_line":"        # to do something similar in the MongoDB driver."}],"source_content_type":"text/x-python","patch_set":3,"id":"da9df570_21d97f83","line":97,"updated":"2014-09-25 07:42:11.000000000","message":"I think we could do something like this in mongodb using js scripts but I\u0027m not sure it\u0027s worth it just yet.","commit_id":"161ee47f1aa81994edff3df71fd0ed34f96ec849"},{"author":{"_account_id":6427,"name":"Kurt Griffiths","email":"kurt.griffiths@rackspace.com","username":"kgriffs"},"change_message_id":"f40b3ca99ad0a0e486db0e1b9d958d9b5cc857a6","unresolved":false,"context_lines":[{"line_number":94,"context_line":"        return self.driver.queue_controller"},{"line_number":95,"context_line":""},{"line_number":96,"context_line":"    @decorators.lazy_property(write\u003dFalse)"},{"line_number":97,"context_line":"    def _scripts(self):"},{"line_number":98,"context_line":"        # NOTE(kgriffs): A watch on a pipe could also be used, but that"},{"line_number":99,"context_line":"        # is less efficient and predictable, based on our experience in having"},{"line_number":100,"context_line":"        # to do something similar in the MongoDB driver."}],"source_content_type":"text/x-python","patch_set":3,"id":"da9df570_3e02706f","line":97,"in_reply_to":"da9df570_21d97f83","updated":"2014-09-25 15:58:37.000000000","message":"Good thought. Something to keep in mind. It couldn\u0027t hurt to do a hacky POC and benchmark it.","commit_id":"161ee47f1aa81994edff3df71fd0ed34f96ec849"},{"author":{"_account_id":6159,"name":"Flavio Percoco Premoli","display_name":"flaper87","email":"flavio.percoco@flyrlabs.com","username":"flaper87"},"change_message_id":"832416a0487348613ae677b32944b43c6083003b","unresolved":false,"context_lines":[{"line_number":98,"context_line":"        # NOTE(kgriffs): A watch on a pipe could also be used, but that"},{"line_number":99,"context_line":"        # is less efficient and predictable, based on our experience in having"},{"line_number":100,"context_line":"        # to do something similar in the MongoDB driver."},{"line_number":101,"context_line":"        claim_messages \u003d \"\"\""},{"line_number":102,"context_line":"            -- Read params"},{"line_number":103,"context_line":"            local msgset_key \u003d KEYS[1]"},{"line_number":104,"context_line":""}],"source_content_type":"text/x-python","patch_set":3,"id":"da9df570_41e84310","line":101,"updated":"2014-09-25 07:42:11.000000000","message":"How many scripts do we expect to have?\n\nWould it be worth it having a `scripts` dir inside the redis driver module where we can keep all the lua scripts in actual lua files (or python if we want to make them \u0027importable\u0027)?","commit_id":"161ee47f1aa81994edff3df71fd0ed34f96ec849"},{"author":{"_account_id":6427,"name":"Kurt Griffiths","email":"kurt.griffiths@rackspace.com","username":"kgriffs"},"change_message_id":"f40b3ca99ad0a0e486db0e1b9d958d9b5cc857a6","unresolved":false,"context_lines":[{"line_number":98,"context_line":"        # NOTE(kgriffs): A watch on a pipe could also be used, but that"},{"line_number":99,"context_line":"        # is less efficient and predictable, based on our experience in having"},{"line_number":100,"context_line":"        # to do something similar in the MongoDB driver."},{"line_number":101,"context_line":"        claim_messages \u003d \"\"\""},{"line_number":102,"context_line":"            -- Read params"},{"line_number":103,"context_line":"            local msgset_key \u003d KEYS[1]"},{"line_number":104,"context_line":""}],"source_content_type":"text/x-python","patch_set":3,"id":"da9df570_bed660ef","line":101,"in_reply_to":"da9df570_41e84310","updated":"2014-09-25 15:58:37.000000000","message":"I think that putting them in *.lua files would be nice since then your editor would syntax-highlight the code and such. That being said, I\u0027d like to just add this as a TODO comment and do it later if/when we add a bunch more scripts.","commit_id":"161ee47f1aa81994edff3df71fd0ed34f96ec849"},{"author":{"_account_id":6427,"name":"Kurt Griffiths","email":"kurt.griffiths@rackspace.com","username":"kgriffs"},"change_message_id":"f40b3ca99ad0a0e486db0e1b9d958d9b5cc857a6","unresolved":false,"context_lines":[{"line_number":110,"context_line":"            local msg_expires \u003d tonumber(ARGV[6])"},{"line_number":111,"context_line":""},{"line_number":112,"context_line":"            -- Scan for up to \u0027limit\u0027 unclaimed messages"},{"line_number":113,"context_line":"            local BATCH_SIZE \u003d 50"},{"line_number":114,"context_line":""},{"line_number":115,"context_line":"            local start \u003d 0"},{"line_number":116,"context_line":"            local claimed_msgs \u003d {}"}],"source_content_type":"text/x-python","patch_set":3,"id":"da9df570_beed80b9","line":113,"updated":"2014-09-25 15:58:37.000000000","message":"I\u0027m going to play with this BATCH_SIZE value to see it\u0027s impact on benchmarks.","commit_id":"161ee47f1aa81994edff3df71fd0ed34f96ec849"},{"author":{"_account_id":6159,"name":"Flavio Percoco Premoli","display_name":"flaper87","email":"flavio.percoco@flyrlabs.com","username":"flaper87"},"change_message_id":"832416a0487348613ae677b32944b43c6083003b","unresolved":false,"context_lines":[{"line_number":119,"context_line":""},{"line_number":120,"context_line":"            while (#claimed_msgs \u003c limit) do"},{"line_number":121,"context_line":"                local stop \u003d (start + BATCH_SIZE - 1)"},{"line_number":122,"context_line":"                local msg_ids \u003d redis.call(\u0027zrange\u0027, msgset_key, start, stop)"},{"line_number":123,"context_line":""},{"line_number":124,"context_line":"                if (#msg_ids \u003d\u003d 0) then"},{"line_number":125,"context_line":"                    break"}],"source_content_type":"text/x-python","patch_set":3,"id":"da9df570_61e38733","line":122,"updated":"2014-09-25 07:42:11.000000000","message":"On more reason to hate lua\u0027s indexes starting from 1... \u003e.\u003e","commit_id":"161ee47f1aa81994edff3df71fd0ed34f96ec849"},{"author":{"_account_id":6427,"name":"Kurt Griffiths","email":"kurt.griffiths@rackspace.com","username":"kgriffs"},"change_message_id":"f40b3ca99ad0a0e486db0e1b9d958d9b5cc857a6","unresolved":false,"context_lines":[{"line_number":119,"context_line":""},{"line_number":120,"context_line":"            while (#claimed_msgs \u003c limit) do"},{"line_number":121,"context_line":"                local stop \u003d (start + BATCH_SIZE - 1)"},{"line_number":122,"context_line":"                local msg_ids \u003d redis.call(\u0027zrange\u0027, msgset_key, start, stop)"},{"line_number":123,"context_line":""},{"line_number":124,"context_line":"                if (#msg_ids \u003d\u003d 0) then"},{"line_number":125,"context_line":"                    break"}],"source_content_type":"text/x-python","patch_set":3,"id":"da9df570_0098eb68","line":122,"in_reply_to":"da9df570_61e38733","updated":"2014-09-25 15:58:37.000000000","message":"srsly.","commit_id":"161ee47f1aa81994edff3df71fd0ed34f96ec849"},{"author":{"_account_id":6159,"name":"Flavio Percoco Premoli","display_name":"flaper87","email":"flavio.percoco@flyrlabs.com","username":"flaper87"},"change_message_id":"832416a0487348613ae677b32944b43c6083003b","unresolved":false,"context_lines":[{"line_number":131,"context_line":"                    -- NOTE(kgriffs): Since execution of this script can not"},{"line_number":132,"context_line":"                    -- happen in parallel, once we find the first unclaimed"},{"line_number":133,"context_line":"                    -- message, the remaining messages will always be"},{"line_number":134,"context_line":"                    -- unclaimed as well."},{"line_number":135,"context_line":""},{"line_number":136,"context_line":"                    if not found_unclaimed then"},{"line_number":137,"context_line":"                        local msg \u003d redis.call(\u0027hmget\u0027, mid, \u0027e\u0027, \u0027c\u0027, \u0027c.e\u0027)"}],"source_content_type":"text/x-python","patch_set":3,"id":"da9df570_81d14b59","line":134,"updated":"2014-09-25 07:42:11.000000000","message":"Is this True for multi-nodes or just a single node?\n\nAFAIU, scripts can be executed in slaves if they\u0027re loaded there. Is there a way we can make sure this scripts exist just in a master node?","commit_id":"161ee47f1aa81994edff3df71fd0ed34f96ec849"},{"author":{"_account_id":6427,"name":"Kurt Griffiths","email":"kurt.griffiths@rackspace.com","username":"kgriffs"},"change_message_id":"bf42101d8d59722cb96f2108e3ab1767cf422dd1","unresolved":false,"context_lines":[{"line_number":131,"context_line":"                    -- NOTE(kgriffs): Since execution of this script can not"},{"line_number":132,"context_line":"                    -- happen in parallel, once we find the first unclaimed"},{"line_number":133,"context_line":"                    -- message, the remaining messages will always be"},{"line_number":134,"context_line":"                    -- unclaimed as well."},{"line_number":135,"context_line":""},{"line_number":136,"context_line":"                    if not found_unclaimed then"},{"line_number":137,"context_line":"                        local msg \u003d redis.call(\u0027hmget\u0027, mid, \u0027e\u0027, \u0027c\u0027, \u0027c.e\u0027)"}],"source_content_type":"text/x-python","patch_set":3,"id":"baa201ad_b188c8e8","line":134,"in_reply_to":"baa201ad_9f403f37","updated":"2014-09-29 17:44:40.000000000","message":"I reviewed the Redis source code and it propagates an EVALSHA just like any other command. After first applying the command to the (local) master DB, the server then replays the command to the AOF and to any slaves.\n\nSince scripts are executed in-band with all other commands, the data should remain consistent between master and slave.","commit_id":"161ee47f1aa81994edff3df71fd0ed34f96ec849"},{"author":{"_account_id":6159,"name":"Flavio Percoco Premoli","display_name":"flaper87","email":"flavio.percoco@flyrlabs.com","username":"flaper87"},"change_message_id":"f46803ac925b02da5c8a7219408134b8b7ef8ef2","unresolved":false,"context_lines":[{"line_number":131,"context_line":"                    -- NOTE(kgriffs): Since execution of this script can not"},{"line_number":132,"context_line":"                    -- happen in parallel, once we find the first unclaimed"},{"line_number":133,"context_line":"                    -- message, the remaining messages will always be"},{"line_number":134,"context_line":"                    -- unclaimed as well."},{"line_number":135,"context_line":""},{"line_number":136,"context_line":"                    if not found_unclaimed then"},{"line_number":137,"context_line":"                        local msg \u003d redis.call(\u0027hmget\u0027, mid, \u0027e\u0027, \u0027c\u0027, \u0027c.e\u0027)"}],"source_content_type":"text/x-python","patch_set":3,"id":"baa201ad_abb4add1","line":134,"in_reply_to":"baa201ad_b188c8e8","updated":"2014-10-01 08:43:12.000000000","message":"Kurt, thanks a lot for taking the time. I had asked this on #redis @ freenode on monday but I ain\u0027t got any answer. \"Read the source, Luke\"\n\nI\u0027m good with this, great work!","commit_id":"161ee47f1aa81994edff3df71fd0ed34f96ec849"},{"author":{"_account_id":6159,"name":"Flavio Percoco Premoli","display_name":"flaper87","email":"flavio.percoco@flyrlabs.com","username":"flaper87"},"change_message_id":"83039129447261b636d1f86ebf38326e025443cf","unresolved":false,"context_lines":[{"line_number":131,"context_line":"                    -- NOTE(kgriffs): Since execution of this script can not"},{"line_number":132,"context_line":"                    -- happen in parallel, once we find the first unclaimed"},{"line_number":133,"context_line":"                    -- message, the remaining messages will always be"},{"line_number":134,"context_line":"                    -- unclaimed as well."},{"line_number":135,"context_line":""},{"line_number":136,"context_line":"                    if not found_unclaimed then"},{"line_number":137,"context_line":"                        local msg \u003d redis.call(\u0027hmget\u0027, mid, \u0027e\u0027, \u0027c\u0027, \u0027c.e\u0027)"}],"source_content_type":"text/x-python","patch_set":3,"id":"baa201ad_9f403f37","line":134,"in_reply_to":"da9df570_3e2a702c","updated":"2014-09-29 08:17:58.000000000","message":"Are you saying the script is always executed in the master node first and then replayed on the slave? \n\nMy concern is whether this script may be executed on the slave with a non-consistent DB.","commit_id":"161ee47f1aa81994edff3df71fd0ed34f96ec849"},{"author":{"_account_id":6427,"name":"Kurt Griffiths","email":"kurt.griffiths@rackspace.com","username":"kgriffs"},"change_message_id":"f40b3ca99ad0a0e486db0e1b9d958d9b5cc857a6","unresolved":false,"context_lines":[{"line_number":131,"context_line":"                    -- NOTE(kgriffs): Since execution of this script can not"},{"line_number":132,"context_line":"                    -- happen in parallel, once we find the first unclaimed"},{"line_number":133,"context_line":"                    -- message, the remaining messages will always be"},{"line_number":134,"context_line":"                    -- unclaimed as well."},{"line_number":135,"context_line":""},{"line_number":136,"context_line":"                    if not found_unclaimed then"},{"line_number":137,"context_line":"                        local msg \u003d redis.call(\u0027hmget\u0027, mid, \u0027e\u0027, \u0027c\u0027, \u0027c.e\u0027)"}],"source_content_type":"text/x-python","patch_set":3,"id":"da9df570_3e2a702c","line":134,"in_reply_to":"da9df570_81d14b59","updated":"2014-09-25 15:58:37.000000000","message":"Indeed, Redis will send the script to the slaves and execute it with the same params as for the master. This is done for efficiency (rather than sending the individual redis commands in the script to the secondary). Given the same starting state of the DB and given the same input to the Lua script, the resulting state will be identical across the master and the slaves. \n\nI could be wrong, but I believe that commands to the slaves are replayed in the same order they are received by the master, the starting state before running each script will be the same on all nodes.","commit_id":"161ee47f1aa81994edff3df71fd0ed34f96ec849"}]}