)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":9234,"name":"Steve Heyman","email":"steve@heyman.com","username":"hockeynut"},"change_message_id":"1fb55076e254a42f4979b71c7d5a25d41448f030","unresolved":false,"context_lines":[{"line_number":10,"context_line":"yet recoverable tasks (such as when remote systems are unavailable) and"},{"line_number":11,"context_line":"for handling updates for long-running order processes such as"},{"line_number":12,"context_line":"certificate generation. This blueprint defines the requirements for"},{"line_number":13,"context_line":"this retry and update processing, and proposes an implementation to add"},{"line_number":14,"context_line":"this feature."},{"line_number":15,"context_line":""},{"line_number":16,"context_line":"Change-Id: Ifce76a6c6b279111f8476e6804db07c519f59a08"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":4,"id":"3a961159_d0323c5e","line":13,"updated":"2014-12-10 23:27:38.000000000","message":"Would it be better to limit this to just retry and put the progress check functionality in a follow-on?","commit_id":"eb432093e73f48bd4bbea78577e7367018f2e0af"}],"specs/kilo/add-worker-retry-update-support.rst":[{"author":{"_account_id":10873,"name":"Juan Antonio Osorio Robles","email":"jaosorior@redhat.com","username":"ejuaoso"},"change_message_id":"90db114a8ad9fcc45d15439cf0959cadeecda7f7","unresolved":false,"context_lines":[{"line_number":50,"context_line":"client updates to that same order. This will be addressed in a future"},{"line_number":51,"context_line":"blueprint."},{"line_number":52,"context_line":""},{"line_number":53,"context_line":"Note also that this blueprint does handle with entities that are \u0027stuck\u0027 in the"},{"line_number":54,"context_line":"PENDING state because of lost messages in the queue or workers that crash"},{"line_number":55,"context_line":"while processing an entity. This will also be addressed in a future blueprint."},{"line_number":56,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"baa201ad_a5a106ab","line":53,"updated":"2014-10-14 14:25:57.000000000","message":"does or does not :/\n\nHere you say it does handle stuck entities but in the end say that it will be handled in a future blueprint.","commit_id":"470a8d9d38ef511376748f084a691bc2991e9ba7"},{"author":{"_account_id":7789,"name":"John Wood","email":"john.wood@rackspace.com","username":"woodster"},"change_message_id":"e61db2c8084f0a85ae7ef5e4fb6a7b6514eac38e","unresolved":false,"context_lines":[{"line_number":50,"context_line":"client updates to that same order. This will be addressed in a future"},{"line_number":51,"context_line":"blueprint."},{"line_number":52,"context_line":""},{"line_number":53,"context_line":"Note also that this blueprint does handle with entities that are \u0027stuck\u0027 in the"},{"line_number":54,"context_line":"PENDING state because of lost messages in the queue or workers that crash"},{"line_number":55,"context_line":"while processing an entity. This will also be addressed in a future blueprint."},{"line_number":56,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"baa201ad_6218e975","line":53,"in_reply_to":"baa201ad_a5a106ab","updated":"2014-10-14 18:47:04.000000000","message":"You are correct Juan, thanks.","commit_id":"470a8d9d38ef511376748f084a691bc2991e9ba7"},{"author":{"_account_id":10873,"name":"Juan Antonio Osorio Robles","email":"jaosorior@redhat.com","username":"ejuaoso"},"change_message_id":"90db114a8ad9fcc45d15439cf0959cadeecda7f7","unresolved":false,"context_lines":[{"line_number":145,"context_line":"    3) created_at: When this table was locked."},{"line_number":146,"context_line":""},{"line_number":147,"context_line":"This entity would only have zero or one records. So the periodic method above"},{"line_number":148,"context_line":"would execute the following psuedo code::"},{"line_number":149,"context_line":""},{"line_number":150,"context_line":"    Start SQLAlchemy session/transaction"},{"line_number":151,"context_line":"    try:"}],"source_content_type":"text/x-rst","patch_set":1,"id":"baa201ad_2a0c0542","line":148,"updated":"2014-10-14 14:25:57.000000000","message":"s/psuedo/pseudo","commit_id":"470a8d9d38ef511376748f084a691bc2991e9ba7"},{"author":{"_account_id":10873,"name":"Juan Antonio Osorio Robles","email":"jaosorior@redhat.com","username":"ejuaoso"},"change_message_id":"90db114a8ad9fcc45d15439cf0959cadeecda7f7","unresolved":false,"context_lines":[{"line_number":218,"context_line":"either long-running orders that can suffer delays until locks are restored, or"},{"line_number":219,"context_line":"(hopefully) rare conditions when resources aren\u0027t available, this condition"},{"line_number":220,"context_line":"should not be critical to resolve. The proposal does suggest a means to remove"},{"line_number":221,"context_line":"stuck locks utilizing their created-at times."},{"line_number":222,"context_line":""},{"line_number":223,"context_line":"Other deployer impact"},{"line_number":224,"context_line":"---------------------"}],"source_content_type":"text/x-rst","patch_set":1,"id":"baa201ad_d050860c","line":221,"updated":"2014-10-14 14:25:57.000000000","message":"Since these locks will be \"simulated\" or implemented as a data model entity, seems to me that this will be a huge performance bottle neck. I would like to further discuss alternatives... Since this is a concern for me. Perhaps we should do so in the summit?","commit_id":"470a8d9d38ef511376748f084a691bc2991e9ba7"},{"author":{"_account_id":7789,"name":"John Wood","email":"john.wood@rackspace.com","username":"woodster"},"change_message_id":"e61db2c8084f0a85ae7ef5e4fb6a7b6514eac38e","unresolved":false,"context_lines":[{"line_number":218,"context_line":"either long-running orders that can suffer delays until locks are restored, or"},{"line_number":219,"context_line":"(hopefully) rare conditions when resources aren\u0027t available, this condition"},{"line_number":220,"context_line":"should not be critical to resolve. The proposal does suggest a means to remove"},{"line_number":221,"context_line":"stuck locks utilizing their created-at times."},{"line_number":222,"context_line":""},{"line_number":223,"context_line":"Other deployer impact"},{"line_number":224,"context_line":"---------------------"}],"source_content_type":"text/x-rst","patch_set":1,"id":"baa201ad_e27f7964","line":221,"in_reply_to":"baa201ad_d050860c","updated":"2014-10-14 18:47:04.000000000","message":"Juan, keep in mind that these locks are only utilized on the worker node side of things, processing asynchronous tasks only, which are not time sensitive processes. Also, the time the lock is utilized will be very brief, just long enough to perform a query on retry tasks and to send those tasks to queue for follow on processing.\n\nThat said, the intent of this blueprint is indeed to spur discussion in front of and during the Kilo design summit.","commit_id":"470a8d9d38ef511376748f084a691bc2991e9ba7"},{"author":{"_account_id":10873,"name":"Juan Antonio Osorio Robles","email":"jaosorior@redhat.com","username":"ejuaoso"},"change_message_id":"9aea7a07e6b7054ab01cda23bf2f92b64935f5a2","unresolved":false,"context_lines":[{"line_number":218,"context_line":"either long-running orders that can suffer delays until locks are restored, or"},{"line_number":219,"context_line":"(hopefully) rare conditions when resources aren\u0027t available, this condition"},{"line_number":220,"context_line":"should not be critical to resolve. The proposal does suggest a means to remove"},{"line_number":221,"context_line":"stuck locks utilizing their created-at times."},{"line_number":222,"context_line":""},{"line_number":223,"context_line":"Other deployer impact"},{"line_number":224,"context_line":"---------------------"}],"source_content_type":"text/x-rst","patch_set":1,"id":"baa201ad_1697513b","line":221,"in_reply_to":"baa201ad_e27f7964","updated":"2014-10-15 07:21:55.000000000","message":"Well, that makes sense, I just wanted to talk about the use case of this, and if it makes sense, then this is alright. That\u0027s actually my only concern before +1ing this commit","commit_id":"470a8d9d38ef511376748f084a691bc2991e9ba7"},{"author":{"_account_id":9914,"name":"Ade Lee","email":"alee@redhat.com","username":"alee"},"change_message_id":"b2d3cbab80729e51995f9be0507edeec00fd7ee9","unresolved":false,"context_lines":[{"line_number":82,"context_line":"         retried in 5 minutes, it would be acceptable if the task was actually"},{"line_number":83,"context_line":"         retried after more than 5 minutes. This allows for more granular retry"},{"line_number":84,"context_line":"         checking intervals, and to allow for delays due to excessive tasks in"},{"line_number":85,"context_line":"         queues during busy times."},{"line_number":86,"context_line":""},{"line_number":87,"context_line":"Proposed Change"},{"line_number":88,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9aa7fdbe_e9af94e1","line":85,"updated":"2014-10-21 20:36:25.000000000","message":"Should there be some sort of warning mechanism if the retry time exceeds a certain threshold?  If a task is supposed to be retried in  5 minutes and it only happens in 15 minutes, perhaps more worker threads are needed?","commit_id":"cee18e672eadb359f5eeb4454a5023d32a48c4bf"},{"author":{"_account_id":7789,"name":"John Wood","email":"john.wood@rackspace.com","username":"woodster"},"change_message_id":"42ae8aecf85fa9b174bdc20b597034a82a18e108","unresolved":false,"context_lines":[{"line_number":82,"context_line":"         retried in 5 minutes, it would be acceptable if the task was actually"},{"line_number":83,"context_line":"         retried after more than 5 minutes. This allows for more granular retry"},{"line_number":84,"context_line":"         checking intervals, and to allow for delays due to excessive tasks in"},{"line_number":85,"context_line":"         queues during busy times."},{"line_number":86,"context_line":""},{"line_number":87,"context_line":"Proposed Change"},{"line_number":88,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":3,"id":"5a890539_dac7ab10","line":85,"in_reply_to":"5a890539_7744db49","updated":"2014-12-02 22:54:28.000000000","message":"At Rackspace anyway, we plan to monitor the individual worker nodes at the system level (cpu %, disk utilization, etc.). I\u0027m thinking this should be sufficient, rather than also trying to monitor deltas between when tasks are expected to be retried versus actually retried. Also, since these retry tasks are not time sensitive, I\u0027m not too concerned about these deltas at this point.","commit_id":"cee18e672eadb359f5eeb4454a5023d32a48c4bf"},{"author":{"_account_id":7789,"name":"John Wood","email":"john.wood@rackspace.com","username":"woodster"},"change_message_id":"0d0323583dffc5ff2a0f1c848f3afbfe997076c5","unresolved":false,"context_lines":[{"line_number":82,"context_line":"         retried in 5 minutes, it would be acceptable if the task was actually"},{"line_number":83,"context_line":"         retried after more than 5 minutes. This allows for more granular retry"},{"line_number":84,"context_line":"         checking intervals, and to allow for delays due to excessive tasks in"},{"line_number":85,"context_line":"         queues during busy times."},{"line_number":86,"context_line":""},{"line_number":87,"context_line":"Proposed Change"},{"line_number":88,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9aa7fdbe_873928a6","line":85,"in_reply_to":"9aa7fdbe_7cdc0074","updated":"2014-10-24 23:07:36.000000000","message":"Yes, I tend to think monitoring plus auto scaling would help out here.","commit_id":"cee18e672eadb359f5eeb4454a5023d32a48c4bf"},{"author":{"_account_id":10873,"name":"Juan Antonio Osorio Robles","email":"jaosorior@redhat.com","username":"ejuaoso"},"change_message_id":"9e05bd4b8eac9eec68df8ba4b5a6a5d609836918","unresolved":false,"context_lines":[{"line_number":82,"context_line":"         retried in 5 minutes, it would be acceptable if the task was actually"},{"line_number":83,"context_line":"         retried after more than 5 minutes. This allows for more granular retry"},{"line_number":84,"context_line":"         checking intervals, and to allow for delays due to excessive tasks in"},{"line_number":85,"context_line":"         queues during busy times."},{"line_number":86,"context_line":""},{"line_number":87,"context_line":"Proposed Change"},{"line_number":88,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":3,"id":"5a890539_7744db49","line":85,"in_reply_to":"9aa7fdbe_873928a6","updated":"2014-11-17 11:42:19.000000000","message":"Is there any proposal for integrating monitoring?","commit_id":"cee18e672eadb359f5eeb4454a5023d32a48c4bf"},{"author":{"_account_id":7262,"name":"John Vrbanac","email":"john.vrbanac@linux.com","username":"john.vrbanac"},"change_message_id":"4063592f3148fcc461ba1f8267f06329548a22c1","unresolved":false,"context_lines":[{"line_number":82,"context_line":"         retried in 5 minutes, it would be acceptable if the task was actually"},{"line_number":83,"context_line":"         retried after more than 5 minutes. This allows for more granular retry"},{"line_number":84,"context_line":"         checking intervals, and to allow for delays due to excessive tasks in"},{"line_number":85,"context_line":"         queues during busy times."},{"line_number":86,"context_line":""},{"line_number":87,"context_line":"Proposed Change"},{"line_number":88,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9aa7fdbe_7cdc0074","line":85,"in_reply_to":"9aa7fdbe_e9af94e1","updated":"2014-10-23 15:29:25.000000000","message":"Ade, would this be more of a monitoring story rather than a feature story? If the queue or the workers are loaded down, then your application monitoring should alert you to that fact, so I\u0027m not sure we should be baking a warning mechanism into Barbican itself.","commit_id":"cee18e672eadb359f5eeb4454a5023d32a48c4bf"},{"author":{"_account_id":9914,"name":"Ade Lee","email":"alee@redhat.com","username":"alee"},"change_message_id":"b2d3cbab80729e51995f9be0507edeec00fd7ee9","unresolved":false,"context_lines":[{"line_number":95,"context_line":"To implement this scheduling process, this blueprint proposes using the Oslo"},{"line_number":96,"context_line":"periodic task feature for each worker node that invoke a method on a scheduled"},{"line_number":97,"context_line":"basis (configurable, say every 15 seconds). This method would then query which"},{"line_number":98,"context_line":"tasks need to be retried (if current time \u003e\u003d retry time), and for each one"},{"line_number":99,"context_line":"issue a retry task message to the queue. Once tasks are enqueued, this method"},{"line_number":100,"context_line":"would remove the retry records from the retry list. Eventually the queue would"},{"line_number":101,"context_line":"invoke workers to implement these retry tasks."}],"source_content_type":"text/x-rst","patch_set":3,"id":"9aa7fdbe_692c8403","line":98,"updated":"2014-10-21 20:36:25.000000000","message":"Can you provide a reference for this \"periodic task feature\"?","commit_id":"cee18e672eadb359f5eeb4454a5023d32a48c4bf"},{"author":{"_account_id":10873,"name":"Juan Antonio Osorio Robles","email":"jaosorior@redhat.com","username":"ejuaoso"},"change_message_id":"9e05bd4b8eac9eec68df8ba4b5a6a5d609836918","unresolved":false,"context_lines":[{"line_number":95,"context_line":"To implement this scheduling process, this blueprint proposes using the Oslo"},{"line_number":96,"context_line":"periodic task feature for each worker node that invoke a method on a scheduled"},{"line_number":97,"context_line":"basis (configurable, say every 15 seconds). This method would then query which"},{"line_number":98,"context_line":"tasks need to be retried (if current time \u003e\u003d retry time), and for each one"},{"line_number":99,"context_line":"issue a retry task message to the queue. Once tasks are enqueued, this method"},{"line_number":100,"context_line":"would remove the retry records from the retry list. Eventually the queue would"},{"line_number":101,"context_line":"invoke workers to implement these retry tasks."}],"source_content_type":"text/x-rst","patch_set":3,"id":"5a890539_37ca53a1","line":98,"in_reply_to":"9aa7fdbe_476020cf","updated":"2014-11-17 11:42:19.000000000","message":"Would sure use that reference :D","commit_id":"cee18e672eadb359f5eeb4454a5023d32a48c4bf"},{"author":{"_account_id":7789,"name":"John Wood","email":"john.wood@rackspace.com","username":"woodster"},"change_message_id":"0d0323583dffc5ff2a0f1c848f3afbfe997076c5","unresolved":false,"context_lines":[{"line_number":95,"context_line":"To implement this scheduling process, this blueprint proposes using the Oslo"},{"line_number":96,"context_line":"periodic task feature for each worker node that invoke a method on a scheduled"},{"line_number":97,"context_line":"basis (configurable, say every 15 seconds). This method would then query which"},{"line_number":98,"context_line":"tasks need to be retried (if current time \u003e\u003d retry time), and for each one"},{"line_number":99,"context_line":"issue a retry task message to the queue. Once tasks are enqueued, this method"},{"line_number":100,"context_line":"would remove the retry records from the retry list. Eventually the queue would"},{"line_number":101,"context_line":"invoke workers to implement these retry tasks."}],"source_content_type":"text/x-rst","patch_set":3,"id":"9aa7fdbe_476020cf","line":98,"in_reply_to":"9aa7fdbe_692c8403","updated":"2014-10-24 23:07:36.000000000","message":"For sure, thanks.","commit_id":"cee18e672eadb359f5eeb4454a5023d32a48c4bf"},{"author":{"_account_id":9914,"name":"Ade Lee","email":"alee@redhat.com","username":"alee"},"change_message_id":"b2d3cbab80729e51995f9be0507edeec00fd7ee9","unresolved":false,"context_lines":[{"line_number":109,"context_line":""},{"line_number":110,"context_line":"This blueprint proposes adding the following new entities to the data model:"},{"line_number":111,"context_line":"OrderRetryTask, EntityLock, and OrderRecordLock."},{"line_number":112,"context_line":""},{"line_number":113,"context_line":"The OrderRetryTask entity would manage which tasks need to be retried on which"},{"line_number":114,"context_line":"entities, and would have the following attributes::"},{"line_number":115,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"9aa7fdbe_694ba487","line":112,"updated":"2014-10-21 20:36:25.000000000","message":"There is no discussion below about the OrderRecordLock table.","commit_id":"cee18e672eadb359f5eeb4454a5023d32a48c4bf"},{"author":{"_account_id":7789,"name":"John Wood","email":"john.wood@rackspace.com","username":"woodster"},"change_message_id":"0d0323583dffc5ff2a0f1c848f3afbfe997076c5","unresolved":false,"context_lines":[{"line_number":109,"context_line":""},{"line_number":110,"context_line":"This blueprint proposes adding the following new entities to the data model:"},{"line_number":111,"context_line":"OrderRetryTask, EntityLock, and OrderRecordLock."},{"line_number":112,"context_line":""},{"line_number":113,"context_line":"The OrderRetryTask entity would manage which tasks need to be retried on which"},{"line_number":114,"context_line":"entities, and would have the following attributes::"},{"line_number":115,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"9aa7fdbe_27679cea","line":112,"in_reply_to":"9aa7fdbe_694ba487","updated":"2014-10-24 23:07:36.000000000","message":"Oh, that is a leftover that needs to be removed, sorry.","commit_id":"cee18e672eadb359f5eeb4454a5023d32a48c4bf"},{"author":{"_account_id":994,"name":"Arvind Tiwari","email":"arvindt7@gmail.com","username":"arvind-tiwari"},"change_message_id":"ae9b39a42ee086b4ecca370e9b6b1e462e3a19b3","unresolved":false,"context_lines":[{"line_number":117,"context_line":""},{"line_number":118,"context_line":"    2) order_id: FK to the order record the retry task is intended for"},{"line_number":119,"context_line":""},{"line_number":120,"context_line":"    3) retry_task: The RPC method to invoke for the retry"},{"line_number":121,"context_line":""},{"line_number":122,"context_line":"    4) retry_at: The timestamp at or after which to retry the task"},{"line_number":123,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"9aa7fdbe_cfbaa7bf","line":120,"updated":"2014-10-22 22:51:06.000000000","message":"Wondering do we really need such a fine grained support for retry?\nCan we just set a flag on order and retry the entire order?","commit_id":"cee18e672eadb359f5eeb4454a5023d32a48c4bf"},{"author":{"_account_id":7789,"name":"John Wood","email":"john.wood@rackspace.com","username":"woodster"},"change_message_id":"0d0323583dffc5ff2a0f1c848f3afbfe997076c5","unresolved":false,"context_lines":[{"line_number":117,"context_line":""},{"line_number":118,"context_line":"    2) order_id: FK to the order record the retry task is intended for"},{"line_number":119,"context_line":""},{"line_number":120,"context_line":"    3) retry_task: The RPC method to invoke for the retry"},{"line_number":121,"context_line":""},{"line_number":122,"context_line":"    4) retry_at: The timestamp at or after which to retry the task"},{"line_number":123,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"9aa7fdbe_8784c85e","line":120,"in_reply_to":"9aa7fdbe_cfbaa7bf","updated":"2014-10-24 23:07:36.000000000","message":"The retry_task specifies which method should be retried on the order, and this isn\u0027t necessarily the same method that was invoked on this order when the retry was configured. For example, when starting a certificate order workflow, the init order establishes an order with a remove CA. If that CA doesn\u0027t create a cert right away, then we need to check status with that CA later. Hence that init cert method would indicate that the update cert order method should be invoked when this order record is retried later.\n\nDoes that make sense?","commit_id":"cee18e672eadb359f5eeb4454a5023d32a48c4bf"},{"author":{"_account_id":10873,"name":"Juan Antonio Osorio Robles","email":"jaosorior@redhat.com","username":"ejuaoso"},"change_message_id":"9e05bd4b8eac9eec68df8ba4b5a6a5d609836918","unresolved":false,"context_lines":[{"line_number":174,"context_line":""},{"line_number":175,"context_line":"To separate coding concerns, it makes sense to implement this process in a"},{"line_number":176,"context_line":"separate Oslo server process, similar to the `Keystone listener approach"},{"line_number":177,"context_line":"\u003chttps://github.com/openstack/barbican/blob/master/barbican/queue/keystone_listener.py#L130\u003e`_"},{"line_number":178,"context_line":""},{"line_number":179,"context_line":"Alternatives"},{"line_number":180,"context_line":"------------"}],"source_content_type":"text/x-rst","patch_set":3,"id":"5a890539_77e05b29","line":177,"updated":"2014-11-17 11:42:19.000000000","message":"I\u0027m guessing this refers to a notifier process that will signal the pending workers that a certain entity has been released? Or will the task not be in a locked state, and just exit and retry later if the entity is taken? The real question is:\n\nA task B attempts to access an entity, but the entity is taken (by task A), so task B cannot acquire the lock. What happens? does the task B just goes to a SLEEP state until the entity is released? or does it exit and a retry is scheduled?","commit_id":"cee18e672eadb359f5eeb4454a5023d32a48c4bf"},{"author":{"_account_id":7789,"name":"John Wood","email":"john.wood@rackspace.com","username":"woodster"},"change_message_id":"42ae8aecf85fa9b174bdc20b597034a82a18e108","unresolved":false,"context_lines":[{"line_number":174,"context_line":""},{"line_number":175,"context_line":"To separate coding concerns, it makes sense to implement this process in a"},{"line_number":176,"context_line":"separate Oslo server process, similar to the `Keystone listener approach"},{"line_number":177,"context_line":"\u003chttps://github.com/openstack/barbican/blob/master/barbican/queue/keystone_listener.py#L130\u003e`_"},{"line_number":178,"context_line":""},{"line_number":179,"context_line":"Alternatives"},{"line_number":180,"context_line":"------------"}],"source_content_type":"text/x-rst","patch_set":3,"id":"5a890539_fa4f0790","line":177,"in_reply_to":"5a890539_77e05b29","updated":"2014-12-02 22:54:28.000000000","message":"Task B gives up and leaves/goes-idle...Task A handles the processing. If for some reason Task A didn\u0027t handle things, the next time Task B runs (or Task A if on a different worker node) things would be retried. Because all state is persisted and managed via the database, the next available node to check for retries will deal with things. Does that makes sense?","commit_id":"cee18e672eadb359f5eeb4454a5023d32a48c4bf"},{"author":{"_account_id":9914,"name":"Ade Lee","email":"alee@redhat.com","username":"alee"},"change_message_id":"b2d3cbab80729e51995f9be0507edeec00fd7ee9","unresolved":false,"context_lines":[{"line_number":248,"context_line":"A configuration parameter called \u0027retry_lock_timeout_seconds\u0027 would be used to"},{"line_number":249,"context_line":"release \u0027stuck\u0027 locks on the retry tasks table. This parameter would also be"},{"line_number":250,"context_line":"added to the \u0027[scheduler]\u0027 group."},{"line_number":251,"context_line":""},{"line_number":252,"context_line":"A configuration parameter called \u0027delay_before_update_seconds\u0027 would be used to"},{"line_number":253,"context_line":"configure the amount of time the SimpleCertificatePlugin delays from"},{"line_number":254,"context_line":"initiating a demo certificate order to the time the update certificate method"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9aa7fdbe_043bdfd1","line":251,"updated":"2014-10-21 20:36:25.000000000","message":"How does this work?","commit_id":"cee18e672eadb359f5eeb4454a5023d32a48c4bf"},{"author":{"_account_id":7789,"name":"John Wood","email":"john.wood@rackspace.com","username":"woodster"},"change_message_id":"0d0323583dffc5ff2a0f1c848f3afbfe997076c5","unresolved":false,"context_lines":[{"line_number":248,"context_line":"A configuration parameter called \u0027retry_lock_timeout_seconds\u0027 would be used to"},{"line_number":249,"context_line":"release \u0027stuck\u0027 locks on the retry tasks table. This parameter would also be"},{"line_number":250,"context_line":"added to the \u0027[scheduler]\u0027 group."},{"line_number":251,"context_line":""},{"line_number":252,"context_line":"A configuration parameter called \u0027delay_before_update_seconds\u0027 would be used to"},{"line_number":253,"context_line":"configure the amount of time the SimpleCertificatePlugin delays from"},{"line_number":254,"context_line":"initiating a demo certificate order to the time the update certificate method"}],"source_content_type":"text/x-rst","patch_set":3,"id":"9aa7fdbe_47bca006","line":251,"in_reply_to":"9aa7fdbe_043bdfd1","updated":"2014-10-24 23:07:36.000000000","message":"This would be a read of the EntityLock table if the lock/insert attempt fails...if current time minus created_time \u003e retry_lock_timeout_seconds then assume the lock is stuck and update the record with the worker\u0027s host name (so steal the lock).  I can add that to the proposed section above.","commit_id":"cee18e672eadb359f5eeb4454a5023d32a48c4bf"},{"author":{"_account_id":10873,"name":"Juan Antonio Osorio Robles","email":"jaosorior@redhat.com","username":"ejuaoso"},"change_message_id":"9e05bd4b8eac9eec68df8ba4b5a6a5d609836918","unresolved":false,"context_lines":[{"line_number":248,"context_line":"A configuration parameter called \u0027retry_lock_timeout_seconds\u0027 would be used to"},{"line_number":249,"context_line":"release \u0027stuck\u0027 locks on the retry tasks table. This parameter would also be"},{"line_number":250,"context_line":"added to the \u0027[scheduler]\u0027 group."},{"line_number":251,"context_line":""},{"line_number":252,"context_line":"A configuration parameter called \u0027delay_before_update_seconds\u0027 would be used to"},{"line_number":253,"context_line":"configure the amount of time the SimpleCertificatePlugin delays from"},{"line_number":254,"context_line":"initiating a demo certificate order to the time the update certificate method"}],"source_content_type":"text/x-rst","patch_set":3,"id":"5a890539_17c1d753","line":251,"in_reply_to":"9aa7fdbe_47bca006","updated":"2014-11-17 11:42:19.000000000","message":"Would be nice to have the workins of this in the blueprint","commit_id":"cee18e672eadb359f5eeb4454a5023d32a48c4bf"},{"author":{"_account_id":9234,"name":"Steve Heyman","email":"steve@heyman.com","username":"hockeynut"},"change_message_id":"1fb55076e254a42f4979b71c7d5a25d41448f030","unresolved":false,"context_lines":[{"line_number":26,"context_line":"successful or unsuccessful termination of the asynchronous task respectively."},{"line_number":27,"context_line":""},{"line_number":28,"context_line":"Barbican worker processes implement these asynchronous tasks, as depicted on"},{"line_number":29,"context_line":"this wiki page: https://github.com/cloudkeep/barbican/wiki/Architecture"},{"line_number":30,"context_line":""},{"line_number":31,"context_line":"As shown in the diagram, a typical deployment can include multiple worker"},{"line_number":32,"context_line":"processes operating in parallel off a tasking queue. The queue invokes task"}],"source_content_type":"text/x-rst","patch_set":4,"id":"3a961159_700188ae","line":29,"updated":"2014-12-10 23:27:38.000000000","message":"This blueprint cries out for an update to this picture to show the DB, queue and process implications.","commit_id":"eb432093e73f48bd4bbea78577e7367018f2e0af"},{"author":{"_account_id":7789,"name":"John Wood","email":"john.wood@rackspace.com","username":"woodster"},"change_message_id":"9db6ce723ec769c65fecc3140b8ec54fca36632e","unresolved":false,"context_lines":[{"line_number":26,"context_line":"successful or unsuccessful termination of the asynchronous task respectively."},{"line_number":27,"context_line":""},{"line_number":28,"context_line":"Barbican worker processes implement these asynchronous tasks, as depicted on"},{"line_number":29,"context_line":"this wiki page: https://github.com/cloudkeep/barbican/wiki/Architecture"},{"line_number":30,"context_line":""},{"line_number":31,"context_line":"As shown in the diagram, a typical deployment can include multiple worker"},{"line_number":32,"context_line":"processes operating in parallel off a tasking queue. The queue invokes task"}],"source_content_type":"text/x-rst","patch_set":4,"id":"3a961159_11b4c6cf","line":29,"in_reply_to":"3a961159_700188ae","updated":"2014-12-11 14:26:33.000000000","message":"This blueprint doesn\u0027t change the architecture other than to allow workers to send messages to the queue.","commit_id":"eb432093e73f48bd4bbea78577e7367018f2e0af"},{"author":{"_account_id":8004,"name":"Paul Kehrer","email":"paul.l.kehrer@gmail.com","username":"reaperhulk"},"change_message_id":"ac0753df1631a48a5d4b5caebed6450d5a3175d1","unresolved":false,"context_lines":[{"line_number":47,"context_line":""},{"line_number":48,"context_line":"Note that this blueprint does not handle concurrent updates made to the"},{"line_number":49,"context_line":"same entity, say to perform a periodic status check on an order and also apply"},{"line_number":50,"context_line":"client updates to that same order. This will be addressed in a future"},{"line_number":51,"context_line":"blueprint."},{"line_number":52,"context_line":""},{"line_number":53,"context_line":"Note also that this blueprint does not handle entities that are \u0027stuck\u0027 in the"}],"source_content_type":"text/x-rst","patch_set":4,"id":"3a961159_79d3735f","line":50,"updated":"2014-12-17 12:50:37.000000000","message":"What does client updates mean in this context?","commit_id":"eb432093e73f48bd4bbea78577e7367018f2e0af"},{"author":{"_account_id":7789,"name":"John Wood","email":"john.wood@rackspace.com","username":"woodster"},"change_message_id":"c18d932406f97887d1be84709f7cbdf48346b431","unresolved":false,"context_lines":[{"line_number":47,"context_line":""},{"line_number":48,"context_line":"Note that this blueprint does not handle concurrent updates made to the"},{"line_number":49,"context_line":"same entity, say to perform a periodic status check on an order and also apply"},{"line_number":50,"context_line":"client updates to that same order. This will be addressed in a future"},{"line_number":51,"context_line":"blueprint."},{"line_number":52,"context_line":""},{"line_number":53,"context_line":"Note also that this blueprint does not handle entities that are \u0027stuck\u0027 in the"}],"source_content_type":"text/x-rst","patch_set":4,"id":"3a961159_e50cdcee","line":50,"in_reply_to":"3a961159_6552acdf","updated":"2014-12-18 21:53:52.000000000","message":"....elaborating a bit here, the issue is that clients could PUT to the orders/ resource for a specific order UUID, and care must be taken not to \u0027clobber\u0027 current order metadata with that new data as update processing could be happening concurrently to the PUT.","commit_id":"eb432093e73f48bd4bbea78577e7367018f2e0af"},{"author":{"_account_id":7789,"name":"John Wood","email":"john.wood@rackspace.com","username":"woodster"},"change_message_id":"ea4691b6b75f967b3ffd488a9dec7a2caeb4b411","unresolved":false,"context_lines":[{"line_number":47,"context_line":""},{"line_number":48,"context_line":"Note that this blueprint does not handle concurrent updates made to the"},{"line_number":49,"context_line":"same entity, say to perform a periodic status check on an order and also apply"},{"line_number":50,"context_line":"client updates to that same order. This will be addressed in a future"},{"line_number":51,"context_line":"blueprint."},{"line_number":52,"context_line":""},{"line_number":53,"context_line":"Note also that this blueprint does not handle entities that are \u0027stuck\u0027 in the"}],"source_content_type":"text/x-rst","patch_set":4,"id":"3a961159_6552acdf","line":50,"in_reply_to":"3a961159_79d3735f","updated":"2014-12-18 21:52:14.000000000","message":"The specific use case is for updates made to initiated ssl cert orders.","commit_id":"eb432093e73f48bd4bbea78577e7367018f2e0af"},{"author":{"_account_id":9234,"name":"Steve Heyman","email":"steve@heyman.com","username":"hockeynut"},"change_message_id":"1fb55076e254a42f4979b71c7d5a25d41448f030","unresolved":false,"context_lines":[{"line_number":94,"context_line":"           this issue, deferring to deployment monitoring and scaling"},{"line_number":95,"context_line":"           processes."},{"line_number":96,"context_line":""},{"line_number":97,"context_line":"Proposed Change"},{"line_number":98,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":99,"context_line":"This blueprint proposes that for requirements R-1 and R-2, the plugins used by"},{"line_number":100,"context_line":"worker tasks (such as the certificate plugin) determine if tasks should be"}],"source_content_type":"text/x-rst","patch_set":4,"id":"3a961159_25cc246a","line":97,"updated":"2014-12-10 23:27:38.000000000","message":"how do we prevent unlimited retries?  I see the retry_count, do we have a configuration option for max retries?  If so then it makes sense to live with the retry interval mentioned below.","commit_id":"eb432093e73f48bd4bbea78577e7367018f2e0af"},{"author":{"_account_id":7789,"name":"John Wood","email":"john.wood@rackspace.com","username":"woodster"},"change_message_id":"9db6ce723ec769c65fecc3140b8ec54fca36632e","unresolved":false,"context_lines":[{"line_number":94,"context_line":"           this issue, deferring to deployment monitoring and scaling"},{"line_number":95,"context_line":"           processes."},{"line_number":96,"context_line":""},{"line_number":97,"context_line":"Proposed Change"},{"line_number":98,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":99,"context_line":"This blueprint proposes that for requirements R-1 and R-2, the plugins used by"},{"line_number":100,"context_line":"worker tasks (such as the certificate plugin) determine if tasks should be"}],"source_content_type":"text/x-rst","patch_set":4,"id":"3a961159_2b9dc00b","line":97,"in_reply_to":"3a961159_25cc246a","updated":"2014-12-11 14:26:33.000000000","message":"I think this will have to be up to the plugins...if could take days or weeks to get some certificates approved so Barbican really couldn\u0027t enforce a time limit on order requests. That said, we should be monitoring for orders that are months old, but I\u0027m thinking that is separate from this CR?","commit_id":"eb432093e73f48bd4bbea78577e7367018f2e0af"},{"author":{"_account_id":11970,"name":"Chelsea Winfree","email":"chelsea.winfree@gmail.com","username":"chellygel"},"change_message_id":"a2f30cdaf299f5605a9715412514129f74d08d19","unresolved":false,"context_lines":[{"line_number":94,"context_line":"           this issue, deferring to deployment monitoring and scaling"},{"line_number":95,"context_line":"           processes."},{"line_number":96,"context_line":""},{"line_number":97,"context_line":"Proposed Change"},{"line_number":98,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":99,"context_line":"This blueprint proposes that for requirements R-1 and R-2, the plugins used by"},{"line_number":100,"context_line":"worker tasks (such as the certificate plugin) determine if tasks should be"}],"source_content_type":"text/x-rst","patch_set":4,"id":"3a961159_99ee83b3","line":97,"in_reply_to":"3a961159_2b9dc00b","updated":"2014-12-16 15:43:30.000000000","message":"in some cases for certificates, it can take a month or more. I completely agree though, I think that can be addressed elsewhere. Having the plugin handle how long is too long may make sense.\n\nAlthough, it might make sense to allow something to be configurable to the owner of the barbican. I\u0027m not sure.","commit_id":"eb432093e73f48bd4bbea78577e7367018f2e0af"},{"author":{"_account_id":8004,"name":"Paul Kehrer","email":"paul.l.kehrer@gmail.com","username":"reaperhulk"},"change_message_id":"ac0753df1631a48a5d4b5caebed6450d5a3175d1","unresolved":false,"context_lines":[{"line_number":94,"context_line":"           this issue, deferring to deployment monitoring and scaling"},{"line_number":95,"context_line":"           processes."},{"line_number":96,"context_line":""},{"line_number":97,"context_line":"Proposed Change"},{"line_number":98,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":99,"context_line":"This blueprint proposes that for requirements R-1 and R-2, the plugins used by"},{"line_number":100,"context_line":"worker tasks (such as the certificate plugin) determine if tasks should be"}],"source_content_type":"text/x-rst","patch_set":4,"id":"3a961159_995adf81","line":97,"in_reply_to":"3a961159_99ee83b3","updated":"2014-12-17 12:50:37.000000000","message":"I agree with John, plugins are going to need to be responsible for their own retry limits.","commit_id":"eb432093e73f48bd4bbea78577e7367018f2e0af"},{"author":{"_account_id":9234,"name":"Steve Heyman","email":"steve@heyman.com","username":"hockeynut"},"change_message_id":"1fb55076e254a42f4979b71c7d5a25d41448f030","unresolved":false,"context_lines":[{"line_number":109,"context_line":""},{"line_number":110,"context_line":"A working example implementation with an older code base is shown here:"},{"line_number":111,"context_line":""},{"line_number":112,"context_line":"https://github.com/cloudkeep/barbican/blob/verify-resource/barbican/queue/server.py#L174"},{"line_number":113,"context_line":""},{"line_number":114,"context_line":"Each worker node could then execute a periodic task service, that invokes a"},{"line_number":115,"context_line":"method on a scheduled basis (configurable, say every 15 seconds). This method"}],"source_content_type":"text/x-rst","patch_set":4,"id":"3a961159_25226452","line":112,"updated":"2014-12-10 23:27:38.000000000","message":"might be good to copy/paste that function here, just in case that verify-resource branch gets zapped some day?","commit_id":"eb432093e73f48bd4bbea78577e7367018f2e0af"},{"author":{"_account_id":7789,"name":"John Wood","email":"john.wood@rackspace.com","username":"woodster"},"change_message_id":"9db6ce723ec769c65fecc3140b8ec54fca36632e","unresolved":false,"context_lines":[{"line_number":109,"context_line":""},{"line_number":110,"context_line":"A working example implementation with an older code base is shown here:"},{"line_number":111,"context_line":""},{"line_number":112,"context_line":"https://github.com/cloudkeep/barbican/blob/verify-resource/barbican/queue/server.py#L174"},{"line_number":113,"context_line":""},{"line_number":114,"context_line":"Each worker node could then execute a periodic task service, that invokes a"},{"line_number":115,"context_line":"method on a scheduled basis (configurable, say every 15 seconds). This method"}],"source_content_type":"text/x-rst","patch_set":4,"id":"3a961159_2e078e1a","line":112,"in_reply_to":"3a961159_25226452","updated":"2014-12-11 14:26:33.000000000","message":"If I can keep it to a few lines I\u0027ll do that, but a number of changes were made to get that feature working.","commit_id":"eb432093e73f48bd4bbea78577e7367018f2e0af"},{"author":{"_account_id":9234,"name":"Steve Heyman","email":"steve@heyman.com","username":"hockeynut"},"change_message_id":"1fb55076e254a42f4979b71c7d5a25d41448f030","unresolved":false,"context_lines":[{"line_number":115,"context_line":"method on a scheduled basis (configurable, say every 15 seconds). This method"},{"line_number":116,"context_line":"would then query which tasks need to be retried (say if current time \u003e\u003d"},{"line_number":117,"context_line":"retry time), and for each one issue a retry task message to the queue. Once"},{"line_number":118,"context_line":"tasks are enqueued, this method would remove the retry records from the retry"},{"line_number":119,"context_line":"list. Eventually the queue would invoke workers to implement these retry tasks."},{"line_number":120,"context_line":""},{"line_number":121,"context_line":"To provide a means to evaluate the retry feature in standalone Barbican per"}],"source_content_type":"text/x-rst","patch_set":4,"id":"3a961159_70c6e869","line":118,"updated":"2014-12-10 23:27:38.000000000","message":"so the periodic task will query the database and possibly slap a new record into the queue.  It will not do any actual retry at this point...just schedule a retry.","commit_id":"eb432093e73f48bd4bbea78577e7367018f2e0af"},{"author":{"_account_id":7789,"name":"John Wood","email":"john.wood@rackspace.com","username":"woodster"},"change_message_id":"9db6ce723ec769c65fecc3140b8ec54fca36632e","unresolved":false,"context_lines":[{"line_number":115,"context_line":"method on a scheduled basis (configurable, say every 15 seconds). This method"},{"line_number":116,"context_line":"would then query which tasks need to be retried (say if current time \u003e\u003d"},{"line_number":117,"context_line":"retry time), and for each one issue a retry task message to the queue. Once"},{"line_number":118,"context_line":"tasks are enqueued, this method would remove the retry records from the retry"},{"line_number":119,"context_line":"list. Eventually the queue would invoke workers to implement these retry tasks."},{"line_number":120,"context_line":""},{"line_number":121,"context_line":"To provide a means to evaluate the retry feature in standalone Barbican per"}],"source_content_type":"text/x-rst","patch_set":4,"id":"3a961159_6e5b562e","line":118,"in_reply_to":"3a961159_70c6e869","updated":"2014-12-11 14:26:33.000000000","message":"That\u0027s correct...so this uses the queue manager to load balance tasks to the workers.  So if a periodic retry process on one worker finds that 10 tasks need to be retried, 10 tasks will be sent to the queue for it to then distribute back out to workers.","commit_id":"eb432093e73f48bd4bbea78577e7367018f2e0af"},{"author":{"_account_id":9234,"name":"Steve Heyman","email":"steve@heyman.com","username":"hockeynut"},"change_message_id":"1fb55076e254a42f4979b71c7d5a25d41448f030","unresolved":false,"context_lines":[{"line_number":133,"context_line":""},{"line_number":134,"context_line":"    1) id: Primary key for this record"},{"line_number":135,"context_line":""},{"line_number":136,"context_line":"    2) order_id: FK to the order record the retry task is intended for"},{"line_number":137,"context_line":""},{"line_number":138,"context_line":"    3) retry_task: The RPC method to invoke for the retry. This method could be"},{"line_number":139,"context_line":"                   a different method than the current one, such as to support"}],"source_content_type":"text/x-rst","patch_set":4,"id":"3a961159_85a5d848","line":136,"updated":"2014-12-10 23:27:38.000000000","message":"In the best case, the first retry will be successful.  In that case we\u0027ll go back and change the original order from PENDING to reflect that success, correct?","commit_id":"eb432093e73f48bd4bbea78577e7367018f2e0af"},{"author":{"_account_id":7789,"name":"John Wood","email":"john.wood@rackspace.com","username":"woodster"},"change_message_id":"9db6ce723ec769c65fecc3140b8ec54fca36632e","unresolved":false,"context_lines":[{"line_number":133,"context_line":""},{"line_number":134,"context_line":"    1) id: Primary key for this record"},{"line_number":135,"context_line":""},{"line_number":136,"context_line":"    2) order_id: FK to the order record the retry task is intended for"},{"line_number":137,"context_line":""},{"line_number":138,"context_line":"    3) retry_task: The RPC method to invoke for the retry. This method could be"},{"line_number":139,"context_line":"                   a different method than the current one, such as to support"}],"source_content_type":"text/x-rst","patch_set":4,"id":"3a961159_8eb16249","line":136,"in_reply_to":"3a961159_85a5d848","updated":"2014-12-11 14:26:33.000000000","message":"The overall order status is determined by the plugins...so if the initiate cert or update_cert_status flows indicate the order is done, then the order will be marked as done. That\u0027s independent of the actual retry/schedule part of this CR though.","commit_id":"eb432093e73f48bd4bbea78577e7367018f2e0af"},{"author":{"_account_id":9234,"name":"Steve Heyman","email":"steve@heyman.com","username":"hockeynut"},"change_message_id":"1fb55076e254a42f4979b71c7d5a25d41448f030","unresolved":false,"context_lines":[{"line_number":136,"context_line":"    2) order_id: FK to the order record the retry task is intended for"},{"line_number":137,"context_line":""},{"line_number":138,"context_line":"    3) retry_task: The RPC method to invoke for the retry. This method could be"},{"line_number":139,"context_line":"                   a different method than the current one, such as to support"},{"line_number":140,"context_line":"                   a SSL certificate plugin checking for certificate updates"},{"line_number":141,"context_line":"                   after initiating the certificate process"},{"line_number":142,"context_line":""}],"source_content_type":"text/x-rst","patch_set":4,"id":"3a961159_10804445","line":139,"updated":"2014-12-10 23:27:38.000000000","message":"This is an interesting one.  The case of SSL certificate here seems more of a progress check than a retry.  To me a retry implies that something completed but didn\u0027t complete successfully.\n\nIs this task is meant to actually perform the retry?  Or just check on its progress?","commit_id":"eb432093e73f48bd4bbea78577e7367018f2e0af"},{"author":{"_account_id":7789,"name":"John Wood","email":"john.wood@rackspace.com","username":"woodster"},"change_message_id":"9db6ce723ec769c65fecc3140b8ec54fca36632e","unresolved":false,"context_lines":[{"line_number":136,"context_line":"    2) order_id: FK to the order record the retry task is intended for"},{"line_number":137,"context_line":""},{"line_number":138,"context_line":"    3) retry_task: The RPC method to invoke for the retry. This method could be"},{"line_number":139,"context_line":"                   a different method than the current one, such as to support"},{"line_number":140,"context_line":"                   a SSL certificate plugin checking for certificate updates"},{"line_number":141,"context_line":"                   after initiating the certificate process"},{"line_number":142,"context_line":""}],"source_content_type":"text/x-rst","patch_set":4,"id":"3a961159_2ed86e7a","line":139,"in_reply_to":"3a961159_10804445","updated":"2014-12-11 14:26:33.000000000","message":"They are similar processes...if the CA wasn\u0027t available, I want to retry that feature again later, and if I want to check for status updates, I\u0027ll retry that feature again later. So \u0027retry\u0027 and \u0027schedule\u0027 are similar concepts here. Does that make sense though?","commit_id":"eb432093e73f48bd4bbea78577e7367018f2e0af"},{"author":{"_account_id":9234,"name":"Steve Heyman","email":"steve@heyman.com","username":"hockeynut"},"change_message_id":"1fb55076e254a42f4979b71c7d5a25d41448f030","unresolved":false,"context_lines":[{"line_number":147,"context_line":""},{"line_number":148,"context_line":"    6) retry_kwargs: A JSON-ified dict of the kwargs to send to retry_task"},{"line_number":149,"context_line":""},{"line_number":150,"context_line":"    7) retry_count: A count of how many times this task has been retried"},{"line_number":151,"context_line":""},{"line_number":152,"context_line":"New retry records would be added for tasks that need to be retried in the"},{"line_number":153,"context_line":"future, as determined by the plugin as part of workflow processing. The next"}],"source_content_type":"text/x-rst","patch_set":4,"id":"3a961159_107204ad","line":150,"updated":"2014-12-10 23:27:38.000000000","message":"if some kind of external lock is needed, should it be part of this entity?","commit_id":"eb432093e73f48bd4bbea78577e7367018f2e0af"},{"author":{"_account_id":7789,"name":"John Wood","email":"john.wood@rackspace.com","username":"woodster"},"change_message_id":"9db6ce723ec769c65fecc3140b8ec54fca36632e","unresolved":false,"context_lines":[{"line_number":147,"context_line":""},{"line_number":148,"context_line":"    6) retry_kwargs: A JSON-ified dict of the kwargs to send to retry_task"},{"line_number":149,"context_line":""},{"line_number":150,"context_line":"    7) retry_count: A count of how many times this task has been retried"},{"line_number":151,"context_line":""},{"line_number":152,"context_line":"New retry records would be added for tasks that need to be retried in the"},{"line_number":153,"context_line":"future, as determined by the plugin as part of workflow processing. The next"}],"source_content_type":"text/x-rst","patch_set":4,"id":"3a961159_ce536ae0","line":150,"in_reply_to":"3a961159_107204ad","updated":"2014-12-11 14:26:33.000000000","message":"The next entity (EntityLock) deals with that issue I think...","commit_id":"eb432093e73f48bd4bbea78577e7367018f2e0af"},{"author":{"_account_id":9234,"name":"Steve Heyman","email":"steve@heyman.com","username":"hockeynut"},"change_message_id":"1fb55076e254a42f4979b71c7d5a25d41448f030","unresolved":false,"context_lines":[{"line_number":154,"context_line":"periodic task method invocation would then send this task to the queue for"},{"line_number":155,"context_line":"another worker to implement later."},{"line_number":156,"context_line":""},{"line_number":157,"context_line":"The EntityLock entity would manage which worker is allowed to delete from the"},{"line_number":158,"context_line":"OrderRetryTask table, since per NF-1 above only one worker should be able to"},{"line_number":159,"context_line":"delete from this table. This entity would have the following attributes::"},{"line_number":160,"context_line":""}],"source_content_type":"text/x-rst","patch_set":4,"id":"3a961159_05164855","line":157,"updated":"2014-12-10 23:27:38.000000000","message":"So a worker can claim a retry attempt from the database using this lock entity?  Does sqlalchemy provide a row locking ability that we could leverage rather than create a new entity?","commit_id":"eb432093e73f48bd4bbea78577e7367018f2e0af"},{"author":{"_account_id":7789,"name":"John Wood","email":"john.wood@rackspace.com","username":"woodster"},"change_message_id":"9db6ce723ec769c65fecc3140b8ec54fca36632e","unresolved":false,"context_lines":[{"line_number":154,"context_line":"periodic task method invocation would then send this task to the queue for"},{"line_number":155,"context_line":"another worker to implement later."},{"line_number":156,"context_line":""},{"line_number":157,"context_line":"The EntityLock entity would manage which worker is allowed to delete from the"},{"line_number":158,"context_line":"OrderRetryTask table, since per NF-1 above only one worker should be able to"},{"line_number":159,"context_line":"delete from this table. This entity would have the following attributes::"},{"line_number":160,"context_line":""}],"source_content_type":"text/x-rst","patch_set":4,"id":"3a961159_093acc98","line":157,"in_reply_to":"3a961159_05164855","updated":"2014-12-11 14:26:33.000000000","message":"Well, we\u0027d need to be able to lock all the records that need to be retried, so they can be deleted as they are posted to the queue. A select for update would lock the queries rows (the ones that have to be retried let\u0027s say), but I\u0027m not sure if you can then delete those rows in the same transaction...worth investigating though.","commit_id":"eb432093e73f48bd4bbea78577e7367018f2e0af"},{"author":{"_account_id":9234,"name":"Steve Heyman","email":"steve@heyman.com","username":"hockeynut"},"change_message_id":"1fb55076e254a42f4979b71c7d5a25d41448f030","unresolved":false,"context_lines":[{"line_number":179,"context_line":"        return"},{"line_number":180,"context_line":""},{"line_number":181,"context_line":"    try:"},{"line_number":182,"context_line":"        Query for retry tasks"},{"line_number":183,"context_line":"        Send retry tasks to the queue"},{"line_number":184,"context_line":"        Remove enqueued retry tasks from OrderRetryTask table"},{"line_number":185,"context_line":"        session.commit()"}],"source_content_type":"text/x-rst","patch_set":4,"id":"3a961159_b0d2105a","line":182,"updated":"2014-12-10 23:27:38.000000000","message":"this is a db query, not a queue query, right?","commit_id":"eb432093e73f48bd4bbea78577e7367018f2e0af"},{"author":{"_account_id":7789,"name":"John Wood","email":"john.wood@rackspace.com","username":"woodster"},"change_message_id":"9db6ce723ec769c65fecc3140b8ec54fca36632e","unresolved":false,"context_lines":[{"line_number":179,"context_line":"        return"},{"line_number":180,"context_line":""},{"line_number":181,"context_line":"    try:"},{"line_number":182,"context_line":"        Query for retry tasks"},{"line_number":183,"context_line":"        Send retry tasks to the queue"},{"line_number":184,"context_line":"        Remove enqueued retry tasks from OrderRetryTask table"},{"line_number":185,"context_line":"        session.commit()"}],"source_content_type":"text/x-rst","patch_set":4,"id":"3a961159_c918c420","line":182,"in_reply_to":"3a961159_b0d2105a","updated":"2014-12-11 14:26:33.000000000","message":"Correct.","commit_id":"eb432093e73f48bd4bbea78577e7367018f2e0af"},{"author":{"_account_id":9234,"name":"Steve Heyman","email":"steve@heyman.com","username":"hockeynut"},"change_message_id":"1fb55076e254a42f4979b71c7d5a25d41448f030","unresolved":false,"context_lines":[{"line_number":181,"context_line":"    try:"},{"line_number":182,"context_line":"        Query for retry tasks"},{"line_number":183,"context_line":"        Send retry tasks to the queue"},{"line_number":184,"context_line":"        Remove enqueued retry tasks from OrderRetryTask table"},{"line_number":185,"context_line":"        session.commit()"},{"line_number":186,"context_line":"    except:"},{"line_number":187,"context_line":"        session.rollback()"}],"source_content_type":"text/x-rst","patch_set":4,"id":"3a961159_b06050a6","line":184,"updated":"2014-12-10 23:27:38.000000000","message":"should we leave these around for audit and possibly problem determination?  They would need some kind of flag so that they are ignored when we query for retry work as shown above.","commit_id":"eb432093e73f48bd4bbea78577e7367018f2e0af"},{"author":{"_account_id":7789,"name":"John Wood","email":"john.wood@rackspace.com","username":"woodster"},"change_message_id":"9db6ce723ec769c65fecc3140b8ec54fca36632e","unresolved":false,"context_lines":[{"line_number":181,"context_line":"    try:"},{"line_number":182,"context_line":"        Query for retry tasks"},{"line_number":183,"context_line":"        Send retry tasks to the queue"},{"line_number":184,"context_line":"        Remove enqueued retry tasks from OrderRetryTask table"},{"line_number":185,"context_line":"        session.commit()"},{"line_number":186,"context_line":"    except:"},{"line_number":187,"context_line":"        session.rollback()"}],"source_content_type":"text/x-rst","patch_set":4,"id":"3a961159_e95020d6","line":184,"in_reply_to":"3a961159_b06050a6","updated":"2014-12-11 14:26:33.000000000","message":"They could be \u0027soft\u0027 deleted, but it might be better to issue an event that could be used for auditing later?","commit_id":"eb432093e73f48bd4bbea78577e7367018f2e0af"},{"author":{"_account_id":9234,"name":"Steve Heyman","email":"steve@heyman.com","username":"hockeynut"},"change_message_id":"1fb55076e254a42f4979b71c7d5a25d41448f030","unresolved":false,"context_lines":[{"line_number":190,"context_line":"        Clear SQLAlchemy session/transaction"},{"line_number":191,"context_line":""},{"line_number":192,"context_line":"Lock tables can be problematic if the locking process crashes without removing"},{"line_number":193,"context_line":"the locks. The overall time a worker holds on to a lock should be brief"},{"line_number":194,"context_line":"however, so the lock attempt rollback process above should check for and remove"},{"line_number":195,"context_line":"a stale lock based on the \u0027created_at\u0027 time on the lock."},{"line_number":196,"context_line":""}],"source_content_type":"text/x-rst","patch_set":4,"id":"3a961159_905d1418","line":193,"updated":"2014-12-10 23:27:38.000000000","message":"If we used a row lock then the dbms should handle this case.  I hope :-)","commit_id":"eb432093e73f48bd4bbea78577e7367018f2e0af"},{"author":{"_account_id":7789,"name":"John Wood","email":"john.wood@rackspace.com","username":"woodster"},"change_message_id":"9db6ce723ec769c65fecc3140b8ec54fca36632e","unresolved":false,"context_lines":[{"line_number":190,"context_line":"        Clear SQLAlchemy session/transaction"},{"line_number":191,"context_line":""},{"line_number":192,"context_line":"Lock tables can be problematic if the locking process crashes without removing"},{"line_number":193,"context_line":"the locks. The overall time a worker holds on to a lock should be brief"},{"line_number":194,"context_line":"however, so the lock attempt rollback process above should check for and remove"},{"line_number":195,"context_line":"a stale lock based on the \u0027created_at\u0027 time on the lock."},{"line_number":196,"context_line":""}],"source_content_type":"text/x-rst","patch_set":4,"id":"3a961159_290dc8e9","line":193,"in_reply_to":"3a961159_905d1418","updated":"2014-12-11 14:26:33.000000000","message":"Agreed...see comment above.","commit_id":"eb432093e73f48bd4bbea78577e7367018f2e0af"},{"author":{"_account_id":9234,"name":"Steve Heyman","email":"steve@heyman.com","username":"hockeynut"},"change_message_id":"1fb55076e254a42f4979b71c7d5a25d41448f030","unresolved":false,"context_lines":[{"line_number":196,"context_line":""},{"line_number":197,"context_line":"To separate coding concerns, it makes sense to implement this process in a"},{"line_number":198,"context_line":"separate Oslo \u0027service\u0027 server process, similar to the `Keystone listener"},{"line_number":199,"context_line":"approach \u003chttps://github.com/openstack/barbican/blob/master/barbican/queue/keystone_listener.py#L130\u003e`_"},{"line_number":200,"context_line":"This service would only run the Oslo periodic task method, to perform the retry"},{"line_number":201,"context_line":"updating process. If the method failed to operate, say due to another worker"},{"line_number":202,"context_line":"locking resource, it could just return/exit. The next periodic call would then"}],"source_content_type":"text/x-rst","patch_set":4,"id":"3a961159_05af2895","line":199,"updated":"2014-12-10 23:27:38.000000000","message":"link with line # can get stale quickly - would it be better to copy the example code here?","commit_id":"eb432093e73f48bd4bbea78577e7367018f2e0af"},{"author":{"_account_id":7789,"name":"John Wood","email":"john.wood@rackspace.com","username":"woodster"},"change_message_id":"9db6ce723ec769c65fecc3140b8ec54fca36632e","unresolved":false,"context_lines":[{"line_number":196,"context_line":""},{"line_number":197,"context_line":"To separate coding concerns, it makes sense to implement this process in a"},{"line_number":198,"context_line":"separate Oslo \u0027service\u0027 server process, similar to the `Keystone listener"},{"line_number":199,"context_line":"approach \u003chttps://github.com/openstack/barbican/blob/master/barbican/queue/keystone_listener.py#L130\u003e`_"},{"line_number":200,"context_line":"This service would only run the Oslo periodic task method, to perform the retry"},{"line_number":201,"context_line":"updating process. If the method failed to operate, say due to another worker"},{"line_number":202,"context_line":"locking resource, it could just return/exit. The next periodic call would then"}],"source_content_type":"text/x-rst","patch_set":4,"id":"3a961159_c901a4b0","line":199,"in_reply_to":"3a961159_05af2895","updated":"2014-12-11 14:26:33.000000000","message":"Agreed as long as not too many lines of code, I\u0027ll see....","commit_id":"eb432093e73f48bd4bbea78577e7367018f2e0af"},{"author":{"_account_id":9234,"name":"Steve Heyman","email":"steve@heyman.com","username":"hockeynut"},"change_message_id":"1fb55076e254a42f4979b71c7d5a25d41448f030","unresolved":false,"context_lines":[{"line_number":269,"context_line":"The Barbican configuration file will need a configuration parameter to"},{"line_number":270,"context_line":"periodically run the retry-query process, called \u0027schedule_period_seconds\u0027,"},{"line_number":271,"context_line":"with a default value of 15 seconds. This parameter would be placed in a new"},{"line_number":272,"context_line":"\u0027[scheduler]\u0027 group."},{"line_number":273,"context_line":""},{"line_number":274,"context_line":"A configuration parameter called \u0027retry_lock_timeout_seconds\u0027 would be used to"},{"line_number":275,"context_line":"release \u0027stuck\u0027 locks on the retry tasks table, as described in the \u0027Proposed"}],"source_content_type":"text/x-rst","patch_set":4,"id":"3a961159_90a8b478","line":272,"updated":"2014-12-10 23:27:38.000000000","message":"retry-scheduler would probably be a better name - more specific to its task than the generic \"scheduler\" term","commit_id":"eb432093e73f48bd4bbea78577e7367018f2e0af"},{"author":{"_account_id":7789,"name":"John Wood","email":"john.wood@rackspace.com","username":"woodster"},"change_message_id":"9db6ce723ec769c65fecc3140b8ec54fca36632e","unresolved":false,"context_lines":[{"line_number":269,"context_line":"The Barbican configuration file will need a configuration parameter to"},{"line_number":270,"context_line":"periodically run the retry-query process, called \u0027schedule_period_seconds\u0027,"},{"line_number":271,"context_line":"with a default value of 15 seconds. This parameter would be placed in a new"},{"line_number":272,"context_line":"\u0027[scheduler]\u0027 group."},{"line_number":273,"context_line":""},{"line_number":274,"context_line":"A configuration parameter called \u0027retry_lock_timeout_seconds\u0027 would be used to"},{"line_number":275,"context_line":"release \u0027stuck\u0027 locks on the retry tasks table, as described in the \u0027Proposed"}],"source_content_type":"text/x-rst","patch_set":4,"id":"3a961159_092f2c40","line":272,"in_reply_to":"3a961159_90a8b478","updated":"2014-12-11 14:26:33.000000000","message":"Sounds good.","commit_id":"eb432093e73f48bd4bbea78577e7367018f2e0af"},{"author":{"_account_id":9234,"name":"Steve Heyman","email":"steve@heyman.com","username":"hockeynut"},"change_message_id":"1fb55076e254a42f4979b71c7d5a25d41448f030","unresolved":false,"context_lines":[{"line_number":283,"context_line":"group."},{"line_number":284,"context_line":""},{"line_number":285,"context_line":"These configurations would be applied and utilized once the revised code base"},{"line_number":286,"context_line":"is deployed."},{"line_number":287,"context_line":""},{"line_number":288,"context_line":"Developer impact"},{"line_number":289,"context_line":"----------------"}],"source_content_type":"text/x-rst","patch_set":4,"id":"3a961159_70c26835","line":286,"updated":"2014-12-10 23:27:38.000000000","message":"this would be a great place for a max_retry_count to limit how many times a task can be retried before we throw up our arms and give up.","commit_id":"eb432093e73f48bd4bbea78577e7367018f2e0af"},{"author":{"_account_id":7789,"name":"John Wood","email":"john.wood@rackspace.com","username":"woodster"},"change_message_id":"9db6ce723ec769c65fecc3140b8ec54fca36632e","unresolved":false,"context_lines":[{"line_number":283,"context_line":"group."},{"line_number":284,"context_line":""},{"line_number":285,"context_line":"These configurations would be applied and utilized once the revised code base"},{"line_number":286,"context_line":"is deployed."},{"line_number":287,"context_line":""},{"line_number":288,"context_line":"Developer impact"},{"line_number":289,"context_line":"----------------"}],"source_content_type":"text/x-rst","patch_set":4,"id":"3a961159_89223c50","line":286,"in_reply_to":"3a961159_70c26835","updated":"2014-12-11 14:26:33.000000000","message":"See comments above.","commit_id":"eb432093e73f48bd4bbea78577e7367018f2e0af"}]}