)]}'
{"specs/kilo/smooth-deletes-over-time.rst":[{"author":{"_account_id":11356,"name":"Louis Taylor","email":"louis@kragniz.eu","username":"kragniz"},"change_message_id":"41f868d67bd5e5c9f3e6061dd4dfee1260bad066","unresolved":false,"context_lines":[{"line_number":37,"context_line":"Proposed change"},{"line_number":38,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":39,"context_line":""},{"line_number":40,"context_line":"1. Read moving average data (if none collected yet, use configured initial"},{"line_number":41,"context_line":"   values)"},{"line_number":42,"context_line":"2. Update in-memory cache with images to be deleted from DB"},{"line_number":43,"context_line":"3. Calculate minimum number of images that need to be deleted now such that all"}],"source_content_type":"text/x-rst","patch_set":1,"id":"baa201ad_73af24e4","line":40,"updated":"2014-09-30 19:40:58.000000000","message":"What is this the moving average of? The time to delete a single image?","commit_id":"0d65e3efd6909106d6dd8945e1c842d863e0d804"},{"author":{"_account_id":11642,"name":"Jesse J. Cook","email":"jesse.cook@rackspace.com","username":"crashenx"},"change_message_id":"3d261b507e310f9997e6cd1e9e06feda26c53313","unresolved":false,"context_lines":[{"line_number":37,"context_line":"Proposed change"},{"line_number":38,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":39,"context_line":""},{"line_number":40,"context_line":"1. Read moving average data (if none collected yet, use configured initial"},{"line_number":41,"context_line":"   values)"},{"line_number":42,"context_line":"2. Update in-memory cache with images to be deleted from DB"},{"line_number":43,"context_line":"3. Calculate minimum number of images that need to be deleted now such that all"}],"source_content_type":"text/x-rst","patch_set":1,"id":"baa201ad_3b5e52e7","line":40,"in_reply_to":"baa201ad_73af24e4","updated":"2014-09-30 22:13:54.000000000","message":"I should update this to explain better. The simplest answer is yes. More precisely, I\u0027m thinking it would be how long it would take a worker (which is assigned to delete an image) to complete it\u0027s work.","commit_id":"0d65e3efd6909106d6dd8945e1c842d863e0d804"},{"author":{"_account_id":11356,"name":"Louis Taylor","email":"louis@kragniz.eu","username":"kragniz"},"change_message_id":"41f868d67bd5e5c9f3e6061dd4dfee1260bad066","unresolved":false,"context_lines":[{"line_number":39,"context_line":""},{"line_number":40,"context_line":"1. Read moving average data (if none collected yet, use configured initial"},{"line_number":41,"context_line":"   values)"},{"line_number":42,"context_line":"2. Update in-memory cache with images to be deleted from DB"},{"line_number":43,"context_line":"3. Calculate minimum number of images that need to be deleted now such that all"},{"line_number":44,"context_line":"   images are deleted within configured period, e.g."},{"line_number":45,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"baa201ad_53822050","line":42,"updated":"2014-09-30 19:40:58.000000000","message":"Is this a cache of the image IDs?","commit_id":"0d65e3efd6909106d6dd8945e1c842d863e0d804"},{"author":{"_account_id":11642,"name":"Jesse J. Cook","email":"jesse.cook@rackspace.com","username":"crashenx"},"change_message_id":"3d261b507e310f9997e6cd1e9e06feda26c53313","unresolved":false,"context_lines":[{"line_number":39,"context_line":""},{"line_number":40,"context_line":"1. Read moving average data (if none collected yet, use configured initial"},{"line_number":41,"context_line":"   values)"},{"line_number":42,"context_line":"2. Update in-memory cache with images to be deleted from DB"},{"line_number":43,"context_line":"3. Calculate minimum number of images that need to be deleted now such that all"},{"line_number":44,"context_line":"   images are deleted within configured period, e.g."},{"line_number":45,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"baa201ad_5bf956c7","line":42,"in_reply_to":"baa201ad_53822050","updated":"2014-09-30 22:13:54.000000000","message":"I will update this with some detail. I was thinking a hash with the key being the image_id, the status in [pending_delete, delete_in_progress], the retry count, and the worker id (used to stop a worker that could be frozen).","commit_id":"0d65e3efd6909106d6dd8945e1c842d863e0d804"},{"author":{"_account_id":11356,"name":"Louis Taylor","email":"louis@kragniz.eu","username":"kragniz"},"change_message_id":"41f868d67bd5e5c9f3e6061dd4dfee1260bad066","unresolved":false,"context_lines":[{"line_number":49,"context_line":"   d. Average delete time is calculated at 5 minutes"},{"line_number":50,"context_line":"   e. Need to be deleting 10 images in parallel now"},{"line_number":51,"context_line":"   f. 7 deletes are already in progress"},{"line_number":52,"context_line":"   g. 3 additional deletes need to be started"},{"line_number":53,"context_line":""},{"line_number":54,"context_line":"4. Kick off a worker for each delete that needs to be started and mark each"},{"line_number":55,"context_line":"   pending_delete as in progress within the in-memory cache"}],"source_content_type":"text/x-rst","patch_set":1,"id":"baa201ad_73dd6470","line":52,"updated":"2014-09-30 19:40:58.000000000","message":"What happens in the case where it\u0027s not possible to delete the images in the configured time period?","commit_id":"0d65e3efd6909106d6dd8945e1c842d863e0d804"},{"author":{"_account_id":11642,"name":"Jesse J. Cook","email":"jesse.cook@rackspace.com","username":"crashenx"},"change_message_id":"3d261b507e310f9997e6cd1e9e06feda26c53313","unresolved":false,"context_lines":[{"line_number":49,"context_line":"   d. Average delete time is calculated at 5 minutes"},{"line_number":50,"context_line":"   e. Need to be deleting 10 images in parallel now"},{"line_number":51,"context_line":"   f. 7 deletes are already in progress"},{"line_number":52,"context_line":"   g. 3 additional deletes need to be started"},{"line_number":53,"context_line":""},{"line_number":54,"context_line":"4. Kick off a worker for each delete that needs to be started and mark each"},{"line_number":55,"context_line":"   pending_delete as in progress within the in-memory cache"}],"source_content_type":"text/x-rst","patch_set":1,"id":"baa201ad_3b04927a","line":52,"in_reply_to":"baa201ad_73dd6470","updated":"2014-09-30 22:13:54.000000000","message":"Let\u0027s take an example to see if I understand. Let\u0027s say there is 2 minutes \"remaining\" (N.B., the time remaining is always one period from now) and it takes 5 minutes to delete and there are 100 images to delete. We would kick off all 100 image deletes (one per worker). It\u0027s fine that it doesn\u0027t finish within the 2 minutes, because the purpose is to smooth out deletes not ensure they are deleted at a specific time.\n\nI can only think of one way this would happen. That would be if the period was set to be really short (i.e. say within one standard deviation of the moving average). This is because the goal is to delete all of the images one period from the current time (now). The decision of how many workers to run is based on the average over the last period (from now), the current backlog, and slicing that up over the next period (from now). If the period was set to be really short, then every time a worker finished, a new worker for every image in pending_delete would be started. This is essentially delete as they come in without busy looping.\n\nThis does bring up an important consideration. There might need to be a max worker threshold. If there was a max worker threshold, then you could get into a situation of always running max workers while falling behind. That would mean you\u0027d need to add more scrubbers, and is probably the correct behavior. It\u0027d be better than causing the server to thrash, although I don\u0027t expect that would be likely.","commit_id":"0d65e3efd6909106d6dd8945e1c842d863e0d804"},{"author":{"_account_id":12807,"name":"Steve Lewis (stevelle)","email":"stevelle@gmail.com","username":"stevelle"},"change_message_id":"77fca0b4ddda3da8d2c888aadbd7a2f056705732","unresolved":false,"context_lines":[{"line_number":187,"context_line":""},{"line_number":188,"context_line":"Other reviewer(s):"},{"line_number":189,"context_line":"  hemanth-makkapati"},{"line_number":190,"context_line":""},{"line_number":191,"context_line":"Work Items"},{"line_number":192,"context_line":"----------"},{"line_number":193,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"5a890539_e607e0c7","line":190,"updated":"2014-12-05 23:29:26.000000000","message":"please include me as another reviewer: steve-lewis","commit_id":"9d77e809f5d2133180634933a12c7c3388ac0024"},{"author":{"_account_id":11642,"name":"Jesse J. Cook","email":"jesse.cook@rackspace.com","username":"crashenx"},"change_message_id":"866da1a00c7fc1d96d2a0748c3473c6c5c926b60","unresolved":false,"context_lines":[{"line_number":187,"context_line":""},{"line_number":188,"context_line":"Other reviewer(s):"},{"line_number":189,"context_line":"  hemanth-makkapati"},{"line_number":190,"context_line":""},{"line_number":191,"context_line":"Work Items"},{"line_number":192,"context_line":"----------"},{"line_number":193,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"3a961159_605aac51","line":190,"in_reply_to":"5a890539_e607e0c7","updated":"2014-12-10 15:06:58.000000000","message":"added you","commit_id":"9d77e809f5d2133180634933a12c7c3388ac0024"}],"specs/liberty/smooth-deletes-over-time.rst":[{"author":{"_account_id":6159,"name":"Flavio Percoco Premoli","display_name":"flaper87","email":"flavio.percoco@flyrlabs.com","username":"flaper87"},"change_message_id":"41586c0c4a79ff2ace366663286e78b6d1970c43","unresolved":false,"context_lines":[{"line_number":35,"context_line":"slow response times due to delete request spikes."},{"line_number":36,"context_line":""},{"line_number":37,"context_line":"Proposed change"},{"line_number":38,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":39,"context_line":""},{"line_number":40,"context_line":"Definitions:"},{"line_number":41,"context_line":""}],"source_content_type":"text/x-rst","patch_set":5,"id":"3a50d1a3_2275aaf9","line":38,"updated":"2015-07-29 23:54:20.000000000","message":"Can we use the taskflow engine for this? we could have a periodic task taking care of this and even move the scrubber to just be a task.","commit_id":"1299022b100ea346bdf7f782acceb785238efb13"},{"author":{"_account_id":11642,"name":"Jesse J. Cook","email":"jesse.cook@rackspace.com","username":"crashenx"},"change_message_id":"5db9a12a7a7eb95f4bd4b34e4ef432e54bab7ae1","unresolved":false,"context_lines":[{"line_number":35,"context_line":"slow response times due to delete request spikes."},{"line_number":36,"context_line":""},{"line_number":37,"context_line":"Proposed change"},{"line_number":38,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":39,"context_line":""},{"line_number":40,"context_line":"Definitions:"},{"line_number":41,"context_line":""}],"source_content_type":"text/x-rst","patch_set":5,"id":"3a50d1a3_f9c2e233","line":38,"in_reply_to":"3a50d1a3_2275aaf9","updated":"2015-07-30 23:50:08.000000000","message":"That\u0027s a really good question. I\u0027m curious what Geetika and Hemanth\u0027s thoughts are.","commit_id":"1299022b100ea346bdf7f782acceb785238efb13"}]}
