)]}'
{"specs/mitaka/ha-aa-cleanup.rst":[{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"980b8bf8e4500048a8a2a30d7a2711d661e96ec9","unresolved":false,"context_lines":[{"line_number":77,"context_line":"are in flight, and therefore all resources that require cleanup when a node"},{"line_number":78,"context_line":"crashes."},{"line_number":79,"context_line":""},{"line_number":80,"context_line":"When a job is received by the manager on a c-vol node of the cluster it will"},{"line_number":81,"context_line":"add a new row in the ``workers`` table setting the cluster, host and backend"},{"line_number":82,"context_line":"information as well as the resource type and uuid of the resource.  And once"},{"line_number":83,"context_line":"the operation has been completed this row will be deleted to indicate"}],"source_content_type":"text/x-rst","patch_set":1,"id":"3a29b11f_73cfcc56","line":80,"updated":"2015-10-19 15:39:29.000000000","message":"Is it possible that one resource will have multiple rows in ``workers`` table? Probably with atomic state transitions - no, but that\u0027s something to consider.","commit_id":"315fa15981f037ac31bee3b577c0474e6b05bb9d"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"05cfe2efbbc6dae5093773f058cb445ee98bf73e","unresolved":false,"context_lines":[{"line_number":77,"context_line":"are in flight, and therefore all resources that require cleanup when a node"},{"line_number":78,"context_line":"crashes."},{"line_number":79,"context_line":""},{"line_number":80,"context_line":"When a job is received by the manager on a c-vol node of the cluster it will"},{"line_number":81,"context_line":"add a new row in the ``workers`` table setting the cluster, host and backend"},{"line_number":82,"context_line":"information as well as the resource type and uuid of the resource.  And once"},{"line_number":83,"context_line":"the operation has been completed this row will be deleted to indicate"}],"source_content_type":"text/x-rst","patch_set":1,"id":"3a29b11f_da124809","line":80,"in_reply_to":"3a29b11f_73cfcc56","updated":"2015-10-20 11:57:18.000000000","message":"Yes it is possible and we want it to be that way.  Added explanation of this since I forgot to do it.","commit_id":"315fa15981f037ac31bee3b577c0474e6b05bb9d"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"980b8bf8e4500048a8a2a30d7a2711d661e96ec9","unresolved":false,"context_lines":[{"line_number":78,"context_line":"crashes."},{"line_number":79,"context_line":""},{"line_number":80,"context_line":"When a job is received by the manager on a c-vol node of the cluster it will"},{"line_number":81,"context_line":"add a new row in the ``workers`` table setting the cluster, host and backend"},{"line_number":82,"context_line":"information as well as the resource type and uuid of the resource.  And once"},{"line_number":83,"context_line":"the operation has been completed this row will be deleted to indicate"},{"line_number":84,"context_line":"processing has concluded and a cleanup will no longer be needed if the node"}],"source_content_type":"text/x-rst","patch_set":1,"id":"3a29b11f_d3634019","line":81,"updated":"2015-10-19 15:39:29.000000000","message":"In case of using more than one resource (for example volume cloning) - are we adding there both?","commit_id":"315fa15981f037ac31bee3b577c0474e6b05bb9d"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"05cfe2efbbc6dae5093773f058cb445ee98bf73e","unresolved":false,"context_lines":[{"line_number":78,"context_line":"crashes."},{"line_number":79,"context_line":""},{"line_number":80,"context_line":"When a job is received by the manager on a c-vol node of the cluster it will"},{"line_number":81,"context_line":"add a new row in the ``workers`` table setting the cluster, host and backend"},{"line_number":82,"context_line":"information as well as the resource type and uuid of the resource.  And once"},{"line_number":83,"context_line":"the operation has been completed this row will be deleted to indicate"},{"line_number":84,"context_line":"processing has concluded and a cleanup will no longer be needed if the node"}],"source_content_type":"text/x-rst","patch_set":1,"id":"3a29b11f_fa2ce44b","line":81,"in_reply_to":"3a29b11f_d3634019","updated":"2015-10-20 11:57:18.000000000","message":"Not for the cleanup, so in this spec I\u0027ll explicitly say no.  But another spec will need it, so it will be added by that implementation.","commit_id":"315fa15981f037ac31bee3b577c0474e6b05bb9d"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"980b8bf8e4500048a8a2a30d7a2711d661e96ec9","unresolved":false,"context_lines":[{"line_number":102,"context_line":"cleanup simultaneously, code needs to be able handle this concurrent cleanup."},{"line_number":103,"context_line":""},{"line_number":104,"context_line":"The way we will allow concurrent cleanups to be performed from the schedulers"},{"line_number":105,"context_line":"and the c-vol nodes is by creating a ``clean_resource`` asynchronous RPC method"},{"line_number":106,"context_line":"that will take care of the cleanup of a resource and will be called both from"},{"line_number":107,"context_line":"the c-vol nodes as well as from the scheduler."},{"line_number":108,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"3a29b11f_b381e44c","line":105,"updated":"2015-10-19 15:39:29.000000000","message":"Where should the method be implemented? Probably c-vol is a better place, but I wasn\u0027t able to find information on that in the spec. What are the method arguments?","commit_id":"315fa15981f037ac31bee3b577c0474e6b05bb9d"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"05cfe2efbbc6dae5093773f058cb445ee98bf73e","unresolved":false,"context_lines":[{"line_number":102,"context_line":"cleanup simultaneously, code needs to be able handle this concurrent cleanup."},{"line_number":103,"context_line":""},{"line_number":104,"context_line":"The way we will allow concurrent cleanups to be performed from the schedulers"},{"line_number":105,"context_line":"and the c-vol nodes is by creating a ``clean_resource`` asynchronous RPC method"},{"line_number":106,"context_line":"that will take care of the cleanup of a resource and will be called both from"},{"line_number":107,"context_line":"the c-vol nodes as well as from the scheduler."},{"line_number":108,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"3a29b11f_9a913055","line":105,"in_reply_to":"3a29b11f_b381e44c","updated":"2015-10-20 11:57:18.000000000","message":"I wasn\u0027t planning on getting down to that level of detail, describing in which files the code goes, what methods are going to be implemented and all the arguments they will need.  (But yes, since it\u0027s an RPC call from Scheduler to c-vol nodes it will go in cinder/volume/manager.py)","commit_id":"315fa15981f037ac31bee3b577c0474e6b05bb9d"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"980b8bf8e4500048a8a2a30d7a2711d661e96ec9","unresolved":false,"context_lines":[{"line_number":106,"context_line":"that will take care of the cleanup of a resource and will be called both from"},{"line_number":107,"context_line":"the c-vol nodes as well as from the scheduler."},{"line_number":108,"context_line":""},{"line_number":109,"context_line":"To prevent multiple calls from different nodes (schedulers and c-vol) to clean"},{"line_number":110,"context_line":"up the same resource we will delete rows that we are going to clean up.  If we"},{"line_number":111,"context_line":"cannot delete the row because it no longer exists then we will not proceed with"},{"line_number":112,"context_line":"the cleanup RPC call, as that means cleanup is already being requested by some"}],"source_content_type":"text/x-rst","patch_set":1,"id":"3a29b11f_b3e24402","line":109,"updated":"2015-10-19 15:39:29.000000000","message":"Yay, idempotent call! ;)","commit_id":"315fa15981f037ac31bee3b577c0474e6b05bb9d"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"980b8bf8e4500048a8a2a30d7a2711d661e96ec9","unresolved":false,"context_lines":[{"line_number":120,"context_line":"configurations with more than one active node means that the cleanup is"},{"line_number":121,"context_line":"distributed among all active nodes."},{"line_number":122,"context_line":""},{"line_number":123,"context_line":""},{"line_number":124,"context_line":"Alternatives"},{"line_number":125,"context_line":"------------"},{"line_number":126,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"3a29b11f_be820b51","line":123,"updated":"2015-10-19 15:39:29.000000000","message":"A thing to consider would be a complicated network partition scenario. For example - if a node loses the connection to the DB it is unable to update a heartbeat, but can be still working on a resource (execution of long dd for example). Then it reconnects to the DB to find the volume it was working on in error state…\n\nOkay, this seems like a good approach. ;) Only we should make sure that such node will stop working and don\u0027t update DB state of a volume from \"error\".","commit_id":"315fa15981f037ac31bee3b577c0474e6b05bb9d"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"05cfe2efbbc6dae5093773f058cb445ee98bf73e","unresolved":false,"context_lines":[{"line_number":120,"context_line":"configurations with more than one active node means that the cleanup is"},{"line_number":121,"context_line":"distributed among all active nodes."},{"line_number":122,"context_line":""},{"line_number":123,"context_line":""},{"line_number":124,"context_line":"Alternatives"},{"line_number":125,"context_line":"------------"},{"line_number":126,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"3a29b11f_1ae1a0d0","line":123,"in_reply_to":"3a29b11f_be820b51","updated":"2015-10-20 11:57:18.000000000","message":"That is part of the specs to prevent data corruption: https://review.openstack.org/#/c/237076/","commit_id":"315fa15981f037ac31bee3b577c0474e6b05bb9d"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"980b8bf8e4500048a8a2a30d7a2711d661e96ec9","unresolved":false,"context_lines":[{"line_number":205,"context_line":"  Gorka Eguileor (geguileo)"},{"line_number":206,"context_line":""},{"line_number":207,"context_line":"Other contributors:"},{"line_number":208,"context_line":"  Anyone is welcome to help"},{"line_number":209,"context_line":""},{"line_number":210,"context_line":"Work Items"},{"line_number":211,"context_line":"----------"}],"source_content_type":"text/x-rst","patch_set":1,"id":"3a29b11f_5303b042","line":208,"updated":"2015-10-19 15:39:29.000000000","message":"You can add me here again. ;)","commit_id":"315fa15981f037ac31bee3b577c0474e6b05bb9d"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"05cfe2efbbc6dae5093773f058cb445ee98bf73e","unresolved":false,"context_lines":[{"line_number":205,"context_line":"  Gorka Eguileor (geguileo)"},{"line_number":206,"context_line":""},{"line_number":207,"context_line":"Other contributors:"},{"line_number":208,"context_line":"  Anyone is welcome to help"},{"line_number":209,"context_line":""},{"line_number":210,"context_line":"Work Items"},{"line_number":211,"context_line":"----------"}],"source_content_type":"text/x-rst","patch_set":1,"id":"3a29b11f_5ad718ed","line":208,"in_reply_to":"3a29b11f_5303b042","updated":"2015-10-20 11:57:18.000000000","message":"Done","commit_id":"315fa15981f037ac31bee3b577c0474e6b05bb9d"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"980b8bf8e4500048a8a2a30d7a2711d661e96ec9","unresolved":false,"context_lines":[{"line_number":212,"context_line":""},{"line_number":213,"context_line":"- Make DB changes to add the new ``workers`` table."},{"line_number":214,"context_line":""},{"line_number":215,"context_line":"- Add rows to ``workers`` table."},{"line_number":216,"context_line":""},{"line_number":217,"context_line":"- Change ``host_init`` to use an RPC call for the cleanup."},{"line_number":218,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"3a29b11f_5355f04d","line":215,"updated":"2015-10-19 15:39:29.000000000","message":"\"Implement adding rows to ``workers`` table.\" would be probably better.","commit_id":"315fa15981f037ac31bee3b577c0474e6b05bb9d"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"05cfe2efbbc6dae5093773f058cb445ee98bf73e","unresolved":false,"context_lines":[{"line_number":212,"context_line":""},{"line_number":213,"context_line":"- Make DB changes to add the new ``workers`` table."},{"line_number":214,"context_line":""},{"line_number":215,"context_line":"- Add rows to ``workers`` table."},{"line_number":216,"context_line":""},{"line_number":217,"context_line":"- Change ``host_init`` to use an RPC call for the cleanup."},{"line_number":218,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"3a29b11f_3adc9c11","line":215,"in_reply_to":"3a29b11f_5355f04d","updated":"2015-10-20 11:57:18.000000000","message":"Done","commit_id":"315fa15981f037ac31bee3b577c0474e6b05bb9d"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"980b8bf8e4500048a8a2a30d7a2711d661e96ec9","unresolved":false,"context_lines":[{"line_number":227,"context_line":"   node has gone down so the scheduler can do the cleanup and we also require"},{"line_number":228,"context_line":"   the new ``cluster`` configuration option.."},{"line_number":229,"context_line":""},{"line_number":230,"context_line":""},{"line_number":231,"context_line":"Testing"},{"line_number":232,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":233,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"3a29b11f_93e8687e","line":230,"updated":"2015-10-19 15:39:29.000000000","message":"I believe that atomic state transitions are also needed to prevent possibility of two nodes working on the same resource.","commit_id":"315fa15981f037ac31bee3b577c0474e6b05bb9d"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"05cfe2efbbc6dae5093773f058cb445ee98bf73e","unresolved":false,"context_lines":[{"line_number":227,"context_line":"   node has gone down so the scheduler can do the cleanup and we also require"},{"line_number":228,"context_line":"   the new ``cluster`` configuration option.."},{"line_number":229,"context_line":""},{"line_number":230,"context_line":""},{"line_number":231,"context_line":"Testing"},{"line_number":232,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":233,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"3a29b11f_5a45d8b0","line":230,"in_reply_to":"3a29b11f_93e8687e","updated":"2015-10-20 11:57:18.000000000","message":"Yep, just added it and added explanation in Proposed Change section.","commit_id":"315fa15981f037ac31bee3b577c0474e6b05bb9d"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"02a6a55da10b7d2377a3dda2835504fb14848b32","unresolved":false,"context_lines":[{"line_number":138,"context_line":"Cleanup method is just like any other c-vol method, so it will also add a row"},{"line_number":139,"context_line":"to the ``workers`` table to indicate that it is working on the resource."},{"line_number":140,"context_line":""},{"line_number":141,"context_line":"Although unlikely, we may still get multiple calls to the cleanup RCP method"},{"line_number":142,"context_line":"when we have multiple entries for a resource and multiple nodes do the delete"},{"line_number":143,"context_line":"of the entries in the DB -each one may delete a couple of entries- so we have"},{"line_number":144,"context_line":"to make sure that the operation is atomic.  The easiest way will be changing"}],"source_content_type":"text/x-rst","patch_set":2,"id":"3a29b11f_e773d613","line":141,"updated":"2015-10-20 18:15:46.000000000","message":"How about doing a DB transaction with required status changes for cleanup, and then delete? That would be sufficient if in the cleanup phase we\u0027re only doing DB operations - I\u0027m not sure if export operation you\u0027ve mentioned earlier is actually idempotent.","commit_id":"7d20074c0e726b24e0d745ebc9b8e9f79b7d7bc5"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"d00ce5a2d2a4d01c824c4d8bb000a269d4611123","unresolved":false,"context_lines":[{"line_number":138,"context_line":"Cleanup method is just like any other c-vol method, so it will also add a row"},{"line_number":139,"context_line":"to the ``workers`` table to indicate that it is working on the resource."},{"line_number":140,"context_line":""},{"line_number":141,"context_line":"Although unlikely, we may still get multiple calls to the cleanup RCP method"},{"line_number":142,"context_line":"when we have multiple entries for a resource and multiple nodes do the delete"},{"line_number":143,"context_line":"of the entries in the DB -each one may delete a couple of entries- so we have"},{"line_number":144,"context_line":"to make sure that the operation is atomic.  The easiest way will be changing"}],"source_content_type":"text/x-rst","patch_set":2,"id":"3a29b11f_4b8077f7","line":141,"in_reply_to":"3a29b11f_3095663e","updated":"2015-10-21 10:45:41.000000000","message":"I\u0027m not sure if a better explanation is needed, It seems pretty easy to understand that cleaning-status-thing is idempotent. :)","commit_id":"7d20074c0e726b24e0d745ebc9b8e9f79b7d7bc5"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"d169cd2d9feb3d599f0238b01e21c7b790bc8090","unresolved":false,"context_lines":[{"line_number":138,"context_line":"Cleanup method is just like any other c-vol method, so it will also add a row"},{"line_number":139,"context_line":"to the ``workers`` table to indicate that it is working on the resource."},{"line_number":140,"context_line":""},{"line_number":141,"context_line":"Although unlikely, we may still get multiple calls to the cleanup RCP method"},{"line_number":142,"context_line":"when we have multiple entries for a resource and multiple nodes do the delete"},{"line_number":143,"context_line":"of the entries in the DB -each one may delete a couple of entries- so we have"},{"line_number":144,"context_line":"to make sure that the operation is atomic.  The easiest way will be changing"}],"source_content_type":"text/x-rst","patch_set":2,"id":"3a29b11f_3095663e","line":141,"in_reply_to":"3a29b11f_e773d613","updated":"2015-10-21 10:28:05.000000000","message":"I actually thought about it, but discarded it because it splits the cleaning process and that would make it harder to follow/understand.\n\nAs for the idempotency of the cleanup method, it will be idempotent since it changes the status of the resource to ``cleaning`` and once process is completed resource will be left in a status that doesn\u0027t do cleanup.\n\nI\u0027ll add a better explanation for it.","commit_id":"7d20074c0e726b24e0d745ebc9b8e9f79b7d7bc5"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"02a6a55da10b7d2377a3dda2835504fb14848b32","unresolved":false,"context_lines":[{"line_number":139,"context_line":"to the ``workers`` table to indicate that it is working on the resource."},{"line_number":140,"context_line":""},{"line_number":141,"context_line":"Although unlikely, we may still get multiple calls to the cleanup RCP method"},{"line_number":142,"context_line":"when we have multiple entries for a resource and multiple nodes do the delete"},{"line_number":143,"context_line":"of the entries in the DB -each one may delete a couple of entries- so we have"},{"line_number":144,"context_line":"to make sure that the operation is atomic.  The easiest way will be changing"},{"line_number":145,"context_line":"the status of the resource to ``cleaning`` or something similar while storing"}],"source_content_type":"text/x-rst","patch_set":2,"id":"3a29b11f_8796a2ee","line":142,"updated":"2015-10-20 18:15:46.000000000","message":"Can you describe an example when we may end up with multiple entries to ``workers`` table for a single resource?","commit_id":"7d20074c0e726b24e0d745ebc9b8e9f79b7d7bc5"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"d169cd2d9feb3d599f0238b01e21c7b790bc8090","unresolved":false,"context_lines":[{"line_number":139,"context_line":"to the ``workers`` table to indicate that it is working on the resource."},{"line_number":140,"context_line":""},{"line_number":141,"context_line":"Although unlikely, we may still get multiple calls to the cleanup RCP method"},{"line_number":142,"context_line":"when we have multiple entries for a resource and multiple nodes do the delete"},{"line_number":143,"context_line":"of the entries in the DB -each one may delete a couple of entries- so we have"},{"line_number":144,"context_line":"to make sure that the operation is atomic.  The easiest way will be changing"},{"line_number":145,"context_line":"the status of the resource to ``cleaning`` or something similar while storing"}],"source_content_type":"text/x-rst","patch_set":2,"id":"3a29b11f_908a3a1a","line":142,"in_reply_to":"3a29b11f_8796a2ee","updated":"2015-10-21 10:28:05.000000000","message":"I\u0027ll add it to the specs.","commit_id":"7d20074c0e726b24e0d745ebc9b8e9f79b7d7bc5"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"6ff6706d078ca2235d189a5e0ea192a71a9c8f22","unresolved":false,"context_lines":[{"line_number":271,"context_line":" - We need compare-and-swap mechanism to be in place for the cleanup RPC"},{"line_number":272,"context_line":"   method."},{"line_number":273,"context_line":""},{"line_number":274,"context_line":""},{"line_number":275,"context_line":"Testing"},{"line_number":276,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":277,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"3a29b11f_9ced9427","line":274,"updated":"2015-10-20 18:43:49.000000000","message":"Ah, don\u0027t we have a dependency on auto-fencing spec here also?\n\nI might got this wrong, getting all the specs around your head is painful. ;)","commit_id":"7d20074c0e726b24e0d745ebc9b8e9f79b7d7bc5"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"d169cd2d9feb3d599f0238b01e21c7b790bc8090","unresolved":false,"context_lines":[{"line_number":271,"context_line":" - We need compare-and-swap mechanism to be in place for the cleanup RPC"},{"line_number":272,"context_line":"   method."},{"line_number":273,"context_line":""},{"line_number":274,"context_line":""},{"line_number":275,"context_line":"Testing"},{"line_number":276,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":277,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"3a29b11f_f0d60ef0","line":274,"in_reply_to":"3a29b11f_9ced9427","updated":"2015-10-21 10:28:05.000000000","message":"I didn\u0027t consider it as a dependency since this can be worked on without that spec being implemented.  And I agree, it\u0027s painful to have all the specs in the head.  :-)","commit_id":"7d20074c0e726b24e0d745ebc9b8e9f79b7d7bc5"},{"author":{"_account_id":15961,"name":"lisali","email":"xiaoyan.li@intel.com","username":"lisali"},"change_message_id":"eff9eaeda5dcede5d15f5f44143e7c1315267dd2","unresolved":false,"context_lines":[{"line_number":68,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":69,"context_line":""},{"line_number":70,"context_line":"Since checking for the status and the host field of the resource is no longer"},{"line_number":71,"context_line":"enough to know if it needs cleanup -because host field will be referring to the"},{"line_number":72,"context_line":"cluster and not an individual host as explained in the `Job Distribution`_"},{"line_number":73,"context_line":"specs- we will create a new table to track which node from the cluster is"},{"line_number":74,"context_line":"working on each resource."}],"source_content_type":"text/x-rst","patch_set":4,"id":"9a8ffd7b_5bbabd6f","line":71,"updated":"2015-11-26 05:52:49.000000000","message":"I am concerned why not use one of following ways:\n\nAdd another field in volume table to specify the host, which is similar to the first alternative.\n\nWhen doing cleanup, it checks the ongoing volumes and finds out the host which is down, calls cleanup() in a volume node to do cleanup. \n\nIn the call, the original host is one of the parameters. So when dealing with cleanup, it at first check whether host are not changed, if not, update the host to the current host, and start to clean. (this is automatic.)\n\nThis solves the situations you mention that multiple schedulers or volume nodes clean up one resource concurrently.\n\nThe benefit for the way:\n\n1. No need the following new table.\n\n2. For schedule. When operating a same volume, it can prefer to  schedule to the same host. It is probably good for DLM as there may be lock cached and still available.","commit_id":"1899ed895f7e746cf4280ad1f64bdb76daa9c074"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"998062d0004e32d3946185bd71c7cbeb4e5adf57","unresolved":false,"context_lines":[{"line_number":68,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":69,"context_line":""},{"line_number":70,"context_line":"Since checking for the status and the host field of the resource is no longer"},{"line_number":71,"context_line":"enough to know if it needs cleanup -because host field will be referring to the"},{"line_number":72,"context_line":"cluster and not an individual host as explained in the `Job Distribution`_"},{"line_number":73,"context_line":"specs- we will create a new table to track which node from the cluster is"},{"line_number":74,"context_line":"working on each resource."}],"source_content_type":"text/x-rst","patch_set":4,"id":"9a8ffd7b_fa1b4b71","line":71,"in_reply_to":"9a8ffd7b_5bbabd6f","updated":"2015-11-26 14:37:33.000000000","message":"Yes, that was the first proposed alternative, but I chose this one because I saw multiple benefits to using the `workers` table over adding a field:\n\n- Adding a field would need to be done to any resource that is susceptible of doing cleanups, for example Snapshots, so it\u0027s not only Volumes table.\n\n- We would need to do a multi-table query or do queries to multiple tables to get all the resources that need cleaning up.\n\n- With the `workers` method we can potentially know at which stage the operation failed in actions that require multiple manager calls (like retype).\n\n- At any given time you can actually query the DB and easily tell what is \"on the fly\" in your Cinder deployment.\n\n- The new `workers` table can be used as a DLM mechanism for deployments that use drivers that don\u0027t require distributed locking, as explained in the spec for removing locks from the manager.\n\nWhen you say \"this is automatic\" I understand you mean \"this is atomic\", the check and change of current host.\n\nAs for the benefits, I agree on the first one (it is mentioned in the spec), as for the second:\n\n- You don\u0027t schedule cleanups, you put them in the queue and c-vol nodes pick the job.\n\n- You certainly don\u0027t want to send the job to the same host (if it\u0027s a passive node with the same host configuration that has become active it would be fine), because we are doing the cleanup because that host is \"down\", which means it doesn\u0027t have writting access to the DB, so it won\u0027t be able to change the status of the resource on the cleanup.  And I don\u0027t understand what you mean with the lock being cached.\n\nHaving said that, I am not totally against this alternative, I just think proposed change leaves us in a better place for other changes/features.","commit_id":"1899ed895f7e746cf4280ad1f64bdb76daa9c074"},{"author":{"_account_id":15961,"name":"lisali","email":"xiaoyan.li@intel.com","username":"lisali"},"change_message_id":"e98ab1bac29c039081f0a7317fc9bd8749bb4671","unresolved":false,"context_lines":[{"line_number":68,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":69,"context_line":""},{"line_number":70,"context_line":"Since checking for the status and the host field of the resource is no longer"},{"line_number":71,"context_line":"enough to know if it needs cleanup -because host field will be referring to the"},{"line_number":72,"context_line":"cluster and not an individual host as explained in the `Job Distribution`_"},{"line_number":73,"context_line":"specs- we will create a new table to track which node from the cluster is"},{"line_number":74,"context_line":"working on each resource."}],"source_content_type":"text/x-rst","patch_set":4,"id":"9a8ffd7b_7d889ec5","line":71,"in_reply_to":"9a8ffd7b_fa1b4b71","updated":"2015-11-27 06:21:32.000000000","message":"I like this point: \nWith the `workers` method we can potentially know at which stage the operation failed in actions that require multiple manager calls (like retype).","commit_id":"1899ed895f7e746cf4280ad1f64bdb76daa9c074"},{"author":{"_account_id":15961,"name":"lisali","email":"xiaoyan.li@intel.com","username":"lisali"},"change_message_id":"e98ab1bac29c039081f0a7317fc9bd8749bb4671","unresolved":false,"context_lines":[{"line_number":98,"context_line":"-they will be removed as they are being completed- for example in the case of a"},{"line_number":99,"context_line":"retype that needs migration.  In that case at some point we will have an entry"},{"line_number":100,"context_line":"for the retype call, another for the migration call and another for an attach,"},{"line_number":101,"context_line":"all on the same resource."},{"line_number":102,"context_line":""},{"line_number":103,"context_line":"Deletion operation of the row will be a real deletion in the DB, not a soft"},{"line_number":104,"context_line":"delete like we do for other tables, because the number of operations, and"}],"source_content_type":"text/x-rst","patch_set":4,"id":"9a8ffd7b_98a23098","line":101,"updated":"2015-11-27 06:21:32.000000000","message":"From above words, it is volume nodes\u0027 responsible to add/remove the job entry in worker table.\n\nTo add a row is not difficult, and I am wonder the codes to remove the entry from worker. Seems that there are a lot of exceptions which will leads the job terminate accidentally. \n\nI am worrying may some job entries leave in worker tables, and lead later problem.\n\nFor example, when a volume is attaching to a VM in volume host A, error happens and the entry is not deleted from the worker table.\nLater user resets its status to available.\nhost A down.\nAt the same time, volume is attached to vm in another host B. But there is cleanup in the left job entry in host A. \n\nAny method to make sure the entry is deleted when a job is completed?","commit_id":"1899ed895f7e746cf4280ad1f64bdb76daa9c074"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"610883d5988b80fc63e6e97905598031fa153a59","unresolved":false,"context_lines":[{"line_number":98,"context_line":"-they will be removed as they are being completed- for example in the case of a"},{"line_number":99,"context_line":"retype that needs migration.  In that case at some point we will have an entry"},{"line_number":100,"context_line":"for the retype call, another for the migration call and another for an attach,"},{"line_number":101,"context_line":"all on the same resource."},{"line_number":102,"context_line":""},{"line_number":103,"context_line":"Deletion operation of the row will be a real deletion in the DB, not a soft"},{"line_number":104,"context_line":"delete like we do for other tables, because the number of operations, and"}],"source_content_type":"text/x-rst","patch_set":4,"id":"9a8ffd7b_cfebf011","line":101,"in_reply_to":"9a8ffd7b_98a23098","updated":"2015-12-03 12:50:00.000000000","message":"In my opinion reset status is a functionality that should be used sparsely and with great care.  In any case, results will depend on the status that we change the resource to, if we change it to any status that doesn\u0027t do cleanup (like available) then the entry in the `workers` table will be cleaned by the scheduler or the node when it comes back up (as normal) and no cleanup will be done. If we set the status to one that requires cleanup (usually ing states) then cleanup will be performed for that status.","commit_id":"1899ed895f7e746cf4280ad1f64bdb76daa9c074"},{"author":{"_account_id":15961,"name":"lisali","email":"xiaoyan.li@intel.com","username":"lisali"},"change_message_id":"eff9eaeda5dcede5d15f5f44143e7c1315267dd2","unresolved":false,"context_lines":[{"line_number":128,"context_line":"care of ensuring that they are exported when dealing with a multi-attach"},{"line_number":129,"context_line":"volume."},{"line_number":130,"context_line":""},{"line_number":131,"context_line":"Since we can have multiple scheduler nodes and the c-vol node trying to do the"},{"line_number":132,"context_line":"cleanup simultaneously, code needs to be able handle this concurrent cleanup."},{"line_number":133,"context_line":""},{"line_number":134,"context_line":"The way we will allow concurrent cleanups to be performed from the schedulers"}],"source_content_type":"text/x-rst","patch_set":4,"id":"9a8ffd7b_c4f80e4c","line":131,"updated":"2015-11-26 05:52:49.000000000","message":"Are we going to support multiple A/A scheduler nodes?","commit_id":"1899ed895f7e746cf4280ad1f64bdb76daa9c074"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"998062d0004e32d3946185bd71c7cbeb4e5adf57","unresolved":false,"context_lines":[{"line_number":128,"context_line":"care of ensuring that they are exported when dealing with a multi-attach"},{"line_number":129,"context_line":"volume."},{"line_number":130,"context_line":""},{"line_number":131,"context_line":"Since we can have multiple scheduler nodes and the c-vol node trying to do the"},{"line_number":132,"context_line":"cleanup simultaneously, code needs to be able handle this concurrent cleanup."},{"line_number":133,"context_line":""},{"line_number":134,"context_line":"The way we will allow concurrent cleanups to be performed from the schedulers"}],"source_content_type":"text/x-rst","patch_set":4,"id":"9a8ffd7b_f10136b0","line":131,"in_reply_to":"9a8ffd7b_c4f80e4c","updated":"2015-11-26 14:37:33.000000000","message":"I don\u0027t understand, API and Scheduler nodes can already work in A/A fashion.","commit_id":"1899ed895f7e746cf4280ad1f64bdb76daa9c074"},{"author":{"_account_id":15961,"name":"lisali","email":"xiaoyan.li@intel.com","username":"lisali"},"change_message_id":"e98ab1bac29c039081f0a7317fc9bd8749bb4671","unresolved":false,"context_lines":[{"line_number":128,"context_line":"care of ensuring that they are exported when dealing with a multi-attach"},{"line_number":129,"context_line":"volume."},{"line_number":130,"context_line":""},{"line_number":131,"context_line":"Since we can have multiple scheduler nodes and the c-vol node trying to do the"},{"line_number":132,"context_line":"cleanup simultaneously, code needs to be able handle this concurrent cleanup."},{"line_number":133,"context_line":""},{"line_number":134,"context_line":"The way we will allow concurrent cleanups to be performed from the schedulers"}],"source_content_type":"text/x-rst","patch_set":4,"id":"9a8ffd7b_dd32ea0f","line":131,"in_reply_to":"9a8ffd7b_f10136b0","updated":"2015-11-27 06:21:32.000000000","message":"My apology.","commit_id":"1899ed895f7e746cf4280ad1f64bdb76daa9c074"},{"author":{"_account_id":15961,"name":"lisali","email":"xiaoyan.li@intel.com","username":"lisali"},"change_message_id":"eff9eaeda5dcede5d15f5f44143e7c1315267dd2","unresolved":false,"context_lines":[{"line_number":173,"context_line":"such as the one explained in `Removing Drivers Locks`_.  We would just use a"},{"line_number":174,"context_line":"lock on the cleanup method -locking on the resource id- that will ensure mutual"},{"line_number":175,"context_line":"exclusion and will remove the need to make the status change to the resource"},{"line_number":176,"context_line":"that was explained in previous paragraph."},{"line_number":177,"context_line":""},{"line_number":178,"context_line":"One thing worth noting is the case of a node cleaning up after itself.  It will"},{"line_number":179,"context_line":"also be calling the RPC asynchronous cleanup method instead of handling the"}],"source_content_type":"text/x-rst","patch_set":4,"id":"9a8ffd7b_8f621fac","line":176,"updated":"2015-11-26 05:52:49.000000000","message":"It needs to use Tzoo lock.\n It is probably users send reset-state status concurrently.","commit_id":"1899ed895f7e746cf4280ad1f64bdb76daa9c074"},{"author":{"_account_id":15961,"name":"lisali","email":"xiaoyan.li@intel.com","username":"lisali"},"change_message_id":"e98ab1bac29c039081f0a7317fc9bd8749bb4671","unresolved":false,"context_lines":[{"line_number":173,"context_line":"such as the one explained in `Removing Drivers Locks`_.  We would just use a"},{"line_number":174,"context_line":"lock on the cleanup method -locking on the resource id- that will ensure mutual"},{"line_number":175,"context_line":"exclusion and will remove the need to make the status change to the resource"},{"line_number":176,"context_line":"that was explained in previous paragraph."},{"line_number":177,"context_line":""},{"line_number":178,"context_line":"One thing worth noting is the case of a node cleaning up after itself.  It will"},{"line_number":179,"context_line":"also be calling the RPC asynchronous cleanup method instead of handling the"}],"source_content_type":"text/x-rst","patch_set":4,"id":"9a8ffd7b_9d41c265","line":176,"in_reply_to":"9a8ffd7b_344ecc3b","updated":"2015-11-27 06:21:32.000000000","message":"I am thinking the case: When volume node is doing cleanup to a volume, user sends a reset-state request to the same volume.\n\nIn this case, are you going to change the state directly? \n\nWhen user sends reset-state to change volume status, if the former job is still ongoing and in worker table, do we allow the state change? I know the problem already exists, just wondering can we do things better once we have the new table workers.","commit_id":"1899ed895f7e746cf4280ad1f64bdb76daa9c074"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"998062d0004e32d3946185bd71c7cbeb4e5adf57","unresolved":false,"context_lines":[{"line_number":173,"context_line":"such as the one explained in `Removing Drivers Locks`_.  We would just use a"},{"line_number":174,"context_line":"lock on the cleanup method -locking on the resource id- that will ensure mutual"},{"line_number":175,"context_line":"exclusion and will remove the need to make the status change to the resource"},{"line_number":176,"context_line":"that was explained in previous paragraph."},{"line_number":177,"context_line":""},{"line_number":178,"context_line":"One thing worth noting is the case of a node cleaning up after itself.  It will"},{"line_number":179,"context_line":"also be calling the RPC asynchronous cleanup method instead of handling the"}],"source_content_type":"text/x-rst","patch_set":4,"id":"9a8ffd7b_344ecc3b","line":176,"in_reply_to":"9a8ffd7b_8f621fac","updated":"2015-11-26 14:37:33.000000000","message":"I don\u0027t understand, why are you referring to reset-state?  We are talking about cleanup, which is not only to change the state.\n\nIt doesn\u0027t *need* to use locks, it just simplifies our lifes if we do.","commit_id":"1899ed895f7e746cf4280ad1f64bdb76daa9c074"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"610883d5988b80fc63e6e97905598031fa153a59","unresolved":false,"context_lines":[{"line_number":173,"context_line":"such as the one explained in `Removing Drivers Locks`_.  We would just use a"},{"line_number":174,"context_line":"lock on the cleanup method -locking on the resource id- that will ensure mutual"},{"line_number":175,"context_line":"exclusion and will remove the need to make the status change to the resource"},{"line_number":176,"context_line":"that was explained in previous paragraph."},{"line_number":177,"context_line":""},{"line_number":178,"context_line":"One thing worth noting is the case of a node cleaning up after itself.  It will"},{"line_number":179,"context_line":"also be calling the RPC asynchronous cleanup method instead of handling the"}],"source_content_type":"text/x-rst","patch_set":4,"id":"9a8ffd7b_af422c0d","line":176,"in_reply_to":"9a8ffd7b_9d41c265","updated":"2015-12-03 12:50:00.000000000","message":"Reset-state will behave as it does now.  Now if a node goes down and you change the status to available and the node comes back up it will not do cleanup for that resource.  And if you change the status to one of the *ing status it will do the cleanup.","commit_id":"1899ed895f7e746cf4280ad1f64bdb76daa9c074"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"a4579d57ed11bd85f58bd1196b20ef940b7c4cc8","unresolved":false,"context_lines":[{"line_number":186,"context_line":"Considering the complexity of the full solution -simultaneous automatic cleanup"},{"line_number":187,"context_line":"from the scheduler nodes and cleanup on startup on c-vol nodes- we will disable"},{"line_number":188,"context_line":"automatic cleanup by default and add a configuration option, for example"},{"line_number":189,"context_line":"``auto_cleanup_enabled``, that will allow us to enable it.  So the default"},{"line_number":190,"context_line":"behavior will be basically the same as it is now, cleanup will only be"},{"line_number":191,"context_line":"performed on node startup.  Manual triggering of the scheduler cleanup job will"},{"line_number":192,"context_line":"also be possible with a new API entry point.  This way we\u0027ll reduce the risks"}],"source_content_type":"text/x-rst","patch_set":4,"id":"ba8a016a_989cb95e","line":189,"updated":"2015-11-23 17:38:23.000000000","message":"As automatic cleanup seems to be a complicated thing I would still be okay if it happened that we wouldn\u0027t be able to finish it in Mitaka. Whole solution would still be useful. That\u0027s probably matter of time and resources.","commit_id":"1899ed895f7e746cf4280ad1f64bdb76daa9c074"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"998062d0004e32d3946185bd71c7cbeb4e5adf57","unresolved":false,"context_lines":[{"line_number":186,"context_line":"Considering the complexity of the full solution -simultaneous automatic cleanup"},{"line_number":187,"context_line":"from the scheduler nodes and cleanup on startup on c-vol nodes- we will disable"},{"line_number":188,"context_line":"automatic cleanup by default and add a configuration option, for example"},{"line_number":189,"context_line":"``auto_cleanup_enabled``, that will allow us to enable it.  So the default"},{"line_number":190,"context_line":"behavior will be basically the same as it is now, cleanup will only be"},{"line_number":191,"context_line":"performed on node startup.  Manual triggering of the scheduler cleanup job will"},{"line_number":192,"context_line":"also be possible with a new API entry point.  This way we\u0027ll reduce the risks"}],"source_content_type":"text/x-rst","patch_set":4,"id":"9a8ffd7b_91b30ae9","line":189,"in_reply_to":"ba8a016a_989cb95e","updated":"2015-11-26 14:37:33.000000000","message":"The automatic part is not really that difficult, it is mostly \"scary\"  ;-)\n\nBut I agree, if manual cleanup is ready, automatic cleanup would not be a priority, although I think it would be really useful to have this feature available as experimental so it can be thoroughly tested and refined.","commit_id":"1899ed895f7e746cf4280ad1f64bdb76daa9c074"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"a4579d57ed11bd85f58bd1196b20ef940b7c4cc8","unresolved":false,"context_lines":[{"line_number":249,"context_line":"REST API impact"},{"line_number":250,"context_line":"---------------"},{"line_number":251,"context_line":""},{"line_number":252,"context_line":"Add a new admin REST API to trigger scheduler\u0027s cleanup job.  It will accept"},{"line_number":253,"context_line":"two optional arguments ``backend`` and ``host``."},{"line_number":254,"context_line":""},{"line_number":255,"context_line":"By default if no optional argument is set cleanup will do the same process as"}],"source_content_type":"text/x-rst","patch_set":4,"id":"ba8a016a_d82fa13c","line":252,"updated":"2015-11-23 17:38:23.000000000","message":"This seems useful if you have external cluster manager like Pacemaker. This call could be final part of c-vol fence job.","commit_id":"1899ed895f7e746cf4280ad1f64bdb76daa9c074"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"998062d0004e32d3946185bd71c7cbeb4e5adf57","unresolved":false,"context_lines":[{"line_number":249,"context_line":"REST API impact"},{"line_number":250,"context_line":"---------------"},{"line_number":251,"context_line":""},{"line_number":252,"context_line":"Add a new admin REST API to trigger scheduler\u0027s cleanup job.  It will accept"},{"line_number":253,"context_line":"two optional arguments ``backend`` and ``host``."},{"line_number":254,"context_line":""},{"line_number":255,"context_line":"By default if no optional argument is set cleanup will do the same process as"}],"source_content_type":"text/x-rst","patch_set":4,"id":"9a8ffd7b_d11992d8","line":252,"in_reply_to":"ba8a016a_d82fa13c","updated":"2015-11-26 14:37:33.000000000","message":"That is an interesting idea, but it would only be useful if failing node is able to communicate with schedulers, and wouldn\u0027t add much once auto-cleanup is stable.\n\nThe idea of this API call was to be able to do trigger the manual cleanup.","commit_id":"1899ed895f7e746cf4280ad1f64bdb76daa9c074"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"a4579d57ed11bd85f58bd1196b20ef940b7c4cc8","unresolved":false,"context_lines":[{"line_number":256,"context_line":"the auto cleanup would do, will look which hosts are down and perform the"},{"line_number":257,"context_line":"cleanup job for operations they left ongoing when they crashed."},{"line_number":258,"context_line":""},{"line_number":259,"context_line":"If ``backend`` and/or ``host`` are set, it wil filter resources that auto"},{"line_number":260,"context_line":"cleanup process would have cleaned up using those arguments and will therefore"},{"line_number":261,"context_line":"limit the cleanup to those resources that match the request."},{"line_number":262,"context_line":""}],"source_content_type":"text/x-rst","patch_set":4,"id":"ba8a016a_18e3e934","line":259,"updated":"2015-11-23 17:38:23.000000000","message":"s/wil/will","commit_id":"1899ed895f7e746cf4280ad1f64bdb76daa9c074"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"998062d0004e32d3946185bd71c7cbeb4e5adf57","unresolved":false,"context_lines":[{"line_number":256,"context_line":"the auto cleanup would do, will look which hosts are down and perform the"},{"line_number":257,"context_line":"cleanup job for operations they left ongoing when they crashed."},{"line_number":258,"context_line":""},{"line_number":259,"context_line":"If ``backend`` and/or ``host`` are set, it wil filter resources that auto"},{"line_number":260,"context_line":"cleanup process would have cleaned up using those arguments and will therefore"},{"line_number":261,"context_line":"limit the cleanup to those resources that match the request."},{"line_number":262,"context_line":""}],"source_content_type":"text/x-rst","patch_set":4,"id":"9a8ffd7b_d433c0a9","line":259,"in_reply_to":"ba8a016a_18e3e934","updated":"2015-11-26 14:37:33.000000000","message":"Done","commit_id":"1899ed895f7e746cf4280ad1f64bdb76daa9c074"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"998062d0004e32d3946185bd71c7cbeb4e5adf57","unresolved":false,"context_lines":[{"line_number":346,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":347,"context_line":""},{"line_number":348,"context_line":"Document new configuration option ``auto_cleanup_enabled`` as well as the"},{"line_number":349,"context_line":"menchanism it enables."},{"line_number":350,"context_line":""},{"line_number":351,"context_line":""},{"line_number":352,"context_line":"References"}],"source_content_type":"text/x-rst","patch_set":4,"id":"9a8ffd7b_f4492461","line":349,"updated":"2015-11-26 14:37:33.000000000","message":"nit: s/menchanism/mechanism","commit_id":"1899ed895f7e746cf4280ad1f64bdb76daa9c074"}],"specs/newton/ha-aa-cleanup.rst":[{"author":{"_account_id":6491,"name":"xing-yang","email":"xingyang105@gmail.com","username":"xing-yang"},"change_message_id":"b1c6e0eacc41921fa342902bea4f65664aad60c8","unresolved":false,"context_lines":[{"line_number":91,"context_line":"that resource in case of a crash and see if it needs cleanup.  The reason why"},{"line_number":92,"context_line":"we add a row on each operation is to make sure that when we have multiple"},{"line_number":93,"context_line":"simultaneous operations on the same resource we don\u0027t remove the row that other"},{"line_number":94,"context_line":"operation has added."},{"line_number":95,"context_line":""},{"line_number":96,"context_line":"Since we are adding a row for each operation performed by the manager, some"},{"line_number":97,"context_line":"operations will trigger the creation of multiple DB rows for the same resource"}],"source_content_type":"text/x-rst","patch_set":7,"id":"5a18252c_b9971747","line":94,"updated":"2016-04-14 04:30:11.000000000","message":"How are you going to track temporary resources, i.e., the temporary volume/snapshot created during backup?  Whose uuid will be in the table?  If it is the backup\u0027s uuid, how do you track the temp resource\u0027s id?","commit_id":"82b9a39db29abb745d0fa3083471b004ecbbd32a"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"b40e372caef6417ae43616af403ee0485eaab448","unresolved":false,"context_lines":[{"line_number":91,"context_line":"that resource in case of a crash and see if it needs cleanup.  The reason why"},{"line_number":92,"context_line":"we add a row on each operation is to make sure that when we have multiple"},{"line_number":93,"context_line":"simultaneous operations on the same resource we don\u0027t remove the row that other"},{"line_number":94,"context_line":"operation has added."},{"line_number":95,"context_line":""},{"line_number":96,"context_line":"Since we are adding a row for each operation performed by the manager, some"},{"line_number":97,"context_line":"operations will trigger the creation of multiple DB rows for the same resource"}],"source_content_type":"text/x-rst","patch_set":7,"id":"1a122d0e_dceaefab","line":94,"in_reply_to":"5a18252c_b9971747","updated":"2016-04-18 15:33:15.000000000","message":"Those resource will not be tracked, what will be tracked is the backup that is being created, and the cleanup will be done as we are doing now.\n\nThere\u0027s now a patch up for review where you can see what I mean: https://review.openstack.org/#/c/303023/","commit_id":"82b9a39db29abb745d0fa3083471b004ecbbd32a"},{"author":{"_account_id":6491,"name":"xing-yang","email":"xingyang105@gmail.com","username":"xing-yang"},"change_message_id":"b1c6e0eacc41921fa342902bea4f65664aad60c8","unresolved":false,"context_lines":[{"line_number":368,"context_line":""},{"line_number":369,"context_line":".. _Job Distribution: https://review.openstack.org/#/c/232595/"},{"line_number":370,"context_line":".. _Removing API Races: https://review.openstack.org/207101/"},{"line_number":371,"context_line":".. _Removing Drivers Locks: https://review.openstack.org/237604"}],"source_content_type":"text/x-rst","patch_set":7,"id":"5a18252c_9925fb0a","line":371,"updated":"2016-04-14 04:30:11.000000000","message":"The above spec is abandoned.  Are you trying to reference the Tooz spec: https://review.openstack.org/202615/ ?","commit_id":"82b9a39db29abb745d0fa3083471b004ecbbd32a"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"b40e372caef6417ae43616af403ee0485eaab448","unresolved":false,"context_lines":[{"line_number":368,"context_line":""},{"line_number":369,"context_line":".. _Job Distribution: https://review.openstack.org/#/c/232595/"},{"line_number":370,"context_line":".. _Removing API Races: https://review.openstack.org/207101/"},{"line_number":371,"context_line":".. _Removing Drivers Locks: https://review.openstack.org/237604"}],"source_content_type":"text/x-rst","patch_set":7,"id":"1a122d0e_fc0c0b96","line":371,"in_reply_to":"5a18252c_9925fb0a","updated":"2016-04-18 15:33:15.000000000","message":"I\u0027ll update the references, thanks for pointing that out.  :-)","commit_id":"82b9a39db29abb745d0fa3083471b004ecbbd32a"},{"author":{"_account_id":5997,"name":"Walt","display_name":"Hemna","email":"waboring@hemna.com","username":"walter-boring","status":"SAP"},"change_message_id":"c06210fdb752a1596b4aafd73fee05dd1fdd3f54","unresolved":false,"context_lines":[{"line_number":71,"context_line":"enough to know if it needs cleanup -because host field will be referring to the"},{"line_number":72,"context_line":"cluster and not an individual host as explained in the `Job Distribution`_"},{"line_number":73,"context_line":"specs- we will create a new table to track which node from the cluster is"},{"line_number":74,"context_line":"working on each resource."},{"line_number":75,"context_line":""},{"line_number":76,"context_line":"We\u0027ll call this new table ``workers`` and it will include all resources that"},{"line_number":77,"context_line":"are being processed with cleanable operations, and therefore would require"}],"source_content_type":"text/x-rst","patch_set":8,"id":"1a122d0e_c90ff786","line":74,"updated":"2016-05-03 04:32:48.000000000","message":"why can\u0027t we keep a host field and it has the actual host that picked up the job?  then cleanup checking would be easy.","commit_id":"9f6e0081662d522a2d7fb2bfd894fca6aa964ad7"},{"author":{"_account_id":5997,"name":"Walt","display_name":"Hemna","email":"waboring@hemna.com","username":"walter-boring","status":"SAP"},"change_message_id":"68568d5f0a291500809907025ff7d6a5fd30fc18","unresolved":false,"context_lines":[{"line_number":71,"context_line":"enough to know if it needs cleanup -because host field will be referring to the"},{"line_number":72,"context_line":"cluster and not an individual host as explained in the `Job Distribution`_"},{"line_number":73,"context_line":"specs- we will create a new table to track which node from the cluster is"},{"line_number":74,"context_line":"working on each resource."},{"line_number":75,"context_line":""},{"line_number":76,"context_line":"We\u0027ll call this new table ``workers`` and it will include all resources that"},{"line_number":77,"context_line":"are being processed with cleanable operations, and therefore would require"}],"source_content_type":"text/x-rst","patch_set":8,"id":"3aaa91ec_a6add0ea","line":74,"in_reply_to":"1a122d0e_05a87076","updated":"2016-06-24 21:04:46.000000000","message":"I see what you are saying now.  I suppose a new table is easier to search and find entries than N tables via a complex join.\n\nThe host term is so overused now in Cinder, it\u0027s nearly impossible to decipher the meaning of it anymore.","commit_id":"9f6e0081662d522a2d7fb2bfd894fca6aa964ad7"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"1db44b79941b072d82fe383b9c09c52a9ae8ef2f","unresolved":false,"context_lines":[{"line_number":71,"context_line":"enough to know if it needs cleanup -because host field will be referring to the"},{"line_number":72,"context_line":"cluster and not an individual host as explained in the `Job Distribution`_"},{"line_number":73,"context_line":"specs- we will create a new table to track which node from the cluster is"},{"line_number":74,"context_line":"working on each resource."},{"line_number":75,"context_line":""},{"line_number":76,"context_line":"We\u0027ll call this new table ``workers`` and it will include all resources that"},{"line_number":77,"context_line":"are being processed with cleanable operations, and therefore would require"}],"source_content_type":"text/x-rst","patch_set":8,"id":"1a122d0e_05a87076","line":74,"in_reply_to":"1a122d0e_699223f8","updated":"2016-05-04 15:45:06.000000000","message":"I\u0027m going to assume that when you say host you are using the nomenclature we agreed last week in Austin (cluster - host) instead of the one current spec uses (host - node).\n\nJust storing the working host in a field on the resource has a couple of disadvantages:\n\n- Queries to multiple tables (Volume, Snapshot, Backup) are required to know which resources need to be cleaned when a host goes down.\n\n- Unlike with proposed mechanism, we cannot use it instead of a DLM for the locks in the manager in deployments using drivers that don\u0027t require distributed locks.\n\n- Unless we can come up with a great name for the DB field, it would lead to misunderstandings, as this field would only be filled when the resource is set to a cleanable state and not always.\n\n\nIf you really think there are benefits to just adding a new field to the resource table we can discuss it on a meeting.","commit_id":"9f6e0081662d522a2d7fb2bfd894fca6aa964ad7"},{"author":{"_account_id":5997,"name":"Walt","display_name":"Hemna","email":"waboring@hemna.com","username":"walter-boring","status":"SAP"},"change_message_id":"66607b40f5ba396468c318efeeb311e05512b95a","unresolved":false,"context_lines":[{"line_number":71,"context_line":"enough to know if it needs cleanup -because host field will be referring to the"},{"line_number":72,"context_line":"cluster and not an individual host as explained in the `Job Distribution`_"},{"line_number":73,"context_line":"specs- we will create a new table to track which node from the cluster is"},{"line_number":74,"context_line":"working on each resource."},{"line_number":75,"context_line":""},{"line_number":76,"context_line":"We\u0027ll call this new table ``workers`` and it will include all resources that"},{"line_number":77,"context_line":"are being processed with cleanable operations, and therefore would require"}],"source_content_type":"text/x-rst","patch_set":8,"id":"1a122d0e_699223f8","line":74,"in_reply_to":"1a122d0e_c90ff786","updated":"2016-05-03 04:49:47.000000000","message":"regardless of the job distribution mechanism we are putting in the other spec.  I still think we should have a unique host field that always has the actual host where the job is running.  This will facilitate easy cleanup.  hosts that don\u0027t have the job, don\u0027t need to cleanup.","commit_id":"9f6e0081662d522a2d7fb2bfd894fca6aa964ad7"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"04f7ae35d10f665a5e24f3dbfb62f09f197e05cf","unresolved":false,"context_lines":[{"line_number":71,"context_line":"enough to know if it needs cleanup -because host field will be referring to the"},{"line_number":72,"context_line":"cluster and not an individual host as explained in the `Job Distribution`_"},{"line_number":73,"context_line":"specs- we will create a new table to track which node from the cluster is"},{"line_number":74,"context_line":"working on each resource."},{"line_number":75,"context_line":""},{"line_number":76,"context_line":"We\u0027ll call this new table ``workers`` and it will include all resources that"},{"line_number":77,"context_line":"are being processed with cleanable operations, and therefore would require"}],"source_content_type":"text/x-rst","patch_set":8,"id":"3aaa91ec_49129bb5","line":74,"in_reply_to":"3aaa91ec_a6add0ea","updated":"2016-06-29 11:23:13.000000000","message":"I agree, we should probably write a devref to write our nomenclature.  :-(","commit_id":"9f6e0081662d522a2d7fb2bfd894fca6aa964ad7"},{"author":{"_account_id":6491,"name":"xing-yang","email":"xingyang105@gmail.com","username":"xing-yang"},"change_message_id":"f4e03da710d06645b11a360eb671213bc304ccae","unresolved":false,"context_lines":[{"line_number":80,"context_line":"When a cleanable job is requested by the API or any of the services -for"},{"line_number":81,"context_line":"example a volume deletion can be requested by the API or by the c-vol service"},{"line_number":82,"context_line":"during a migration- we will create a new row in the ``workers`` table with the"},{"line_number":83,"context_line":"resource we are working on an who is working on it.  And once the operation has"},{"line_number":84,"context_line":"been completed this row will be deleted to indicate processing has concluded"},{"line_number":85,"context_line":"and a cleanup will no longer be needed if the node dies."},{"line_number":86,"context_line":""}],"source_content_type":"text/x-rst","patch_set":8,"id":"1a122d0e_a698f46b","line":83,"updated":"2016-05-02 16:36:35.000000000","message":"nit: \"an\" should be \"and\"","commit_id":"9f6e0081662d522a2d7fb2bfd894fca6aa964ad7"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"04f7ae35d10f665a5e24f3dbfb62f09f197e05cf","unresolved":false,"context_lines":[{"line_number":80,"context_line":"When a cleanable job is requested by the API or any of the services -for"},{"line_number":81,"context_line":"example a volume deletion can be requested by the API or by the c-vol service"},{"line_number":82,"context_line":"during a migration- we will create a new row in the ``workers`` table with the"},{"line_number":83,"context_line":"resource we are working on an who is working on it.  And once the operation has"},{"line_number":84,"context_line":"been completed this row will be deleted to indicate processing has concluded"},{"line_number":85,"context_line":"and a cleanup will no longer be needed if the node dies."},{"line_number":86,"context_line":""}],"source_content_type":"text/x-rst","patch_set":8,"id":"3aaa91ec_7412ca0a","line":83,"in_reply_to":"1a122d0e_a698f46b","updated":"2016-06-29 11:23:13.000000000","message":"Done","commit_id":"9f6e0081662d522a2d7fb2bfd894fca6aa964ad7"},{"author":{"_account_id":12924,"name":"Patrick East","email":"east.patrick@gmail.com","username":"patrick.east"},"change_message_id":"8dd13083631cd4d8c656ebe872b2e25f3fc3c22b","unresolved":false,"context_lines":[{"line_number":259,"context_line":"Developer impact"},{"line_number":260,"context_line":"----------------"},{"line_number":261,"context_line":""},{"line_number":262,"context_line":"None"},{"line_number":263,"context_line":""},{"line_number":264,"context_line":""},{"line_number":265,"context_line":"Implementation"}],"source_content_type":"text/x-rst","patch_set":8,"id":"1a122d0e_f011d405","line":262,"range":{"start_line":262,"start_character":0,"end_line":262,"end_character":4},"updated":"2016-04-22 22:17:47.000000000","message":"May be worthwhile to document in the devref what type of new functionality in cinder should be using the workers table and is considered \u0027cleanable\u0027. Might be easier for folks not super familiar with the way it works to steer clear of adding new ones superfluously.","commit_id":"9f6e0081662d522a2d7fb2bfd894fca6aa964ad7"},{"author":{"_account_id":5997,"name":"Walt","display_name":"Hemna","email":"waboring@hemna.com","username":"walter-boring","status":"SAP"},"change_message_id":"68568d5f0a291500809907025ff7d6a5fd30fc18","unresolved":false,"context_lines":[{"line_number":259,"context_line":"Developer impact"},{"line_number":260,"context_line":"----------------"},{"line_number":261,"context_line":""},{"line_number":262,"context_line":"None"},{"line_number":263,"context_line":""},{"line_number":264,"context_line":""},{"line_number":265,"context_line":"Implementation"}],"source_content_type":"text/x-rst","patch_set":8,"id":"3aaa91ec_463a2c9a","line":262,"range":{"start_line":262,"start_character":0,"end_line":262,"end_character":4},"in_reply_to":"1a122d0e_f011d405","updated":"2016-06-24 21:04:46.000000000","message":"I would also like to see a list of the \u0027cleanable\u0027 operations here or in a devref.  This definitely will affect cinder developers working on new Cinder features.","commit_id":"9f6e0081662d522a2d7fb2bfd894fca6aa964ad7"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"04f7ae35d10f665a5e24f3dbfb62f09f197e05cf","unresolved":false,"context_lines":[{"line_number":259,"context_line":"Developer impact"},{"line_number":260,"context_line":"----------------"},{"line_number":261,"context_line":""},{"line_number":262,"context_line":"None"},{"line_number":263,"context_line":""},{"line_number":264,"context_line":""},{"line_number":265,"context_line":"Implementation"}],"source_content_type":"text/x-rst","patch_set":8,"id":"3aaa91ec_c9b04b8e","line":262,"range":{"start_line":262,"start_character":0,"end_line":262,"end_character":4},"in_reply_to":"3aaa91ec_463a2c9a","updated":"2016-06-29 11:23:13.000000000","message":"You are both right, I will update the developer impact section here and add to the work items the creation of the devref.\n\nI said it doesn\u0027t have a developer impact because we haven\u0027t changed anything on the cleanup for a long, long time.  We are cleaning the exact same resource states as we were always cleaning even though we have been adding new statuses, so in most cases it will have no impact.","commit_id":"9f6e0081662d522a2d7fb2bfd894fca6aa964ad7"},{"author":{"_account_id":6491,"name":"xing-yang","email":"xingyang105@gmail.com","username":"xing-yang"},"change_message_id":"57a79cd3cc57b3ea3f1e9b393be2e98fc8bd92dc","unresolved":false,"context_lines":[{"line_number":77,"context_line":""},{"line_number":78,"context_line":"Since checking for the status and the ``host`` field of the resource is no"},{"line_number":79,"context_line":"longer enough to know if it needs cleanup -because the ``host`` field will be"},{"line_number":80,"context_line":"referring to the ``host`` configuration of the volume service  that created the"},{"line_number":81,"context_line":"resource and not the owner of the resource as explained in the `Job"},{"line_number":82,"context_line":"Distribution`_ specs- we will create a new table to track which service from"},{"line_number":83,"context_line":"the cluster is working on each resource."}],"source_content_type":"text/x-rst","patch_set":10,"id":"1aa78d24_9dbfe2fb","line":80,"updated":"2016-07-14 13:43:56.000000000","message":"nit: double space between \"service\" and \"that\".","commit_id":"7b9dcb89fd36b95df89f07ba7902b9f11ad9d008"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"f3301c9125eaedc93f33d2754be322af7b2f2274","unresolved":false,"context_lines":[{"line_number":122,"context_line":"the scheduler decide which services should be cleaned up based on their status"},{"line_number":123,"context_line":"and how long they have been down."},{"line_number":124,"context_line":""},{"line_number":125,"context_line":"Automatic cleanup mechanism will consist of a periodic task that will sample"},{"line_number":126,"context_line":"services that are down, with a frequency of ``service_down_time`` seconds, and"},{"line_number":127,"context_line":"will proceed to clean up resources that were left by those services that are"},{"line_number":128,"context_line":"down after ``auto_cleanup_checks`` x ``service_down_time`` seconds have passed"},{"line_number":129,"context_line":"since the service went down."},{"line_number":130,"context_line":""},{"line_number":131,"context_line":"Since we can have multiple Scheduler services and the cinder-volume service all"},{"line_number":132,"context_line":"trying to do the cleanup simultaneously, code needs to be able handle these"}],"source_content_type":"text/x-rst","patch_set":10,"id":"1aa78d24_f116694a","line":129,"range":{"start_line":125,"start_character":0,"end_line":129,"end_character":28},"updated":"2016-07-07 10:22:28.000000000","message":"I don\u0027t think services table in DB is a 100% reliable source of truth when it comes to service liveness. Therefore I would be very reluctant to turn on automatic cleanup mechanism in  my cloud. It\u0027s just too many situations that can go wrong. I suggest should be a P2 item for now.","commit_id":"7b9dcb89fd36b95df89f07ba7902b9f11ad9d008"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"b9513ae75c419a642455b98a304a126249e5851b","unresolved":false,"context_lines":[{"line_number":122,"context_line":"the scheduler decide which services should be cleaned up based on their status"},{"line_number":123,"context_line":"and how long they have been down."},{"line_number":124,"context_line":""},{"line_number":125,"context_line":"Automatic cleanup mechanism will consist of a periodic task that will sample"},{"line_number":126,"context_line":"services that are down, with a frequency of ``service_down_time`` seconds, and"},{"line_number":127,"context_line":"will proceed to clean up resources that were left by those services that are"},{"line_number":128,"context_line":"down after ``auto_cleanup_checks`` x ``service_down_time`` seconds have passed"},{"line_number":129,"context_line":"since the service went down."},{"line_number":130,"context_line":""},{"line_number":131,"context_line":"Since we can have multiple Scheduler services and the cinder-volume service all"},{"line_number":132,"context_line":"trying to do the cleanup simultaneously, code needs to be able handle these"}],"source_content_type":"text/x-rst","patch_set":10,"id":"1aa78d24_55d1e02e","line":129,"range":{"start_line":125,"start_character":0,"end_line":129,"end_character":28},"in_reply_to":"1aa78d24_37cc3d69","updated":"2016-07-14 09:38:55.000000000","message":":)","commit_id":"7b9dcb89fd36b95df89f07ba7902b9f11ad9d008"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"ac9ac5ac9d47614d3efa07eb9491a8e3d6fcf6de","unresolved":false,"context_lines":[{"line_number":122,"context_line":"the scheduler decide which services should be cleaned up based on their status"},{"line_number":123,"context_line":"and how long they have been down."},{"line_number":124,"context_line":""},{"line_number":125,"context_line":"Automatic cleanup mechanism will consist of a periodic task that will sample"},{"line_number":126,"context_line":"services that are down, with a frequency of ``service_down_time`` seconds, and"},{"line_number":127,"context_line":"will proceed to clean up resources that were left by those services that are"},{"line_number":128,"context_line":"down after ``auto_cleanup_checks`` x ``service_down_time`` seconds have passed"},{"line_number":129,"context_line":"since the service went down."},{"line_number":130,"context_line":""},{"line_number":131,"context_line":"Since we can have multiple Scheduler services and the cinder-volume service all"},{"line_number":132,"context_line":"trying to do the cleanup simultaneously, code needs to be able handle these"}],"source_content_type":"text/x-rst","patch_set":10,"id":"1aa78d24_37cc3d69","line":129,"range":{"start_line":125,"start_character":0,"end_line":129,"end_character":28},"in_reply_to":"1aa78d24_f116694a","updated":"2016-07-13 17:18:27.000000000","message":"Oh, it is a P100 item at this time, don\u0027t worry.  ;-)","commit_id":"7b9dcb89fd36b95df89f07ba7902b9f11ad9d008"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"f3301c9125eaedc93f33d2754be322af7b2f2274","unresolved":false,"context_lines":[{"line_number":132,"context_line":"trying to do the cleanup simultaneously, code needs to be able handle these"},{"line_number":133,"context_line":"situations."},{"line_number":134,"context_line":""},{"line_number":135,"context_line":"On one hand, to prevent multiple Schedulers from cleaning the same services\u0027s"},{"line_number":136,"context_line":"resources they will be reporting all automatic cleanup operations requested to"},{"line_number":137,"context_line":"the cinder-volumes to the other Scheduler services and will ask other scheduler"},{"line_number":138,"context_line":"services which services have already been cleaned on service start."},{"line_number":139,"context_line":""},{"line_number":140,"context_line":"On the other hand, to prevent cleanup concurrency issues if a cleanup is"},{"line_number":141,"context_line":"requested on a service that is already being cleaned up, we will issue all"}],"source_content_type":"text/x-rst","patch_set":10,"id":"1aa78d24_ce246c0e","line":138,"range":{"start_line":135,"start_character":0,"end_line":138,"end_character":67},"updated":"2016-07-07 10:22:28.000000000","message":"This seems very race-condition prone to me. What if two schedulers report at the same time?","commit_id":"7b9dcb89fd36b95df89f07ba7902b9f11ad9d008"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"b9513ae75c419a642455b98a304a126249e5851b","unresolved":false,"context_lines":[{"line_number":132,"context_line":"trying to do the cleanup simultaneously, code needs to be able handle these"},{"line_number":133,"context_line":"situations."},{"line_number":134,"context_line":""},{"line_number":135,"context_line":"On one hand, to prevent multiple Schedulers from cleaning the same services\u0027s"},{"line_number":136,"context_line":"resources they will be reporting all automatic cleanup operations requested to"},{"line_number":137,"context_line":"the cinder-volumes to the other Scheduler services and will ask other scheduler"},{"line_number":138,"context_line":"services which services have already been cleaned on service start."},{"line_number":139,"context_line":""},{"line_number":140,"context_line":"On the other hand, to prevent cleanup concurrency issues if a cleanup is"},{"line_number":141,"context_line":"requested on a service that is already being cleaned up, we will issue all"}],"source_content_type":"text/x-rst","patch_set":10,"id":"1aa78d24_f871e145","line":138,"range":{"start_line":135,"start_character":0,"end_line":138,"end_character":67},"in_reply_to":"1aa78d24_17ff8134","updated":"2016-07-14 09:38:55.000000000","message":"I\u0027m still worried of reporting start of the operation between schedulers, but if cleanup is done with some kind of conditional update, then we can address the problem of wasted CPU later.","commit_id":"7b9dcb89fd36b95df89f07ba7902b9f11ad9d008"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"ac9ac5ac9d47614d3efa07eb9491a8e3d6fcf6de","unresolved":false,"context_lines":[{"line_number":132,"context_line":"trying to do the cleanup simultaneously, code needs to be able handle these"},{"line_number":133,"context_line":"situations."},{"line_number":134,"context_line":""},{"line_number":135,"context_line":"On one hand, to prevent multiple Schedulers from cleaning the same services\u0027s"},{"line_number":136,"context_line":"resources they will be reporting all automatic cleanup operations requested to"},{"line_number":137,"context_line":"the cinder-volumes to the other Scheduler services and will ask other scheduler"},{"line_number":138,"context_line":"services which services have already been cleaned on service start."},{"line_number":139,"context_line":""},{"line_number":140,"context_line":"On the other hand, to prevent cleanup concurrency issues if a cleanup is"},{"line_number":141,"context_line":"requested on a service that is already being cleaned up, we will issue all"}],"source_content_type":"text/x-rst","patch_set":10,"id":"1aa78d24_17ff8134","line":138,"range":{"start_line":135,"start_character":0,"end_line":138,"end_character":67},"in_reply_to":"1aa78d24_ce246c0e","updated":"2016-07-13 17:18:27.000000000","message":"It\u0027s not a problem, everything works even if all schedulers request the cleanup at the same time, it\u0027s just that it is wasteful in terms of cpu, network bandwidth, and DB usage.  So that\u0027s all this is trying to achieve reduce the likelihood of that happening.","commit_id":"7b9dcb89fd36b95df89f07ba7902b9f11ad9d008"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"f3301c9125eaedc93f33d2754be322af7b2f2274","unresolved":false,"context_lines":[{"line_number":167,"context_line":"on that resource (claiming the ``worker``\u0027s entry) and that the data on the"},{"line_number":168,"context_line":"``workers`` entry is still valid for the given resource (status matches) since"},{"line_number":169,"context_line":"a user may have forcefully issued another action on the resource in the"},{"line_number":170,"context_line":"meantime.."},{"line_number":171,"context_line":""},{"line_number":172,"context_line":""},{"line_number":173,"context_line":"Alternatives"}],"source_content_type":"text/x-rst","patch_set":10,"id":"1aa78d24_51649db7","line":170,"range":{"start_line":170,"start_character":8,"end_line":170,"end_character":10},"updated":"2016-07-07 10:22:28.000000000","message":"Double dots?","commit_id":"7b9dcb89fd36b95df89f07ba7902b9f11ad9d008"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"ac9ac5ac9d47614d3efa07eb9491a8e3d6fcf6de","unresolved":false,"context_lines":[{"line_number":167,"context_line":"on that resource (claiming the ``worker``\u0027s entry) and that the data on the"},{"line_number":168,"context_line":"``workers`` entry is still valid for the given resource (status matches) since"},{"line_number":169,"context_line":"a user may have forcefully issued another action on the resource in the"},{"line_number":170,"context_line":"meantime.."},{"line_number":171,"context_line":""},{"line_number":172,"context_line":""},{"line_number":173,"context_line":"Alternatives"}],"source_content_type":"text/x-rst","patch_set":10,"id":"1aa78d24_b764ad21","line":170,"range":{"start_line":170,"start_character":8,"end_line":170,"end_character":10},"in_reply_to":"1aa78d24_51649db7","updated":"2016-07-13 17:18:27.000000000","message":"Done, will upload when there are more changes to current version.","commit_id":"7b9dcb89fd36b95df89f07ba7902b9f11ad9d008"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"f3301c9125eaedc93f33d2754be322af7b2f2274","unresolved":false,"context_lines":[{"line_number":216,"context_line":"- ``deleted_at``: Will not be used"},{"line_number":217,"context_line":"- ``resource_type``: Resource type (Volume, Backup, Snapshot...)"},{"line_number":218,"context_line":"- ``resource_id``: UUID of the resource"},{"line_number":219,"context_line":"- ``status``: The status that should be cleaned on service failure"},{"line_number":220,"context_line":"- ``service_id``: service working on the resource"},{"line_number":221,"context_line":""},{"line_number":222,"context_line":""}],"source_content_type":"text/x-rst","patch_set":10,"id":"1aa78d24_5195bd30","line":219,"range":{"start_line":219,"start_character":0,"end_line":219,"end_character":66},"updated":"2016-07-07 10:22:28.000000000","message":"You mean something like \"previous_status\"? Or \"on_failure_status\"?\n\n\"status\" mnemonic seems a little vague here.","commit_id":"7b9dcb89fd36b95df89f07ba7902b9f11ad9d008"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"ac9ac5ac9d47614d3efa07eb9491a8e3d6fcf6de","unresolved":false,"context_lines":[{"line_number":216,"context_line":"- ``deleted_at``: Will not be used"},{"line_number":217,"context_line":"- ``resource_type``: Resource type (Volume, Backup, Snapshot...)"},{"line_number":218,"context_line":"- ``resource_id``: UUID of the resource"},{"line_number":219,"context_line":"- ``status``: The status that should be cleaned on service failure"},{"line_number":220,"context_line":"- ``service_id``: service working on the resource"},{"line_number":221,"context_line":""},{"line_number":222,"context_line":""}],"source_content_type":"text/x-rst","patch_set":10,"id":"1aa78d24_d7121960","line":219,"range":{"start_line":219,"start_character":0,"end_line":219,"end_character":66},"in_reply_to":"1aa78d24_5195bd30","updated":"2016-07-13 17:18:27.000000000","message":"No, as in the status the resource had when the cleanable operation begun.  Let\u0027s say you start a deleting operation, then this would be \"deleting\".","commit_id":"7b9dcb89fd36b95df89f07ba7902b9f11ad9d008"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"b9513ae75c419a642455b98a304a126249e5851b","unresolved":false,"context_lines":[{"line_number":216,"context_line":"- ``deleted_at``: Will not be used"},{"line_number":217,"context_line":"- ``resource_type``: Resource type (Volume, Backup, Snapshot...)"},{"line_number":218,"context_line":"- ``resource_id``: UUID of the resource"},{"line_number":219,"context_line":"- ``status``: The status that should be cleaned on service failure"},{"line_number":220,"context_line":"- ``service_id``: service working on the resource"},{"line_number":221,"context_line":""},{"line_number":222,"context_line":""}],"source_content_type":"text/x-rst","patch_set":10,"id":"1aa78d24_f5156ce5","line":219,"range":{"start_line":219,"start_character":0,"end_line":219,"end_character":66},"in_reply_to":"1aa78d24_d7121960","updated":"2016-07-14 09:38:55.000000000","message":"Oh, okay, that\u0027s pretty obvious.","commit_id":"7b9dcb89fd36b95df89f07ba7902b9f11ad9d008"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"f3301c9125eaedc93f33d2754be322af7b2f2274","unresolved":false,"context_lines":[{"line_number":232,"context_line":"be cleaned using parameters `service_id`, `cluster_name`, `host`, `binary`,"},{"line_number":233,"context_line":"`disabled`."},{"line_number":234,"context_line":""},{"line_number":235,"context_line":"Cleaning specific resources is also possible using `resource_type` and"},{"line_number":236,"context_line":"`resource_id` parameters."},{"line_number":237,"context_line":""},{"line_number":238,"context_line":"Cleanup cannot be triggered during a cloud upgrade, but a restarted service"},{"line_number":239,"context_line":"will still cleanup it\u0027s own resources during an upgrade."}],"source_content_type":"text/x-rst","patch_set":10,"id":"1aa78d24_d1c88d57","line":236,"range":{"start_line":235,"start_character":0,"end_line":236,"end_character":25},"updated":"2016-07-07 10:22:28.000000000","message":"Interesting. Isn\u0027t it redundant to reset-state? Seems even like a superior solution to me.","commit_id":"7b9dcb89fd36b95df89f07ba7902b9f11ad9d008"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"b9513ae75c419a642455b98a304a126249e5851b","unresolved":false,"context_lines":[{"line_number":232,"context_line":"be cleaned using parameters `service_id`, `cluster_name`, `host`, `binary`,"},{"line_number":233,"context_line":"`disabled`."},{"line_number":234,"context_line":""},{"line_number":235,"context_line":"Cleaning specific resources is also possible using `resource_type` and"},{"line_number":236,"context_line":"`resource_id` parameters."},{"line_number":237,"context_line":""},{"line_number":238,"context_line":"Cleanup cannot be triggered during a cloud upgrade, but a restarted service"},{"line_number":239,"context_line":"will still cleanup it\u0027s own resources during an upgrade."}],"source_content_type":"text/x-rst","patch_set":10,"id":"1aa78d24_588d9529","line":236,"range":{"start_line":235,"start_character":0,"end_line":236,"end_character":25},"in_reply_to":"1aa78d24_8b286d87","updated":"2016-07-14 09:38:55.000000000","message":"Good point.","commit_id":"7b9dcb89fd36b95df89f07ba7902b9f11ad9d008"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"ac9ac5ac9d47614d3efa07eb9491a8e3d6fcf6de","unresolved":false,"context_lines":[{"line_number":232,"context_line":"be cleaned using parameters `service_id`, `cluster_name`, `host`, `binary`,"},{"line_number":233,"context_line":"`disabled`."},{"line_number":234,"context_line":""},{"line_number":235,"context_line":"Cleaning specific resources is also possible using `resource_type` and"},{"line_number":236,"context_line":"`resource_id` parameters."},{"line_number":237,"context_line":""},{"line_number":238,"context_line":"Cleanup cannot be triggered during a cloud upgrade, but a restarted service"},{"line_number":239,"context_line":"will still cleanup it\u0027s own resources during an upgrade."}],"source_content_type":"text/x-rst","patch_set":10,"id":"1aa78d24_8b286d87","line":236,"range":{"start_line":235,"start_character":0,"end_line":236,"end_character":25},"in_reply_to":"1aa78d24_d1c88d57","updated":"2016-07-13 17:18:27.000000000","message":"Well, reset-state \"works\" even when there is not service to do anything, and it can also handle other cases like changing error to available and things like that.","commit_id":"7b9dcb89fd36b95df89f07ba7902b9f11ad9d008"}]}
