)]}'
{"specs/mitaka/ha-aa-prevent-corruption.rst":[{"author":{"_account_id":7173,"name":"Scott DAngelo","email":"scott.dangelo@gmail.com","username":"scottda"},"change_message_id":"fc6460165fb395829d70d46688370612f4fa1475","unresolved":false,"context_lines":[{"line_number":13,"context_line":"Right now cinder-volume service can run only in Active/Passive HA fashion."},{"line_number":14,"context_line":""},{"line_number":15,"context_line":"One of the reasons for this is that a node will not stop processing or"},{"line_number":16,"context_line":"receiving requests for as long as he is running, regardless of the connection"},{"line_number":17,"context_line":"to the DB, Message Broker and Storage Backend."},{"line_number":18,"context_line":""},{"line_number":19,"context_line":"This is fine when only Active/Passive configurations are allowed, but when we"}],"source_content_type":"text/x-rst","patch_set":1,"id":"3a29b11f_8251e6da","line":16,"updated":"2015-10-20 21:16:31.000000000","message":"\"he -\u003e he or she\"\nRemember, we encourage female nodes to run OpenStack :)","commit_id":"78355bf35e1f62def677513236e65c3d2e92f844"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"275d84a6d2be1253c8d0941bab1b6e231529afe7","unresolved":false,"context_lines":[{"line_number":13,"context_line":"Right now cinder-volume service can run only in Active/Passive HA fashion."},{"line_number":14,"context_line":""},{"line_number":15,"context_line":"One of the reasons for this is that a node will not stop processing or"},{"line_number":16,"context_line":"receiving requests for as long as he is running, regardless of the connection"},{"line_number":17,"context_line":"to the DB, Message Broker and Storage Backend."},{"line_number":18,"context_line":""},{"line_number":19,"context_line":"This is fine when only Active/Passive configurations are allowed, but when we"}],"source_content_type":"text/x-rst","patch_set":1,"id":"3a29b11f_132b9195","line":16,"in_reply_to":"3a29b11f_8251e6da","updated":"2015-10-21 12:38:51.000000000","message":"Ok, changed to \u0027it\u0027, since reading it now it sounded weird as it was.  :-)","commit_id":"78355bf35e1f62def677513236e65c3d2e92f844"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"0aa9b0c503bbfdf4a54b5dcc362a4a15c6b0af7f","unresolved":false,"context_lines":[{"line_number":67,"context_line":""},{"line_number":68,"context_line":"We should also stop all ongoing jobs that are accessing resources on the"},{"line_number":69,"context_line":"Storage Backend, for this we\u0027ll need to add a ``kill_all`` method in the"},{"line_number":70,"context_line":"different Executors of ``futures`` module and in Oslo messaging Executors, as"},{"line_number":71,"context_line":"we don\u0027t currently have a way to kill jobs."},{"line_number":72,"context_line":""},{"line_number":73,"context_line":"When DB connection is restored and we start sending heartbeats we should do"}],"source_content_type":"text/x-rst","patch_set":1,"id":"3a29b11f_990926e0","line":70,"updated":"2015-10-20 18:45:42.000000000","message":"I believe you mean oslo\u0027s futurist library [1], which is hidden under futures module in TaskFlow.\n\n[1] https://github.com/openstack/futurist","commit_id":"78355bf35e1f62def677513236e65c3d2e92f844"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"275d84a6d2be1253c8d0941bab1b6e231529afe7","unresolved":false,"context_lines":[{"line_number":67,"context_line":""},{"line_number":68,"context_line":"We should also stop all ongoing jobs that are accessing resources on the"},{"line_number":69,"context_line":"Storage Backend, for this we\u0027ll need to add a ``kill_all`` method in the"},{"line_number":70,"context_line":"different Executors of ``futures`` module and in Oslo messaging Executors, as"},{"line_number":71,"context_line":"we don\u0027t currently have a way to kill jobs."},{"line_number":72,"context_line":""},{"line_number":73,"context_line":"When DB connection is restored and we start sending heartbeats we should do"}],"source_content_type":"text/x-rst","patch_set":1,"id":"3a29b11f_763f5387","line":70,"in_reply_to":"3a29b11f_990926e0","updated":"2015-10-21 12:38:51.000000000","message":"Well, that\u0027s the name of the file (_futures.py) and somehow I wrote it from memory instead of looking at the correct name of the library.  Changing it now.","commit_id":"78355bf35e1f62def677513236e65c3d2e92f844"},{"author":{"_account_id":7173,"name":"Scott DAngelo","email":"scott.dangelo@gmail.com","username":"scottda"},"change_message_id":"fc6460165fb395829d70d46688370612f4fa1475","unresolved":false,"context_lines":[{"line_number":75,"context_line":"Message Handler."},{"line_number":76,"context_line":""},{"line_number":77,"context_line":"We should increase the frequency of heartbeats to every second once a heartbeat"},{"line_number":78,"context_line":"fails and until one succeeds.  If we do this we could postpone stopping current"},{"line_number":79,"context_line":"jobs for a number of tries.  For example 1/6th of ``periodic_interval``."},{"line_number":80,"context_line":""},{"line_number":81,"context_line":"In the future we could add additional auto-fencing cases, for example when we"}],"source_content_type":"text/x-rst","patch_set":1,"id":"3a29b11f_824566b4","line":78,"updated":"2015-10-20 21:16:31.000000000","message":"It is not clear what this means.","commit_id":"78355bf35e1f62def677513236e65c3d2e92f844"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"275d84a6d2be1253c8d0941bab1b6e231529afe7","unresolved":false,"context_lines":[{"line_number":75,"context_line":"Message Handler."},{"line_number":76,"context_line":""},{"line_number":77,"context_line":"We should increase the frequency of heartbeats to every second once a heartbeat"},{"line_number":78,"context_line":"fails and until one succeeds.  If we do this we could postpone stopping current"},{"line_number":79,"context_line":"jobs for a number of tries.  For example 1/6th of ``periodic_interval``."},{"line_number":80,"context_line":""},{"line_number":81,"context_line":"In the future we could add additional auto-fencing cases, for example when we"}],"source_content_type":"text/x-rst","patch_set":1,"id":"3a29b11f_113069e7","line":78,"in_reply_to":"3a29b11f_824566b4","updated":"2015-10-21 12:38:51.000000000","message":"You are right, that was only clear to me that knew what I meant.  ;-)\n\nI have improved the explanation.","commit_id":"78355bf35e1f62def677513236e65c3d2e92f844"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"0aa9b0c503bbfdf4a54b5dcc362a4a15c6b0af7f","unresolved":false,"context_lines":[{"line_number":76,"context_line":""},{"line_number":77,"context_line":"We should increase the frequency of heartbeats to every second once a heartbeat"},{"line_number":78,"context_line":"fails and until one succeeds.  If we do this we could postpone stopping current"},{"line_number":79,"context_line":"jobs for a number of tries.  For example 1/6th of ``periodic_interval``."},{"line_number":80,"context_line":""},{"line_number":81,"context_line":"In the future we could add additional auto-fencing cases, for example when we"},{"line_number":82,"context_line":"lose connection to the Storage Backend or if we are using a DLM when we lose"}],"source_content_type":"text/x-rst","patch_set":1,"id":"3a29b11f_b9774239","line":79,"updated":"2015-10-20 18:45:42.000000000","message":"This is something I\u0027m a little afraid. We\u0027re actually trying to implement a cluster management system and there are cluster management systems that work quite well... I get that this is outcome of a decision to not use a DLM.\n\nInstead of increasing frequency we can implement alternative Service Groups driver using Tooz and have information on that almost instantly.","commit_id":"78355bf35e1f62def677513236e65c3d2e92f844"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"275d84a6d2be1253c8d0941bab1b6e231529afe7","unresolved":false,"context_lines":[{"line_number":76,"context_line":""},{"line_number":77,"context_line":"We should increase the frequency of heartbeats to every second once a heartbeat"},{"line_number":78,"context_line":"fails and until one succeeds.  If we do this we could postpone stopping current"},{"line_number":79,"context_line":"jobs for a number of tries.  For example 1/6th of ``periodic_interval``."},{"line_number":80,"context_line":""},{"line_number":81,"context_line":"In the future we could add additional auto-fencing cases, for example when we"},{"line_number":82,"context_line":"lose connection to the Storage Backend or if we are using a DLM when we lose"}],"source_content_type":"text/x-rst","patch_set":1,"id":"3a29b11f_91b799f4","line":79,"in_reply_to":"3a29b11f_b9774239","updated":"2015-10-21 12:38:51.000000000","message":"I am not happy with this spec, but we really need this to prevent issues with the cleanup.\n\nUnfortunately this is not related to the DLM decision, since we would also have this problem if the node lost the connection to the DLM.  In our case the DLM is the DB.\n\nI don\u0027t see how that would help us, increasing the frequency is basically a way to prevent false positives.  Could you elaborate?","commit_id":"78355bf35e1f62def677513236e65c3d2e92f844"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"0aa9b0c503bbfdf4a54b5dcc362a4a15c6b0af7f","unresolved":false,"context_lines":[{"line_number":90,"context_line":"Alternatives"},{"line_number":91,"context_line":"------------"},{"line_number":92,"context_line":""},{"line_number":93,"context_line":"We could just abruptly exit the running program and let an external cluster"},{"line_number":94,"context_line":"manager fix things."},{"line_number":95,"context_line":""},{"line_number":96,"context_line":"Or we could create our own monitor that will monitor when DB connection is lost"}],"source_content_type":"text/x-rst","patch_set":1,"id":"3a29b11f_b9f3c29d","line":93,"updated":"2015-10-20 18:45:42.000000000","message":"This may be something to think of - what if we\u0027re running Cinder under Pacemaker? Does auto-fencing interfere with Pacemaker\u0027s fencing? I don\u0027t see any conflict yet, but - just a food for thoughts.","commit_id":"78355bf35e1f62def677513236e65c3d2e92f844"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"275d84a6d2be1253c8d0941bab1b6e231529afe7","unresolved":false,"context_lines":[{"line_number":90,"context_line":"Alternatives"},{"line_number":91,"context_line":"------------"},{"line_number":92,"context_line":""},{"line_number":93,"context_line":"We could just abruptly exit the running program and let an external cluster"},{"line_number":94,"context_line":"manager fix things."},{"line_number":95,"context_line":""},{"line_number":96,"context_line":"Or we could create our own monitor that will monitor when DB connection is lost"}],"source_content_type":"text/x-rst","patch_set":1,"id":"3a29b11f_0ce37234","line":93,"in_reply_to":"3a29b11f_b9f3c29d","updated":"2015-10-21 12:38:51.000000000","message":"It doesn\u0027t interfere, and as I see it they handle different cases, as the node could still be accessible from pacemaker but have no DB connection.","commit_id":"78355bf35e1f62def677513236e65c3d2e92f844"},{"author":{"_account_id":7173,"name":"Scott DAngelo","email":"scott.dangelo@gmail.com","username":"scottda"},"change_message_id":"fc6460165fb395829d70d46688370612f4fa1475","unresolved":false,"context_lines":[{"line_number":95,"context_line":""},{"line_number":96,"context_line":"Or we could create our own monitor that will monitor when DB connection is lost"},{"line_number":97,"context_line":"and kills Cinder -or waits for Cinder to exit because of DB connection issue-"},{"line_number":98,"context_line":"and waits for the connection to be restored before restarting the service."},{"line_number":99,"context_line":""},{"line_number":100,"context_line":"Data model impact"},{"line_number":101,"context_line":"-----------------"}],"source_content_type":"text/x-rst","patch_set":1,"id":"3a29b11f_a382ab92","line":98,"updated":"2015-10-20 21:16:31.000000000","message":"I think this idea shows promise. After all, isn\u0027t that what Active-Active HA is for? We just kill the c-vol that needs fencing and let the other c-vol(s) take over.\nAs for the resumption of running jobs, you had outlined some changes to the way we ACK messages off the AMQP queue in a blog post, waiting to ACK until the task was complete. Could this be used?\nCleanup might be required for volumes stuck in *-ing states, but we\u0027ve already got that problem to solve.","commit_id":"78355bf35e1f62def677513236e65c3d2e92f844"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"275d84a6d2be1253c8d0941bab1b6e231529afe7","unresolved":false,"context_lines":[{"line_number":95,"context_line":""},{"line_number":96,"context_line":"Or we could create our own monitor that will monitor when DB connection is lost"},{"line_number":97,"context_line":"and kills Cinder -or waits for Cinder to exit because of DB connection issue-"},{"line_number":98,"context_line":"and waits for the connection to be restored before restarting the service."},{"line_number":99,"context_line":""},{"line_number":100,"context_line":"Data model impact"},{"line_number":101,"context_line":"-----------------"}],"source_content_type":"text/x-rst","patch_set":1,"id":"3a29b11f_0cd7f2ed","line":98,"in_reply_to":"3a29b11f_a382ab92","updated":"2015-10-21 12:38:51.000000000","message":"I\u0027m not totally against this, but I\u0027d rather not have to create an additional monitoring application that needs configuration and maintenance.\n\nYes, I had a working implementation of changing the ACK of AMQP messages, but it is a little complex and I think we can resume jobs using the ``workers`` table and/or persistence of Taskflow jobs.  But I\u0027d rather leave that for later, as the HA A/A problem is hard enough as it is without adding new functionality.  :-)","commit_id":"78355bf35e1f62def677513236e65c3d2e92f844"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"0aa9b0c503bbfdf4a54b5dcc362a4a15c6b0af7f","unresolved":false,"context_lines":[{"line_number":152,"context_line":"Work Items"},{"line_number":153,"context_line":"----------"},{"line_number":154,"context_line":""},{"line_number":155,"context_line":"- Add ``kill_all`` method to ``futures`` executors that stops all ongoing"},{"line_number":156,"context_line":"  threads/greenthreads and clears queued jobs."},{"line_number":157,"context_line":""},{"line_number":158,"context_line":"- Add ``kill_all`` method to Oslo Messaging Executors, or at least add an"}],"source_content_type":"text/x-rst","patch_set":1,"id":"3a29b11f_9967e624","line":155,"updated":"2015-10-20 18:45:42.000000000","message":"If I understand correctly that would be oslo\u0027s futurist library + TaskFlow.","commit_id":"78355bf35e1f62def677513236e65c3d2e92f844"},{"author":{"_account_id":11600,"name":"Michał Dulko","email":"michal.dulko@gmail.com","username":"dulek"},"change_message_id":"0aa9b0c503bbfdf4a54b5dcc362a4a15c6b0af7f","unresolved":false,"context_lines":[{"line_number":157,"context_line":""},{"line_number":158,"context_line":"- Add ``kill_all`` method to Oslo Messaging Executors, or at least add an"},{"line_number":159,"context_line":"  optional force argument that defaults to False to current ``stop`` method so"},{"line_number":160,"context_line":"  it can call ``kill_all`` method from ``futures``."},{"line_number":161,"context_line":""},{"line_number":162,"context_line":"- Add calls to ``stop`` and ``kill_all`` in  Service\u0027s ``report_state`` method."},{"line_number":163,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"3a29b11f_fc3b68cf","line":160,"updated":"2015-10-20 18:45:42.000000000","message":"I may be wrong but from brief look into oslo.messaging I see that it is also using futurist. That would mean we\u0027re getting two with one shot.","commit_id":"78355bf35e1f62def677513236e65c3d2e92f844"},{"author":{"_account_id":1297,"name":"Joshua Harlow","email":"jxharlow@godaddy.com","username":"harlowja"},"change_message_id":"6f5c3175bc8c246697a8e32c1739ff9f5dd91eaf","unresolved":false,"context_lines":[{"line_number":75,"context_line":"We should also stop all ongoing jobs that are accessing resources on the"},{"line_number":76,"context_line":"Storage Backend, for this we\u0027ll need to add a ``kill_all`` method in the"},{"line_number":77,"context_line":"different Executors of `futurist library`_ (in ``_futures.py`` file) and in"},{"line_number":78,"context_line":"Oslo messaging Executors, as we don\u0027t currently have a way to kill jobs."},{"line_number":79,"context_line":""},{"line_number":80,"context_line":"When DB connection is restored and we start sending heartbeats we should do"},{"line_number":81,"context_line":"resource cleanup -as explained in `HA A/A Cleanup specs`_- and ``start`` the"}],"source_content_type":"text/x-rst","patch_set":4,"id":"ba8a016a_c206921b","line":78,"updated":"2015-11-18 19:42:43.000000000","message":"Killing is dangerous imho, threads should be told they need to stop, and the thread should cleanly stop, killing is bad imho, it doesn\u0027t let the thread clean up itself, and that causes further bad things...\n\nSo I don\u0027t know if futurist is the right place for this, but instead the oslo messaging executors (which have a stop() method, should just be used, that will cleanly stop new messages from getting handled).\n\nKilling active things that cinder is working on is a whole different question, and doing that cleanly without isolated tasks and jobs and such is going to be hard (which is one of the reasons taskkflow has tasks and flows in the first place, because the taskflow engine can stop between tasks... and resume later or cleanup or...)\n\nSo no I\u0027m not really a fan of \u0027kill_all\u0027, stop() already exists, killing scares me...","commit_id":"732b5fc97d4a906ac441be8c02b523f5d48309c6"},{"author":{"_account_id":1297,"name":"Joshua Harlow","email":"jxharlow@godaddy.com","username":"harlowja"},"change_message_id":"c4c73a0deb175b92efd44dd2de206fbb00348644","unresolved":false,"context_lines":[{"line_number":75,"context_line":"We should also stop all ongoing jobs that are accessing resources on the"},{"line_number":76,"context_line":"Storage Backend, for this we\u0027ll need to add a ``kill_all`` method in the"},{"line_number":77,"context_line":"different Executors of `futurist library`_ (in ``_futures.py`` file) and in"},{"line_number":78,"context_line":"Oslo messaging Executors, as we don\u0027t currently have a way to kill jobs."},{"line_number":79,"context_line":""},{"line_number":80,"context_line":"When DB connection is restored and we start sending heartbeats we should do"},{"line_number":81,"context_line":"resource cleanup -as explained in `HA A/A Cleanup specs`_- and ``start`` the"}],"source_content_type":"text/x-rst","patch_set":4,"id":"ba8a016a_d97f3513","line":78,"in_reply_to":"ba8a016a_923d14df","updated":"2015-11-19 16:01:47.000000000","message":"Yes, but futurist can\u0027t support kill either thats the thing. U can\u0027t force regular threads to just die, at least not with any exposed python API (perhaps something in pthreads could do this). Greenthreads u can do something for and u can try to kill them, but greenthread execution is not the only API futurist supports.","commit_id":"732b5fc97d4a906ac441be8c02b523f5d48309c6"},{"author":{"_account_id":1297,"name":"Joshua Harlow","email":"jxharlow@godaddy.com","username":"harlowja"},"change_message_id":"65e3d3ac62024e3f78534636a671ce1813add32d","unresolved":false,"context_lines":[{"line_number":75,"context_line":"We should also stop all ongoing jobs that are accessing resources on the"},{"line_number":76,"context_line":"Storage Backend, for this we\u0027ll need to add a ``kill_all`` method in the"},{"line_number":77,"context_line":"different Executors of `futurist library`_ (in ``_futures.py`` file) and in"},{"line_number":78,"context_line":"Oslo messaging Executors, as we don\u0027t currently have a way to kill jobs."},{"line_number":79,"context_line":""},{"line_number":80,"context_line":"When DB connection is restored and we start sending heartbeats we should do"},{"line_number":81,"context_line":"resource cleanup -as explained in `HA A/A Cleanup specs`_- and ``start`` the"}],"source_content_type":"text/x-rst","patch_set":4,"id":"ba8a016a_f6b9a67a","line":78,"in_reply_to":"ba8a016a_a59b1f17","updated":"2015-11-20 16:24:27.000000000","message":"Yes, certain projects use regular threads only (keystone I think doesn\u0027t use any threads? cue and octavia might just use regular threads only so ya, they do exist, and futurist and such aren\u0027t just openstack libraries, so they really do have to cater to the larger audience).\n\nOslo.messaging also has executors that aren\u0027t going to be able to support kill:\n\nhttps://github.com/openstack/oslo.messaging/tree/master/oslo_messaging/_executors\n\nThe only ones that could in theory are the eventlet (and maybe the aioeventlet) ones... Which is 2 out of 5 :-/","commit_id":"732b5fc97d4a906ac441be8c02b523f5d48309c6"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"01b114e97d2b47a89eca4300d602a2e0c17a581d","unresolved":false,"context_lines":[{"line_number":75,"context_line":"We should also stop all ongoing jobs that are accessing resources on the"},{"line_number":76,"context_line":"Storage Backend, for this we\u0027ll need to add a ``kill_all`` method in the"},{"line_number":77,"context_line":"different Executors of `futurist library`_ (in ``_futures.py`` file) and in"},{"line_number":78,"context_line":"Oslo messaging Executors, as we don\u0027t currently have a way to kill jobs."},{"line_number":79,"context_line":""},{"line_number":80,"context_line":"When DB connection is restored and we start sending heartbeats we should do"},{"line_number":81,"context_line":"resource cleanup -as explained in `HA A/A Cleanup specs`_- and ``start`` the"}],"source_content_type":"text/x-rst","patch_set":4,"id":"ba8a016a_923d14df","line":78,"in_reply_to":"ba8a016a_c206921b","updated":"2015-11-19 12:53:21.000000000","message":"The part about using stop to receive messages is treated in the previous paragraph in the same manner you describe.  ;-)\n\nI\u0027m not a fan of killing things either, I like them stopping by themselves, but we have to be a little realistic here, we have 6 big changes (each with its own spec) that are needed to support Active-Active, so it\u0027s already a huge effort.  If we also want all operations to be able to stop properly it means we have to change *everything*, not only the manager but also all the drivers as well.\n\nSo yes, that would be the ideal solution, but I don\u0027t think that\u0027s something we can actually do any time soon.","commit_id":"732b5fc97d4a906ac441be8c02b523f5d48309c6"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"1900f0e961d3d309ab6fecfa65c582a3e9689c81","unresolved":false,"context_lines":[{"line_number":75,"context_line":"We should also stop all ongoing jobs that are accessing resources on the"},{"line_number":76,"context_line":"Storage Backend, for this we\u0027ll need to add a ``kill_all`` method in the"},{"line_number":77,"context_line":"different Executors of `futurist library`_ (in ``_futures.py`` file) and in"},{"line_number":78,"context_line":"Oslo messaging Executors, as we don\u0027t currently have a way to kill jobs."},{"line_number":79,"context_line":""},{"line_number":80,"context_line":"When DB connection is restored and we start sending heartbeats we should do"},{"line_number":81,"context_line":"resource cleanup -as explained in `HA A/A Cleanup specs`_- and ``start`` the"}],"source_content_type":"text/x-rst","patch_set":4,"id":"ba8a016a_a59b1f17","line":78,"in_reply_to":"ba8a016a_d97f3513","updated":"2015-11-19 18:03:37.000000000","message":"Are we using in openstack something other than greenthreads?","commit_id":"732b5fc97d4a906ac441be8c02b523f5d48309c6"},{"author":{"_account_id":1297,"name":"Joshua Harlow","email":"jxharlow@godaddy.com","username":"harlowja"},"change_message_id":"12d190abfb38e0a4eb100cb869c8b485b3ba3bb8","unresolved":false,"context_lines":[{"line_number":102,"context_line":"been started -if our connection to the Message Broker is not lost-, but this is"},{"line_number":103,"context_line":"quite far-fetched, since we would need to retrieve jobs from ``_delayed_work``"},{"line_number":104,"context_line":"queue in ``futures`` executor."},{"line_number":105,"context_line":""},{"line_number":106,"context_line":"This solution not only does not interfere with ``pacemaker`` but they"},{"line_number":107,"context_line":"complement each other, since they solve different problems.  Fencing proposed"},{"line_number":108,"context_line":"in this specification takes care of situations where we lose connection to the"}],"source_content_type":"text/x-rst","patch_set":4,"id":"ba8a016a_02149a9d","line":105,"updated":"2015-11-18 19:57:33.000000000","message":"Ya, hmmm,\n\nhttp://docs.openstack.org/developer/taskflow/jobs.html\n\n... (there are reasons that exists to, ha)","commit_id":"732b5fc97d4a906ac441be8c02b523f5d48309c6"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"01b114e97d2b47a89eca4300d602a2e0c17a581d","unresolved":false,"context_lines":[{"line_number":102,"context_line":"been started -if our connection to the Message Broker is not lost-, but this is"},{"line_number":103,"context_line":"quite far-fetched, since we would need to retrieve jobs from ``_delayed_work``"},{"line_number":104,"context_line":"queue in ``futures`` executor."},{"line_number":105,"context_line":""},{"line_number":106,"context_line":"This solution not only does not interfere with ``pacemaker`` but they"},{"line_number":107,"context_line":"complement each other, since they solve different problems.  Fencing proposed"},{"line_number":108,"context_line":"in this specification takes care of situations where we lose connection to the"}],"source_content_type":"text/x-rst","patch_set":4,"id":"ba8a016a_b215b89c","line":105,"in_reply_to":"ba8a016a_02149a9d","updated":"2015-11-19 12:53:21.000000000","message":"I don\u0027t understand what you mean with your comment.","commit_id":"732b5fc97d4a906ac441be8c02b523f5d48309c6"},{"author":{"_account_id":1297,"name":"Joshua Harlow","email":"jxharlow@godaddy.com","username":"harlowja"},"change_message_id":"c4c73a0deb175b92efd44dd2de206fbb00348644","unresolved":false,"context_lines":[{"line_number":102,"context_line":"been started -if our connection to the Message Broker is not lost-, but this is"},{"line_number":103,"context_line":"quite far-fetched, since we would need to retrieve jobs from ``_delayed_work``"},{"line_number":104,"context_line":"queue in ``futures`` executor."},{"line_number":105,"context_line":""},{"line_number":106,"context_line":"This solution not only does not interfere with ``pacemaker`` but they"},{"line_number":107,"context_line":"complement each other, since they solve different problems.  Fencing proposed"},{"line_number":108,"context_line":"in this specification takes care of situations where we lose connection to the"}],"source_content_type":"text/x-rst","patch_set":4,"id":"ba8a016a_f9f6b98d","line":105,"in_reply_to":"ba8a016a_b215b89c","updated":"2015-11-19 16:01:47.000000000","message":"That job implementation does automatic rescheduling to other nodes if the machine they are running on dies.","commit_id":"732b5fc97d4a906ac441be8c02b523f5d48309c6"},{"author":{"_account_id":1297,"name":"Joshua Harlow","email":"jxharlow@godaddy.com","username":"harlowja"},"change_message_id":"65e3d3ac62024e3f78534636a671ce1813add32d","unresolved":false,"context_lines":[{"line_number":102,"context_line":"been started -if our connection to the Message Broker is not lost-, but this is"},{"line_number":103,"context_line":"quite far-fetched, since we would need to retrieve jobs from ``_delayed_work``"},{"line_number":104,"context_line":"queue in ``futures`` executor."},{"line_number":105,"context_line":""},{"line_number":106,"context_line":"This solution not only does not interfere with ``pacemaker`` but they"},{"line_number":107,"context_line":"complement each other, since they solve different problems.  Fencing proposed"},{"line_number":108,"context_line":"in this specification takes care of situations where we lose connection to the"}],"source_content_type":"text/x-rst","patch_set":4,"id":"ba8a016a_5680b28b","line":105,"in_reply_to":"ba8a016a_e54367ad","updated":"2015-11-20 16:24:27.000000000","message":"Ya, agreed, the ideal world :-P","commit_id":"732b5fc97d4a906ac441be8c02b523f5d48309c6"},{"author":{"_account_id":1297,"name":"Joshua Harlow","email":"jxharlow@godaddy.com","username":"harlowja"},"change_message_id":"bab6c7c1a7436ea0e4c0b54c8a35a56a5919ad60","unresolved":false,"context_lines":[{"line_number":102,"context_line":"been started -if our connection to the Message Broker is not lost-, but this is"},{"line_number":103,"context_line":"quite far-fetched, since we would need to retrieve jobs from ``_delayed_work``"},{"line_number":104,"context_line":"queue in ``futures`` executor."},{"line_number":105,"context_line":""},{"line_number":106,"context_line":"This solution not only does not interfere with ``pacemaker`` but they"},{"line_number":107,"context_line":"complement each other, since they solve different problems.  Fencing proposed"},{"line_number":108,"context_line":"in this specification takes care of situations where we lose connection to the"}],"source_content_type":"text/x-rst","patch_set":4,"id":"ba8a016a_ca6bd283","line":105,"in_reply_to":"ba8a016a_f9f6b98d","updated":"2015-11-19 17:30:42.000000000","message":"If u get time try:\n\nhttps://github.com/openstack/taskflow/blob/master/taskflow/examples/99_bottles.py#L42\n\nWhich shows this in action...","commit_id":"732b5fc97d4a906ac441be8c02b523f5d48309c6"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"1900f0e961d3d309ab6fecfa65c582a3e9689c81","unresolved":false,"context_lines":[{"line_number":102,"context_line":"been started -if our connection to the Message Broker is not lost-, but this is"},{"line_number":103,"context_line":"quite far-fetched, since we would need to retrieve jobs from ``_delayed_work``"},{"line_number":104,"context_line":"queue in ``futures`` executor."},{"line_number":105,"context_line":""},{"line_number":106,"context_line":"This solution not only does not interfere with ``pacemaker`` but they"},{"line_number":107,"context_line":"complement each other, since they solve different problems.  Fencing proposed"},{"line_number":108,"context_line":"in this specification takes care of situations where we lose connection to the"}],"source_content_type":"text/x-rst","patch_set":4,"id":"ba8a016a_e54367ad","line":105,"in_reply_to":"ba8a016a_f9f6b98d","updated":"2015-11-19 18:03:37.000000000","message":"Oh that!, yes but we don\u0027t have Taksflow jobs for most of the Cinder tasks, so that\u0027s why I said that\u0027d be in an ideal world, because we cannot generalize our behavior from just a couple of operations that actually use them.  :-(","commit_id":"732b5fc97d4a906ac441be8c02b523f5d48309c6"},{"author":{"_account_id":1297,"name":"Joshua Harlow","email":"jxharlow@godaddy.com","username":"harlowja"},"change_message_id":"6f5c3175bc8c246697a8e32c1739ff9f5dd91eaf","unresolved":false,"context_lines":[{"line_number":113,"context_line":"------------"},{"line_number":114,"context_line":""},{"line_number":115,"context_line":"We could just abruptly exit the running program and let an external cluster"},{"line_number":116,"context_line":"manager fix things."},{"line_number":117,"context_line":""},{"line_number":118,"context_line":"Or we could create our own monitor that will monitor when DB connection is lost"},{"line_number":119,"context_line":"and kills Cinder -or waits for Cinder to exit because of DB connection issue-"}],"source_content_type":"text/x-rst","patch_set":4,"id":"ba8a016a_c26bf2b7","line":116,"updated":"2015-11-18 19:42:43.000000000","message":"I get this alternative being here but this imho is crap, we can do better as application creators...\n\nWe don\u0027t need a \u0027garbage/crap collection crew\u0027 to fix stuff in our own code imho, just fix the original code in the first place...","commit_id":"732b5fc97d4a906ac441be8c02b523f5d48309c6"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"01b114e97d2b47a89eca4300d602a2e0c17a581d","unresolved":false,"context_lines":[{"line_number":113,"context_line":"------------"},{"line_number":114,"context_line":""},{"line_number":115,"context_line":"We could just abruptly exit the running program and let an external cluster"},{"line_number":116,"context_line":"manager fix things."},{"line_number":117,"context_line":""},{"line_number":118,"context_line":"Or we could create our own monitor that will monitor when DB connection is lost"},{"line_number":119,"context_line":"and kills Cinder -or waits for Cinder to exit because of DB connection issue-"}],"source_content_type":"text/x-rst","patch_set":4,"id":"ba8a016a_6d1f755b","line":116,"in_reply_to":"ba8a016a_c26bf2b7","updated":"2015-11-19 12:53:21.000000000","message":"I don\u0027t like it either.  ;-)","commit_id":"732b5fc97d4a906ac441be8c02b523f5d48309c6"},{"author":{"_account_id":1297,"name":"Joshua Harlow","email":"jxharlow@godaddy.com","username":"harlowja"},"change_message_id":"6f5c3175bc8c246697a8e32c1739ff9f5dd91eaf","unresolved":false,"context_lines":[{"line_number":118,"context_line":"Or we could create our own monitor that will monitor when DB connection is lost"},{"line_number":119,"context_line":"and kills Cinder -or waits for Cinder to exit because of DB connection issue-"},{"line_number":120,"context_line":"and waits for the connection to be restored before restarting the service."},{"line_number":121,"context_line":""},{"line_number":122,"context_line":"Data model impact"},{"line_number":123,"context_line":"-----------------"},{"line_number":124,"context_line":""}],"source_content_type":"text/x-rst","patch_set":4,"id":"ba8a016a_426062cf","line":121,"updated":"2015-11-18 19:42:43.000000000","message":"Has same need for \u0027garbage/crap collection crew\u0027 as above.","commit_id":"732b5fc97d4a906ac441be8c02b523f5d48309c6"},{"author":{"_account_id":1297,"name":"Joshua Harlow","email":"jxharlow@godaddy.com","username":"harlowja"},"change_message_id":"6f5c3175bc8c246697a8e32c1739ff9f5dd91eaf","unresolved":false,"context_lines":[{"line_number":175,"context_line":"----------"},{"line_number":176,"context_line":""},{"line_number":177,"context_line":"- Add ``kill_all`` method to ``futures`` executors that stops all ongoing"},{"line_number":178,"context_line":"  threads/greenthreads and clears queued jobs."},{"line_number":179,"context_line":""},{"line_number":180,"context_line":"- Add ``kill_all`` method to Oslo Messaging Executors, or at least add an"},{"line_number":181,"context_line":"  optional force argument that defaults to False to current ``stop`` method so"}],"source_content_type":"text/x-rst","patch_set":4,"id":"ba8a016a_42b7023a","line":178,"updated":"2015-11-18 19:42:43.000000000","message":"https://docs.python.org/dev/library/concurrent.futures.html#concurrent.futures.Executor.shutdown already exists, and stop() exists on oslo.messaging executors, should be good enough imho, killing \u003d\u003d bad.","commit_id":"732b5fc97d4a906ac441be8c02b523f5d48309c6"},{"author":{"_account_id":1297,"name":"Joshua Harlow","email":"jxharlow@godaddy.com","username":"harlowja"},"change_message_id":"6f5c3175bc8c246697a8e32c1739ff9f5dd91eaf","unresolved":false,"context_lines":[{"line_number":179,"context_line":""},{"line_number":180,"context_line":"- Add ``kill_all`` method to Oslo Messaging Executors, or at least add an"},{"line_number":181,"context_line":"  optional force argument that defaults to False to current ``stop`` method so"},{"line_number":182,"context_line":"  it can call ``kill_all`` method from ``futures``."},{"line_number":183,"context_line":""},{"line_number":184,"context_line":"- Add calls to ``stop`` and ``kill_all`` in  Service\u0027s ``report_state`` method."},{"line_number":185,"context_line":""}],"source_content_type":"text/x-rst","patch_set":4,"id":"ba8a016a_22899e75","line":182,"updated":"2015-11-18 19:42:43.000000000","message":"No killing please...\n\nKilling threads without asking them to cleanup themeselves \u003d\u003d bad and will once again require a \u0027garbage/crap collection crew\u0027","commit_id":"732b5fc97d4a906ac441be8c02b523f5d48309c6"}],"specs/newton/ha-aa-prevent-corruption.rst":[{"author":{"_account_id":1207,"name":"Duncan Thomas","email":"duncan.thomas@gmail.com","username":"duncan-thomas"},"change_message_id":"ee29c29ca5b38951000613a040a10b6c43cbd800","unresolved":false,"context_lines":[{"line_number":58,"context_line":"Proposed change"},{"line_number":59,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":60,"context_line":""},{"line_number":61,"context_line":"The idea is for Volume nodes to be able to auto-fence themselves to prevent"},{"line_number":62,"context_line":"data corruption and picking new jobs when we know beforehand that we will not"},{"line_number":63,"context_line":"be able to carry them out (connection to the DB is lost or we cannot acquire"},{"line_number":64,"context_line":"locks)."},{"line_number":65,"context_line":""},{"line_number":66,"context_line":"This new auto-fencing mechanism will require two changes, one that prevents the"},{"line_number":67,"context_line":"node from getting new jobs and another that stops current jobs execution."}],"source_content_type":"text/x-rst","patch_set":5,"id":"bacf61ea_69c2a5ea","line":64,"range":{"start_line":61,"start_character":0,"end_line":64,"end_character":7},"updated":"2016-08-01 12:07:57.000000000","message":"How any you auto-fence situations like child dd processes stuck in D state during I/O? What about a node running at a crawl due to a a swap storm?\n\nThere are lots of corner cases like these where self fencing simple cannot work, and you end up with data corruption if you assume it can.","commit_id":"78299cf40ffe98ad23d827cabc447779ac370c52"},{"author":{"_account_id":2394,"name":"Adam Spiers","email":"aspiers@suse.com","username":"adam.spiers"},"change_message_id":"0d841e228362170974b13cae099968f2ef8ec0ad","unresolved":false,"context_lines":[{"line_number":58,"context_line":"Proposed change"},{"line_number":59,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":60,"context_line":""},{"line_number":61,"context_line":"The idea is for Volume nodes to be able to auto-fence themselves to prevent"},{"line_number":62,"context_line":"data corruption and picking new jobs when we know beforehand that we will not"},{"line_number":63,"context_line":"be able to carry them out (connection to the DB is lost or we cannot acquire"},{"line_number":64,"context_line":"locks)."},{"line_number":65,"context_line":""},{"line_number":66,"context_line":"This new auto-fencing mechanism will require two changes, one that prevents the"},{"line_number":67,"context_line":"node from getting new jobs and another that stops current jobs execution."}],"source_content_type":"text/x-rst","patch_set":5,"id":"5f4e5783_f7d8a37b","line":64,"range":{"start_line":61,"start_character":0,"end_line":64,"end_character":7},"in_reply_to":"bacf61ea_69c2a5ea","updated":"2017-10-13 19:29:50.000000000","message":"I strongly agree with Duncan\u0027s concerns here; there are definitely corner cases where you cannot rely on normal system behaviour and hence you cannot rely on in-band fencing.\n\nHowever it should still be possible to achieve reliable *out-of-band* self-fencing.  For example, SBD can accomplish this with the help of watchdogs:\n\nhttp://www.linux-ha.org/wiki/SBD_Fencing\nhttps://www.suse.com/documentation/sle_ha/book_sleha/data/sec_ha_storage_protect_fencing.html\nhttp://blog.clusterlabs.org/blog/2015/sbd-fun-and-profit\n\nI\u0027m not saying that SBD is necessarily the solution here, but the point is that reliable self-fencing *is* possible if you have watchdogs.\n\nAnd actually ... (although this is maybe not the best place for brainstorming) in this particular scenario we do have shared storage (by definition), and SBD works best when it is using the same storage which you are trying to protect via fencing.  So *maybe* an alternative approach here might be for cinder to use SBD for fencing, by reserving a tiny slice of the shared storage for its communication channel.  Or maybe that\u0027s a dumb idea.  I guess it depends on whether you want a proper cluster-wide quorum and fencing, or whether self-fencing of individual nodes is enough.  If the latter, maybe SBD without Pacemaker is a possible solution, but I\u0027m not sure.  I\u0027ll ask the guy who wrote it :-)","commit_id":"78299cf40ffe98ad23d827cabc447779ac370c52"},{"author":{"_account_id":1297,"name":"Joshua Harlow","email":"jxharlow@godaddy.com","username":"harlowja"},"change_message_id":"1bca8d2ae545fad6c0bebbdb38af367d81cc28b1","unresolved":false,"context_lines":[{"line_number":75,"context_line":"We should also stop all ongoing jobs that are accessing resources on the"},{"line_number":76,"context_line":"Storage Backend, for this we\u0027ll need to add a ``kill_all`` method in the"},{"line_number":77,"context_line":"different Executors of `futurist library`_ (in ``_futures.py`` file) and in"},{"line_number":78,"context_line":"Oslo messaging Executors, as we don\u0027t currently have a way to kill jobs."},{"line_number":79,"context_line":""},{"line_number":80,"context_line":"When DB connection is restored and we start sending heartbeats we should do"},{"line_number":81,"context_line":"resource cleanup -as explained in `HA A/A Cleanup specs`_- and ``start`` the"}],"source_content_type":"text/x-rst","patch_set":5,"id":"7af24918_2e2e1743","line":78,"updated":"2016-03-02 21:38:12.000000000","message":"This will be tough, but ok.","commit_id":"78299cf40ffe98ad23d827cabc447779ac370c52"},{"author":{"_account_id":2394,"name":"Adam Spiers","email":"aspiers@suse.com","username":"adam.spiers"},"change_message_id":"0d841e228362170974b13cae099968f2ef8ec0ad","unresolved":false,"context_lines":[{"line_number":113,"context_line":"------------"},{"line_number":114,"context_line":""},{"line_number":115,"context_line":"We could just abruptly exit the running program and let an external cluster"},{"line_number":116,"context_line":"manager fix things."},{"line_number":117,"context_line":""},{"line_number":118,"context_line":"Or we could create our own monitor that will monitor when DB connection is lost"},{"line_number":119,"context_line":"and kills Cinder -or waits for Cinder to exit because of DB connection issue-"}],"source_content_type":"text/x-rst","patch_set":5,"id":"5f4e5783_37c4fb5e","line":116,"updated":"2017-10-13 19:29:50.000000000","message":"If an external cluster manager was set up (say Pacemaker + SBD) then it could take care of everything - Cinder would not need to exit or kill anything.","commit_id":"78299cf40ffe98ad23d827cabc447779ac370c52"},{"author":{"_account_id":2394,"name":"Adam Spiers","email":"aspiers@suse.com","username":"adam.spiers"},"change_message_id":"0d841e228362170974b13cae099968f2ef8ec0ad","unresolved":false,"context_lines":[{"line_number":115,"context_line":"We could just abruptly exit the running program and let an external cluster"},{"line_number":116,"context_line":"manager fix things."},{"line_number":117,"context_line":""},{"line_number":118,"context_line":"Or we could create our own monitor that will monitor when DB connection is lost"},{"line_number":119,"context_line":"and kills Cinder -or waits for Cinder to exit because of DB connection issue-"},{"line_number":120,"context_line":"and waits for the connection to be restored before restarting the service."},{"line_number":121,"context_line":""}],"source_content_type":"text/x-rst","patch_set":5,"id":"5f4e5783_97d16f26","line":118,"updated":"2017-10-13 19:29:50.000000000","message":"As mentioned above you could not rely on that monitor if the kernel is in a bad state such as low memory / lots of blocking I/O etc.  That\u0027s why you would need a watchdog and some out-of-band fencing mechanism.","commit_id":"78299cf40ffe98ad23d827cabc447779ac370c52"},{"author":{"_account_id":1297,"name":"Joshua Harlow","email":"jxharlow@godaddy.com","username":"harlowja"},"change_message_id":"1bca8d2ae545fad6c0bebbdb38af367d81cc28b1","unresolved":false,"context_lines":[{"line_number":174,"context_line":"Work Items"},{"line_number":175,"context_line":"----------"},{"line_number":176,"context_line":""},{"line_number":177,"context_line":"- Add ``kill_all`` method to ``futures`` executors that stops all ongoing"},{"line_number":178,"context_line":"  threads/greenthreads and clears queued jobs."},{"line_number":179,"context_line":""},{"line_number":180,"context_line":"- Add ``kill_all`` method to Oslo Messaging Executors, or at least add an"}],"source_content_type":"text/x-rst","patch_set":5,"id":"7af24918_aee987e8","line":177,"updated":"2016-03-02 21:38:12.000000000","message":"The non-green-thread ones will be really hard, because the changes will be in the std-lib, not in futurist.","commit_id":"78299cf40ffe98ad23d827cabc447779ac370c52"}]}