)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"01b0e9b9cf4c39ea564e5d850f5ac68efab6664b","unresolved":false,"context_lines":[{"line_number":4,"context_line":"Commit:     MultipleCrashes \u003cinimitableharish@gmail.com\u003e"},{"line_number":5,"context_line":"CommitDate: 2018-05-07 22:49:09 +0530"},{"line_number":6,"context_line":""},{"line_number":7,"context_line":"NCP-2367-deallocate-network-retry"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"retry fix for VM going to ERROR state during auto-scale delete instance"},{"line_number":10,"context_line":""}],"source_content_type":"text/x-gerrit-commit-message","patch_set":4,"id":"5f7c97a3_e6770c3a","line":7,"range":{"start_line":7,"start_character":0,"end_line":7,"end_character":8},"updated":"2018-05-07 20:15:45.000000000","message":"I assume you\u0027re linking this back to your internal corporate JIRA somehow? This doesn\u0027t mean anything upstream where we don\u0027t use JIRA to track issues, we just use launchpad.","commit_id":"6ccd2f527873ba289be0aaeac9367ba811be6366"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"9c0d6e316909842dd884f575c872768e4616a1b5","unresolved":false,"context_lines":[{"line_number":15,"context_line":"instance vm_state to ERROR\".Failure in do_terminate_instance."},{"line_number":16,"context_line":""},{"line_number":17,"context_line":"Change-Id: I32212b4d8180e947fdc958449aebd822f50e97fd"},{"line_number":18,"context_line":"Closes-Bug: 1765942"},{"line_number":19,"context_line":"JIRA: NCP-2367"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":4,"id":"5f7c97a3_c6bf70ce","line":18,"updated":"2018-05-07 20:15:03.000000000","message":"It would be helpful if you actually included the nova-compute logs in the bug report from when this happened. Is the token timing out while talking to neutron? Is neutron failing in some random way?","commit_id":"6ccd2f527873ba289be0aaeac9367ba811be6366"},{"author":{"_account_id":21813,"name":"Andrey Volkov","email":"m@amadev.ru","username":"avolkov"},"change_message_id":"d4d291b948badeb7e9ea0b71e788d476535a0717","unresolved":false,"context_lines":[{"line_number":13,"context_line":"Delete issued to building server leaves the instance in error state due to"},{"line_number":14,"context_line":"not able to deallocate network error.During auto-scale delete, some of"},{"line_number":15,"context_line":"the machines go into ERROR state rather than being deleted.This happens"},{"line_number":16,"context_line":"once in a while during deletion of the instances at scale.Log line for"},{"line_number":17,"context_line":"the failure is \u0027Setting instance vm_state to ERROR\u0027,"},{"line_number":18,"context_line":"Failure in do_terminate_instance."},{"line_number":19,"context_line":""}],"source_content_type":"text/x-gerrit-commit-message","patch_set":6,"id":"5f7c97a3_b024e1b0","line":16,"range":{"start_line":16,"start_character":49,"end_line":16,"end_character":62},"updated":"2018-05-21 15:23:40.000000000","message":"is it your local system artifact?","commit_id":"ff3934cf8eb7f8fdecaa3a2d75332c8f0f785f9c"},{"author":{"_account_id":6062,"name":"jichenjc","email":"jichenjc@cn.ibm.com","username":"jichenjc"},"change_message_id":"bfc639aa752af42f2c2af2c57072b136fe28711d","unresolved":false,"context_lines":[{"line_number":13,"context_line":"Delete issued to building server leaves the instance in error state due to"},{"line_number":14,"context_line":"not able to deallocate network error.During auto-scale delete, some of"},{"line_number":15,"context_line":"the machines go into ERROR state rather than being deleted.This happens"},{"line_number":16,"context_line":"once in a while during deletion of the instances at scale.Log line for"},{"line_number":17,"context_line":"the failure is \u0027Setting instance vm_state to ERROR\u0027,"},{"line_number":18,"context_line":"Failure in do_terminate_instance."},{"line_number":19,"context_line":""}],"source_content_type":"text/x-gerrit-commit-message","patch_set":6,"id":"5f7c97a3_1acb6ba5","line":16,"range":{"start_line":16,"start_character":49,"end_line":16,"end_character":62},"in_reply_to":"5f7c97a3_b024e1b0","updated":"2018-05-22 03:23:59.000000000","message":"I guess it\u0027s \nat scale. Log line for ..\n\nwe do have a log like following in source code\nSetting instance vm_state to ERROR","commit_id":"ff3934cf8eb7f8fdecaa3a2d75332c8f0f785f9c"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"4284233e839edb422d142eace3a3be0b5be40e93","unresolved":false,"context_lines":[{"line_number":3,"context_line":"AuthorDate: 2018-05-08 23:19:27 +0530"},{"line_number":4,"context_line":"Commit:     MultipleCrashes \u003cinimitableharish@gmail.com\u003e"},{"line_number":5,"context_line":"CommitDate: 2018-06-04 19:39:29 +0530"},{"line_number":6,"context_line":""},{"line_number":7,"context_line":"Retry decorator fix for autoscale delete"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"deallocate-network-retry"},{"line_number":10,"context_line":""}],"source_content_type":"text/x-gerrit-commit-message","patch_set":11,"id":"5f7c97a3_c9821c53","line":7,"range":{"start_line":6,"start_character":0,"end_line":7,"end_character":40},"updated":"2018-06-06 15:17:07.000000000","message":"I have no idea what \"autoscale delete\" is.","commit_id":"83ccb5525e1aefcc3ce96ad40bbb114d560c3690"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"4284233e839edb422d142eace3a3be0b5be40e93","unresolved":false,"context_lines":[{"line_number":6,"context_line":""},{"line_number":7,"context_line":"Retry decorator fix for autoscale delete"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"deallocate-network-retry"},{"line_number":10,"context_line":""},{"line_number":11,"context_line":"retry fix for VM going to ERROR state during auto-scale delete instance"},{"line_number":12,"context_line":""}],"source_content_type":"text/x-gerrit-commit-message","patch_set":11,"id":"5f7c97a3_897c2473","line":9,"range":{"start_line":9,"start_character":0,"end_line":9,"end_character":24},"updated":"2018-06-06 15:17:07.000000000","message":"Is this supposed to be a reference to a blueprint?","commit_id":"83ccb5525e1aefcc3ce96ad40bbb114d560c3690"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"4284233e839edb422d142eace3a3be0b5be40e93","unresolved":false,"context_lines":[{"line_number":13,"context_line":"Delete issued to building server leaves the instance in error state due to"},{"line_number":14,"context_line":"not able to deallocate network error.During auto-scale delete, some of"},{"line_number":15,"context_line":"the machines go into ERROR state rather than being deleted.This happens"},{"line_number":16,"context_line":"once in a while during deletion of the instances at scale.Log line for"},{"line_number":17,"context_line":"the failure is \u0027Setting instance vm_state to ERROR\u0027,"},{"line_number":18,"context_line":"Failure in do_terminate_instance."},{"line_number":19,"context_line":""}],"source_content_type":"text/x-gerrit-commit-message","patch_set":11,"id":"5f7c97a3_49960c97","line":16,"range":{"start_line":16,"start_character":16,"end_line":16,"end_character":57},"updated":"2018-06-06 15:17:07.000000000","message":"I\u0027m not sure what this means. Did you mean \"when attempting to delete many instances at once\"?","commit_id":"83ccb5525e1aefcc3ce96ad40bbb114d560c3690"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"4284233e839edb422d142eace3a3be0b5be40e93","unresolved":false,"context_lines":[{"line_number":15,"context_line":"the machines go into ERROR state rather than being deleted.This happens"},{"line_number":16,"context_line":"once in a while during deletion of the instances at scale.Log line for"},{"line_number":17,"context_line":"the failure is \u0027Setting instance vm_state to ERROR\u0027,"},{"line_number":18,"context_line":"Failure in do_terminate_instance."},{"line_number":19,"context_line":""},{"line_number":20,"context_line":"Change-Id: I32212b4d8180e947fdc958449aebd822f50e97fd"},{"line_number":21,"context_line":"Closes-Bug: #1765942"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":11,"id":"5f7c97a3_09901495","line":18,"updated":"2018-06-06 15:17:07.000000000","message":"This commit message doesn\u0027t describe what is being fixed, why the patch is doing what it\u0027s doing nor how the fix is implemented. This commit message is merely re-stating the bug report, which isn\u0027t very helpful.","commit_id":"83ccb5525e1aefcc3ce96ad40bbb114d560c3690"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"0ae97bdb1f12a49a9b2d07293e3532f0810b9023","unresolved":false,"context_lines":[{"line_number":4,"context_line":"Commit:     MultipleCrashes \u003cinimitableharish@gmail.com\u003e"},{"line_number":5,"context_line":"CommitDate: 2018-07-04 16:55:41 +0530"},{"line_number":6,"context_line":""},{"line_number":7,"context_line":"Retry decorator fix for autoscale delete"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"During bulk delete, some of the machines go into ERROR state"},{"line_number":10,"context_line":"rather than being deleted.This happens once in a while during"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":13,"id":"5f7c97a3_dd459f9c","line":7,"range":{"start_line":7,"start_character":24,"end_line":7,"end_character":40},"updated":"2018-07-05 14:18:25.000000000","message":"there is nothing in Nova called \"autoscale delete\". Would you mind changing this commit summary to:\n\n add retry decorator to _deallocate_network()","commit_id":"80865cfd617234f40eb49938348e40f03afb9730"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"0ae97bdb1f12a49a9b2d07293e3532f0810b9023","unresolved":false,"context_lines":[{"line_number":7,"context_line":"Retry decorator fix for autoscale delete"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"During bulk delete, some of the machines go into ERROR state"},{"line_number":10,"context_line":"rather than being deleted.This happens once in a while during"},{"line_number":11,"context_line":"deletion huge number of machines concurrently.The failure occurs"},{"line_number":12,"context_line":"during deallocation of network.At a later point in time the VM in"},{"line_number":13,"context_line":"error state gets deleted freeing up the resources when tried"},{"line_number":14,"context_line":"manually.This fix proposes for retires for certain number of"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":13,"id":"5f7c97a3_9dc96726","line":11,"range":{"start_line":10,"start_character":55,"end_line":11,"end_character":8},"updated":"2018-07-05 14:18:25.000000000","message":"s/while deletion/when deleting a/","commit_id":"80865cfd617234f40eb49938348e40f03afb9730"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"0ae97bdb1f12a49a9b2d07293e3532f0810b9023","unresolved":false,"context_lines":[{"line_number":10,"context_line":"rather than being deleted.This happens once in a while during"},{"line_number":11,"context_line":"deletion huge number of machines concurrently.The failure occurs"},{"line_number":12,"context_line":"during deallocation of network.At a later point in time the VM in"},{"line_number":13,"context_line":"error state gets deleted freeing up the resources when tried"},{"line_number":14,"context_line":"manually.This fix proposes for retires for certain number of"},{"line_number":15,"context_line":"time with variable delay to allow proper deallocation of network."},{"line_number":16,"context_line":""},{"line_number":17,"context_line":"Change-Id: I32212b4d8180e947fdc958449aebd822f50e97fd"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":13,"id":"5f7c97a3_9df2c76e","line":14,"range":{"start_line":13,"start_character":49,"end_line":14,"end_character":9},"updated":"2018-07-05 14:18:25.000000000","message":"could you elaborate on what you mean by \"when tried manually\"? Are you saying that the ERROR state can get \"cleared\" if the user manually deletes the instance later on?","commit_id":"80865cfd617234f40eb49938348e40f03afb9730"}],"nova/compute/manager.py":[{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"36ba915ae0a02d11b72603c1c7e4719504879d17","unresolved":false,"context_lines":[{"line_number":2555,"context_line":"            retry_count \u003d 0"},{"line_number":2556,"context_line":"            max_retries \u003d 3"},{"line_number":2557,"context_line":"            delay_time \u003d 30  # starting delay 30 sec"},{"line_number":2558,"context_line":"            while True:"},{"line_number":2559,"context_line":"                try:"},{"line_number":2560,"context_line":"                    self._delete_instance(context, instance, bdms)"},{"line_number":2561,"context_line":"                except exception.InstanceNotFound:"}],"source_content_type":"text/x-python","patch_set":1,"id":"9f6a8fd7_6615e49b","line":2558,"updated":"2018-04-21 13:56:28.000000000","message":"This is quite a bit of of behavior change without knowing the root cause of the network de-allocation failure. Maybe if we know the root cause we could have a more targeted fix for this issue.","commit_id":"592454b2b19e8cdebd5862ba0c92bfa227f15455"},{"author":{"_account_id":27482,"name":"MultipleCrashes","email":"inimitableharish@gmail.com","username":"MultipleCrashes"},"change_message_id":"e1cf1b60306a54996b68c17e11d52e2de3f3fb0a","unresolved":false,"context_lines":[{"line_number":2555,"context_line":"            retry_count \u003d 0"},{"line_number":2556,"context_line":"            max_retries \u003d 3"},{"line_number":2557,"context_line":"            delay_time \u003d 30  # starting delay 30 sec"},{"line_number":2558,"context_line":"            while True:"},{"line_number":2559,"context_line":"                try:"},{"line_number":2560,"context_line":"                    self._delete_instance(context, instance, bdms)"},{"line_number":2561,"context_line":"                except exception.InstanceNotFound:"}],"source_content_type":"text/x-python","patch_set":1,"id":"9f6a8fd7_bad5091e","line":2558,"in_reply_to":"9f6a8fd7_6615e49b","updated":"2018-05-07 15:26:29.000000000","message":"Nailing down to the exact root cause for the issue is not yet known, possible causes could be a load on the server, memory/CPU utilization or race conditions.Traceback or other tools and methods are not of much help in pinpointing.","commit_id":"592454b2b19e8cdebd5862ba0c92bfa227f15455"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"36ba915ae0a02d11b72603c1c7e4719504879d17","unresolved":false,"context_lines":[{"line_number":2562,"context_line":"                    LOG.info(\"Instance disappeared during terminate\","},{"line_number":2563,"context_line":"                             instance\u003dinstance)"},{"line_number":2564,"context_line":"                except Exception as e:"},{"line_number":2565,"context_line":"                    LOG.info(\u0027CPU usage %s and memory usage %s during \u0027"},{"line_number":2566,"context_line":"                             \u0027exception \u0027 % (psutil.cpu_percent(),"},{"line_number":2567,"context_line":"                              psutil.virtual_memory()))"},{"line_number":2568,"context_line":"                    retry_count \u003d retry_count + 1"},{"line_number":2569,"context_line":"                    LOG.info(\u0027Retrying _delete_instance \u0027"},{"line_number":2570,"context_line":"                             \u0027retrying count %s \u0027, retry_count)"},{"line_number":2571,"context_line":"                    if retry_count \u003d\u003d max_retries: # max retries done, raise"}],"source_content_type":"text/x-python","patch_set":1,"id":"9f6a8fd7_a6453cb6","line":2568,"range":{"start_line":2565,"start_character":1,"end_line":2568,"end_character":1},"updated":"2018-04-21 13:56:28.000000000","message":"I don\u0027t think we need to log cpu and memory usage while handling the exception. Is there any particular reason to log this data?","commit_id":"592454b2b19e8cdebd5862ba0c92bfa227f15455"},{"author":{"_account_id":27482,"name":"MultipleCrashes","email":"inimitableharish@gmail.com","username":"MultipleCrashes"},"change_message_id":"e1cf1b60306a54996b68c17e11d52e2de3f3fb0a","unresolved":false,"context_lines":[{"line_number":2562,"context_line":"                    LOG.info(\"Instance disappeared during terminate\","},{"line_number":2563,"context_line":"                             instance\u003dinstance)"},{"line_number":2564,"context_line":"                except Exception as e:"},{"line_number":2565,"context_line":"                    LOG.info(\u0027CPU usage %s and memory usage %s during \u0027"},{"line_number":2566,"context_line":"                             \u0027exception \u0027 % (psutil.cpu_percent(),"},{"line_number":2567,"context_line":"                              psutil.virtual_memory()))"},{"line_number":2568,"context_line":"                    retry_count \u003d retry_count + 1"},{"line_number":2569,"context_line":"                    LOG.info(\u0027Retrying _delete_instance \u0027"},{"line_number":2570,"context_line":"                             \u0027retrying count %s \u0027, retry_count)"},{"line_number":2571,"context_line":"                    if retry_count \u003d\u003d max_retries: # max retries done, raise"}],"source_content_type":"text/x-python","patch_set":1,"id":"9f6a8fd7_ba8c2936","line":2568,"range":{"start_line":2565,"start_character":1,"end_line":2568,"end_character":1},"in_reply_to":"9f6a8fd7_a6453cb6","updated":"2018-05-07 15:26:29.000000000","message":"Agree, this was mostly to track if system resources are the bottleneck.","commit_id":"592454b2b19e8cdebd5862ba0c92bfa227f15455"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"4f8b9069fa80ab94ca937c5239a29def3fa9ee38","unresolved":false,"context_lines":[{"line_number":2562,"context_line":"                             instance\u003dinstance)"},{"line_number":2563,"context_line":"                except Exception:"},{"line_number":2564,"context_line":"                    retry_count \u003d retry_count + 1"},{"line_number":2565,"context_line":"                    LOG.info(\u0027Retrying _delete_instance after failure \u0027"},{"line_number":2566,"context_line":"                             \u0027retrying count %s \u0027, retry_count)"},{"line_number":2567,"context_line":"                    if retry_count \u003d\u003d max_retries:  # max retries done, raise"},{"line_number":2568,"context_line":"                        # As we\u0027re trying to delete always go to Error if"},{"line_number":2569,"context_line":"                        # something goes wrong that _delete_instance"}],"source_content_type":"text/x-python","patch_set":2,"id":"9f6a8fd7_d779a6c4","line":2566,"range":{"start_line":2565,"start_character":20,"end_line":2566,"end_character":63},"updated":"2018-04-25 22:46:54.000000000","message":"Don\u0027t log this unless you\u0027re retrying (i.e. after the max_retries check).","commit_id":"e19de7dcb5276d4df0eec546e25a766e4705cee2"},{"author":{"_account_id":27482,"name":"MultipleCrashes","email":"inimitableharish@gmail.com","username":"MultipleCrashes"},"change_message_id":"e1cf1b60306a54996b68c17e11d52e2de3f3fb0a","unresolved":false,"context_lines":[{"line_number":2562,"context_line":"                             instance\u003dinstance)"},{"line_number":2563,"context_line":"                except Exception:"},{"line_number":2564,"context_line":"                    retry_count \u003d retry_count + 1"},{"line_number":2565,"context_line":"                    LOG.info(\u0027Retrying _delete_instance after failure \u0027"},{"line_number":2566,"context_line":"                             \u0027retrying count %s \u0027, retry_count)"},{"line_number":2567,"context_line":"                    if retry_count \u003d\u003d max_retries:  # max retries done, raise"},{"line_number":2568,"context_line":"                        # As we\u0027re trying to delete always go to Error if"},{"line_number":2569,"context_line":"                        # something goes wrong that _delete_instance"}],"source_content_type":"text/x-python","patch_set":2,"id":"5f7c97a3_dfa41c0b","line":2566,"range":{"start_line":2565,"start_character":20,"end_line":2566,"end_character":63},"in_reply_to":"9f6a8fd7_d779a6c4","updated":"2018-05-07 15:26:29.000000000","message":"Done","commit_id":"e19de7dcb5276d4df0eec546e25a766e4705cee2"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"4f8b9069fa80ab94ca937c5239a29def3fa9ee38","unresolved":false,"context_lines":[{"line_number":2568,"context_line":"                        # As we\u0027re trying to delete always go to Error if"},{"line_number":2569,"context_line":"                        # something goes wrong that _delete_instance"},{"line_number":2570,"context_line":"                        # can\u0027t handle."},{"line_number":2571,"context_line":"                        with excutils.save_and_reraise_exception():"},{"line_number":2572,"context_line":"                            LOG.exception(\u0027Setting instance vm_state to ERROR\u0027,"},{"line_number":2573,"context_line":"                                          instance\u003dinstance)"},{"line_number":2574,"context_line":"                            self._set_instance_obj_error_state(context,"}],"source_content_type":"text/x-python","patch_set":2,"id":"9f6a8fd7_b77e72ac","line":2571,"range":{"start_line":2571,"start_character":24,"end_line":2571,"end_character":67},"updated":"2018-04-25 22:46:54.000000000","message":"This needs to be the first thing after the `except` line.  Which means you\u0027ll have to have it reraise conditionally.\n\n except Exception:\n     with save_and_reraise_exception(reraise\u003dFalse) as sare:\n         .... stuff that doesn\u0027t reraise\n         if whatever:\n             sare.reraise \u003d True","commit_id":"e19de7dcb5276d4df0eec546e25a766e4705cee2"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"26729ab82dc358ff2fff584f2c3aa823fbc1efcf","unresolved":false,"context_lines":[{"line_number":2561,"context_line":"                    LOG.info(\"Instance disappeared during terminate\","},{"line_number":2562,"context_line":"                             instance\u003dinstance)"},{"line_number":2563,"context_line":"                except keystone_exception.connection.ConnectFailure:"},{"line_number":2564,"context_line":"                    if retry_count \u003c max_retries:  # max retries not done"},{"line_number":2565,"context_line":"                        LOG.info(\u0027Retrying _delete_instance after failure \u0027"},{"line_number":2566,"context_line":"                                 \u0027retrying count %s \u0027, retry_count)"},{"line_number":2567,"context_line":"                        time.sleep(delay_time)"}],"source_content_type":"text/x-python","patch_set":3,"id":"5f7c97a3_ff1238b1","line":2564,"range":{"start_line":2564,"start_character":20,"end_line":2564,"end_character":49},"updated":"2018-05-07 15:40:00.000000000","message":"If you run out of retries, this will cause you to fall through to the `break` on L2579, whereas I think what you want to do is log and reraise like L2574-8.","commit_id":"a83395dbb761f8b41b347fc3a3322367a05fea62"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"9c0d6e316909842dd884f575c872768e4616a1b5","unresolved":false,"context_lines":[{"line_number":2554,"context_line":"            retry_count \u003d 1"},{"line_number":2555,"context_line":"            max_retries \u003d 3"},{"line_number":2556,"context_line":"            delay_time \u003d 30  # starting delay 30 sec"},{"line_number":2557,"context_line":"            while True:"},{"line_number":2558,"context_line":"                try:"},{"line_number":2559,"context_line":"                    self._delete_instance(context, instance, bdms)"},{"line_number":2560,"context_line":"                except exception.InstanceNotFound:"}],"source_content_type":"text/x-python","patch_set":4,"id":"5f7c97a3_66abfc88","line":2557,"updated":"2018-05-07 20:15:03.000000000","message":"If we did retries in here, I\u0027d rather seem something re-used like this:\n\nhttps://github.com/openstack/nova/blob/b60c3fc394a0b6851b2910600bf9fcf5161260fa/nova/scheduler/client/report.py#L95\n\nOr something from the loopingcall module:\n\nhttps://github.com/openstack/oslo.service/blob/master/oslo_service/loopingcall.py#L401","commit_id":"6ccd2f527873ba289be0aaeac9367ba811be6366"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"3ac439de7599eda5f48b88edad08a36ce820ec1a","unresolved":false,"context_lines":[{"line_number":2571,"context_line":"                            retry_count \u003d retry_count + 1"},{"line_number":2572,"context_line":"                            continue"},{"line_number":2573,"context_line":"                        else:"},{"line_number":2574,"context_line":"                            LOG.exception(\u00273 Retryies done\u0027)"},{"line_number":2575,"context_line":"                            sare.reraise \u003d True"},{"line_number":2576,"context_line":"                except Exception:"},{"line_number":2577,"context_line":"                    # As we\u0027re trying to delete always go to Error if"}],"source_content_type":"text/x-python","patch_set":4,"id":"5f7c97a3_66845c0d","line":2574,"range":{"start_line":2574,"start_character":49,"end_line":2574,"end_character":50},"updated":"2018-05-07 20:16:14.000000000","message":"strike.\n\nBut this should be doing the same thing as the `except Exception` clause below: namely setting the vm_state to ERROR.  To distinguish the two cases, you may want to use a slightly different log message.  Suggest:\n\n LOG.exception(\u0027Setting instance vm_state to ERROR after %d connection failures\u0027, max_retries, instance\u003dinstance)\n self._set_instance_obj_error_state(context, instance)\n sare.reraise \u003d True","commit_id":"6ccd2f527873ba289be0aaeac9367ba811be6366"},{"author":{"_account_id":12531,"name":"alexander medvedev","email":"xroot@yahoo.com","username":"xroot"},"change_message_id":"84f62a61bc24e9733e8e714e8ec69a964456ceb7","unresolved":false,"context_lines":[{"line_number":2572,"context_line":"                            continue"},{"line_number":2573,"context_line":"                        else:"},{"line_number":2574,"context_line":"                            LOG.exception(\u00273 Retryies done\u0027)"},{"line_number":2575,"context_line":"                            sare.reraise \u003d True"},{"line_number":2576,"context_line":"                except Exception:"},{"line_number":2577,"context_line":"                    # As we\u0027re trying to delete always go to Error if"},{"line_number":2578,"context_line":"                    # something goes wrong that _delete_instance can\u0027t handle."}],"source_content_type":"text/x-python","patch_set":4,"id":"5f7c97a3_693748e5","line":2575,"updated":"2018-05-08 03:27:00.000000000","message":"sare.reraise must be a typo (?) I still think neutronclient is better suited to do the retries, but this is for nova people to decide : )","commit_id":"6ccd2f527873ba289be0aaeac9367ba811be6366"},{"author":{"_account_id":6062,"name":"jichenjc","email":"jichenjc@cn.ibm.com","username":"jichenjc"},"change_message_id":"bfc639aa752af42f2c2af2c57072b136fe28711d","unresolved":false,"context_lines":[{"line_number":2294,"context_line":"                            inc_sleep_time\u003d10,"},{"line_number":2295,"context_line":"                            max_sleep_time\u003d30,"},{"line_number":2296,"context_line":"                            exceptions\u003d("},{"line_number":2297,"context_line":"                                keystone_exception.connection.ConnectFailure,))"},{"line_number":2298,"context_line":"    def _try_deallocate_network(self, context, instance,"},{"line_number":2299,"context_line":"                                requested_networks\u003dNone):"},{"line_number":2300,"context_line":"        try:"}],"source_content_type":"text/x-python","patch_set":6,"id":"5f7c97a3_fabf8ff9","line":2297,"range":{"start_line":2297,"start_character":32,"end_line":2297,"end_character":76},"updated":"2018-05-22 03:23:59.000000000","message":"why retry on this exception?","commit_id":"ff3934cf8eb7f8fdecaa3a2d75332c8f0f785f9c"},{"author":{"_account_id":6062,"name":"jichenjc","email":"jichenjc@cn.ibm.com","username":"jichenjc"},"change_message_id":"305d5aee9186bc7933a23708c88fcea0231506d3","unresolved":false,"context_lines":[{"line_number":2294,"context_line":"                            inc_sleep_time\u003d10,"},{"line_number":2295,"context_line":"                            max_sleep_time\u003d30,"},{"line_number":2296,"context_line":"                            exceptions\u003d("},{"line_number":2297,"context_line":"                                keystone_exception.connection.ConnectFailure,))"},{"line_number":2298,"context_line":"    def _try_deallocate_network(self, context, instance,"},{"line_number":2299,"context_line":"                                requested_networks\u003dNone):"},{"line_number":2300,"context_line":"        try:"}],"source_content_type":"text/x-python","patch_set":9,"id":"5f7c97a3_2aa122aa","line":2297,"range":{"start_line":2297,"start_character":32,"end_line":2297,"end_character":77},"updated":"2018-05-23 08:51:13.000000000","message":"again, curious on this as mentioned in PS6\nno info about it in both bug and commit msg","commit_id":"0e026e7da2268dde9f1ec4af042368a53e07d287"},{"author":{"_account_id":27482,"name":"MultipleCrashes","email":"inimitableharish@gmail.com","username":"MultipleCrashes"},"change_message_id":"84ed385c09e91482c1d46d52a1af1c63beca868f","unresolved":false,"context_lines":[{"line_number":2294,"context_line":"                            inc_sleep_time\u003d10,"},{"line_number":2295,"context_line":"                            max_sleep_time\u003d30,"},{"line_number":2296,"context_line":"                            exceptions\u003d("},{"line_number":2297,"context_line":"                                keystone_exception.connection.ConnectFailure,))"},{"line_number":2298,"context_line":"    def _try_deallocate_network(self, context, instance,"},{"line_number":2299,"context_line":"                                requested_networks\u003dNone):"},{"line_number":2300,"context_line":"        try:"}],"source_content_type":"text/x-python","patch_set":9,"id":"5f7c97a3_8ac0762c","line":2297,"range":{"start_line":2297,"start_character":32,"end_line":2297,"end_character":77},"in_reply_to":"5f7c97a3_2aa122aa","updated":"2018-05-23 10:35:04.000000000","message":"The comment section of the bug list has exact traceback.Please check at this link https://bugs.launchpad.net/nova/+bug/1765942. Also the closes bug section of the commit message has link to the issue.","commit_id":"0e026e7da2268dde9f1ec4af042368a53e07d287"},{"author":{"_account_id":6062,"name":"jichenjc","email":"jichenjc@cn.ibm.com","username":"jichenjc"},"change_message_id":"8cb64ae287ab95ca9e6cd8700e315e12c415529b","unresolved":false,"context_lines":[{"line_number":2294,"context_line":"                            inc_sleep_time\u003d10,"},{"line_number":2295,"context_line":"                            max_sleep_time\u003d30,"},{"line_number":2296,"context_line":"                            exceptions\u003d("},{"line_number":2297,"context_line":"                                keystone_exception.connection.ConnectFailure,))"},{"line_number":2298,"context_line":"    def _try_deallocate_network(self, context, instance,"},{"line_number":2299,"context_line":"                                requested_networks\u003dNone):"},{"line_number":2300,"context_line":"        try:"}],"source_content_type":"text/x-python","patch_set":9,"id":"5f7c97a3_e911eb05","line":2297,"range":{"start_line":2297,"start_character":32,"end_line":2297,"end_character":77},"in_reply_to":"5f7c97a3_8ac0762c","updated":"2018-05-24 06:28:12.000000000","message":"ok, saw the  log exception \nbecause it\u0027s a connection issue, will retry really help ? maybe due to network unstable or something?\n\nanyway, thanks for pointing out the bug detailed log\nI didn\u0027t see it at first time","commit_id":"0e026e7da2268dde9f1ec4af042368a53e07d287"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"4284233e839edb422d142eace3a3be0b5be40e93","unresolved":false,"context_lines":[{"line_number":1646,"context_line":"            system_metadata\u003dsystem_metadata,"},{"line_number":1647,"context_line":"            extra_usage_info\u003dextra_usage_info, fault\u003dfault)"},{"line_number":1648,"context_line":""},{"line_number":1649,"context_line":"    def _deallocate_network(self, context, instance,"},{"line_number":1650,"context_line":"                            requested_networks\u003dNone):"},{"line_number":1651,"context_line":"        # If we were told not to allocate networks let\u0027s save ourselves"},{"line_number":1652,"context_line":"        # the trouble of calling the network API."},{"line_number":1653,"context_line":"        if requested_networks and requested_networks.no_allocate:"}],"source_content_type":"text/x-python","patch_set":11,"id":"5f7c97a3_c9353caa","line":1650,"range":{"start_line":1649,"start_character":4,"end_line":1650,"end_character":53},"updated":"2018-06-06 15:17:07.000000000","message":"This is the method I would decorate with a retry loop.","commit_id":"83ccb5525e1aefcc3ce96ad40bbb114d560c3690"},{"author":{"_account_id":27482,"name":"MultipleCrashes","email":"inimitableharish@gmail.com","username":"MultipleCrashes"},"change_message_id":"3dbde0945144c7b8b603f4e540ed2c296c0bcec8","unresolved":false,"context_lines":[{"line_number":1646,"context_line":"            system_metadata\u003dsystem_metadata,"},{"line_number":1647,"context_line":"            extra_usage_info\u003dextra_usage_info, fault\u003dfault)"},{"line_number":1648,"context_line":""},{"line_number":1649,"context_line":"    def _deallocate_network(self, context, instance,"},{"line_number":1650,"context_line":"                            requested_networks\u003dNone):"},{"line_number":1651,"context_line":"        # If we were told not to allocate networks let\u0027s save ourselves"},{"line_number":1652,"context_line":"        # the trouble of calling the network API."},{"line_number":1653,"context_line":"        if requested_networks and requested_networks.no_allocate:"}],"source_content_type":"text/x-python","patch_set":11,"id":"5f7c97a3_0816974b","line":1650,"range":{"start_line":1649,"start_character":4,"end_line":1650,"end_character":53},"in_reply_to":"5f7c97a3_c9353caa","updated":"2018-07-03 20:44:10.000000000","message":"Applying decorator to the below function helps to log additional log line, which will make the debugging of the retry easier .Therefore trying to caputre it there.","commit_id":"83ccb5525e1aefcc3ce96ad40bbb114d560c3690"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"4284233e839edb422d142eace3a3be0b5be40e93","unresolved":false,"context_lines":[{"line_number":2294,"context_line":"                            inc_sleep_time\u003d10,"},{"line_number":2295,"context_line":"                            max_sleep_time\u003d30,"},{"line_number":2296,"context_line":"                            exceptions\u003d("},{"line_number":2297,"context_line":"                                keystone_exception.connection.ConnectFailure,))"},{"line_number":2298,"context_line":"    def _try_deallocate_network(self, context, instance,"},{"line_number":2299,"context_line":"                                requested_networks\u003dNone):"},{"line_number":2300,"context_line":"        try:"}],"source_content_type":"text/x-python","patch_set":11,"id":"5f7c97a3_49db6c7c","line":2297,"range":{"start_line":2297,"start_character":32,"end_line":2297,"end_character":79},"updated":"2018-06-06 15:17:07.000000000","message":"I would have at least expected that you mention in the commit message or bug report that the cause of the failure that you are seeing for multiple concurrent instance deletes is a failure to connect to Keystone. If that is really the case, it seems to me that the most appropriate place to put a retry would actually be on the neutron client\u0027s interaction with Keystone, not in Nova\u0027s compute manager...","commit_id":"83ccb5525e1aefcc3ce96ad40bbb114d560c3690"},{"author":{"_account_id":27482,"name":"MultipleCrashes","email":"inimitableharish@gmail.com","username":"MultipleCrashes"},"change_message_id":"e5269a15f8bb26790dc065a20c71a7b2b57c9564","unresolved":false,"context_lines":[{"line_number":2294,"context_line":"                            inc_sleep_time\u003d10,"},{"line_number":2295,"context_line":"                            max_sleep_time\u003d30,"},{"line_number":2296,"context_line":"                            exceptions\u003d("},{"line_number":2297,"context_line":"                                keystone_exception.connection.ConnectFailure,))"},{"line_number":2298,"context_line":"    def _try_deallocate_network(self, context, instance,"},{"line_number":2299,"context_line":"                                requested_networks\u003dNone):"},{"line_number":2300,"context_line":"        try:"}],"source_content_type":"text/x-python","patch_set":11,"id":"5f7c97a3_44bb3ce4","line":2297,"range":{"start_line":2297,"start_character":32,"end_line":2297,"end_character":79},"in_reply_to":"5f7c97a3_49db6c7c","updated":"2018-07-03 10:39:44.000000000","message":"Yeah, commit message needs to be changed , will modifiy it now.Making the change at neutronclient interaction we are not getting the robustness of retries with variable delay, also it seems difficult to retry for a particular exception (in this case ConnectFailure ) we will need to have much more code change and our fresh implementaion of retry to allow us do that.The unit tests are also not straight forward and appears to be little covoluted.Tried out this in one of the commits earlier :https://github.com/MultipleCrashes/nova/commit/9aa8af7cdc38167e00d413d0712293d1369c7d4b\n\nCommit message is updated to reflect the changes.","commit_id":"83ccb5525e1aefcc3ce96ad40bbb114d560c3690"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"4284233e839edb422d142eace3a3be0b5be40e93","unresolved":false,"context_lines":[{"line_number":2299,"context_line":"                                requested_networks\u003dNone):"},{"line_number":2300,"context_line":"        try:"},{"line_number":2301,"context_line":"            # tear down allocated network structure"},{"line_number":2302,"context_line":"            self._deallocate_network(context, instance, requested_networks)"},{"line_number":2303,"context_line":"        except Exception as ex:"},{"line_number":2304,"context_line":"            with excutils.save_and_reraise_exception():"},{"line_number":2305,"context_line":"                LOG.error(\u0027Failed to deallocate network for instance. \u0027"}],"source_content_type":"text/x-python","patch_set":11,"id":"5f7c97a3_89cae49e","line":2302,"range":{"start_line":2302,"start_character":12,"end_line":2302,"end_character":75},"updated":"2018-06-06 15:17:07.000000000","message":"why not decorate this method with the retry loop?","commit_id":"83ccb5525e1aefcc3ce96ad40bbb114d560c3690"},{"author":{"_account_id":27482,"name":"MultipleCrashes","email":"inimitableharish@gmail.com","username":"MultipleCrashes"},"change_message_id":"e5269a15f8bb26790dc065a20c71a7b2b57c9564","unresolved":false,"context_lines":[{"line_number":2299,"context_line":"                                requested_networks\u003dNone):"},{"line_number":2300,"context_line":"        try:"},{"line_number":2301,"context_line":"            # tear down allocated network structure"},{"line_number":2302,"context_line":"            self._deallocate_network(context, instance, requested_networks)"},{"line_number":2303,"context_line":"        except Exception as ex:"},{"line_number":2304,"context_line":"            with excutils.save_and_reraise_exception():"},{"line_number":2305,"context_line":"                LOG.error(\u0027Failed to deallocate network for instance. \u0027"}],"source_content_type":"text/x-python","patch_set":11,"id":"5f7c97a3_e1591a5b","line":2302,"range":{"start_line":2302,"start_character":12,"end_line":2302,"end_character":75},"in_reply_to":"5f7c97a3_89cae49e","updated":"2018-07-03 10:39:44.000000000","message":"It\u0027s a good idea to decorate _deallocate_network to have the retry decorator closest to the function throwing exception.Looking at the traceback we see that the log line which gets entry in the logs file is \u0027Failed to deallocate..\u0027 present in _try_deallocate_network.This would help during debugging to find which section is being retried.The only thing which our _try_deallocate_network does is calling the _deallocate_network and save_and_reraise_exception.Moving the decorator to the called function, if needed, should be easier as well - given that we are willing to miss the log line.","commit_id":"83ccb5525e1aefcc3ce96ad40bbb114d560c3690"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"0ae97bdb1f12a49a9b2d07293e3532f0810b9023","unresolved":false,"context_lines":[{"line_number":2283,"context_line":"            # exception will be raised by instance.save()"},{"line_number":2284,"context_line":"            pass"},{"line_number":2285,"context_line":""},{"line_number":2286,"context_line":"    @loopingcall.RetryDecorator(max_retry_count\u003d3,"},{"line_number":2287,"context_line":"                            inc_sleep_time\u003d10,"},{"line_number":2288,"context_line":"                            max_sleep_time\u003d30,"},{"line_number":2289,"context_line":"                            exceptions\u003d("},{"line_number":2290,"context_line":"                                keystone_exception.connection.ConnectFailure,))"},{"line_number":2291,"context_line":"    def _try_deallocate_network(self, context, instance,"},{"line_number":2292,"context_line":"                                requested_networks\u003dNone):"},{"line_number":2293,"context_line":"        try:"}],"source_content_type":"text/x-python","patch_set":13,"id":"5f7c97a3_788c992d","line":2290,"range":{"start_line":2286,"start_character":4,"end_line":2290,"end_character":79},"updated":"2018-07-05 14:18:25.000000000","message":"The problem with decorating this method instead of the _deallocate_network() method is that the LOG.error() message on lines 2298-2299 will be output, multiple times for each time the retry is done. This will be confusing to operators, because there is no indication that a retry operation will be attempted and that there\u0027s nothing they should do to remedy the situation.\n\nThis is why I recommended putting the decorator around _deallocate_network().","commit_id":"80865cfd617234f40eb49938348e40f03afb9730"},{"author":{"_account_id":27482,"name":"MultipleCrashes","email":"inimitableharish@gmail.com","username":"MultipleCrashes"},"change_message_id":"03ebffc04112c3d98f5eea90afcc4f3e4062ce14","unresolved":false,"context_lines":[{"line_number":2283,"context_line":"            # exception will be raised by instance.save()"},{"line_number":2284,"context_line":"            pass"},{"line_number":2285,"context_line":""},{"line_number":2286,"context_line":"    @loopingcall.RetryDecorator(max_retry_count\u003d3,"},{"line_number":2287,"context_line":"                            inc_sleep_time\u003d10,"},{"line_number":2288,"context_line":"                            max_sleep_time\u003d30,"},{"line_number":2289,"context_line":"                            exceptions\u003d("},{"line_number":2290,"context_line":"                                keystone_exception.connection.ConnectFailure,))"},{"line_number":2291,"context_line":"    def _try_deallocate_network(self, context, instance,"},{"line_number":2292,"context_line":"                                requested_networks\u003dNone):"},{"line_number":2293,"context_line":"        try:"}],"source_content_type":"text/x-python","patch_set":13,"id":"5f7c97a3_2fa422a3","line":2290,"range":{"start_line":2286,"start_character":4,"end_line":2290,"end_character":79},"in_reply_to":"5f7c97a3_788c992d","updated":"2018-07-11 18:04:14.000000000","message":"In the _deallocate_network() method we are not raising, whereas in this function we are raising the particular exception.In case we put the retry to _deallocate_network() we would have to additionally have a try except block in there, tried doing so get it working.Also, if we look at the log it has instance id attached in the log Failed to deallocate log line, which would mean that in case we do a grep on the logs with log line and instance id ..\u0027n\u0027 occurence of this line would mean n failures and n retires. \nSome more input/feedback might help.","commit_id":"80865cfd617234f40eb49938348e40f03afb9730"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"f7e9f02b1fba790db1b304b817cb4489576a2b31","unresolved":false,"context_lines":[{"line_number":2284,"context_line":"            pass"},{"line_number":2285,"context_line":""},{"line_number":2286,"context_line":"    @loopingcall.RetryDecorator(max_retry_count\u003d3,"},{"line_number":2287,"context_line":"                            inc_sleep_time\u003d2,"},{"line_number":2288,"context_line":"                            max_sleep_time\u003d12,"},{"line_number":2289,"context_line":"                            exceptions\u003d("},{"line_number":2290,"context_line":"                                keystone_exception.connection.ConnectFailure,))"},{"line_number":2291,"context_line":"    def _try_deallocate_network(self, context, instance,"},{"line_number":2292,"context_line":"                                requested_networks\u003dNone):"}],"source_content_type":"text/x-python","patch_set":16,"id":"5f7c97a3_c69b7ec5","line":2289,"range":{"start_line":2287,"start_character":28,"end_line":2289,"end_character":40},"updated":"2018-07-30 19:52:08.000000000","message":"nit: fix the indents here","commit_id":"f5056806f8e2dc8642a393443f3939e16e47a877"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"f7e9f02b1fba790db1b304b817cb4489576a2b31","unresolved":false,"context_lines":[{"line_number":2295,"context_line":"            self._deallocate_network(context, instance, requested_networks)"},{"line_number":2296,"context_line":"        except Exception as ex:"},{"line_number":2297,"context_line":"            with excutils.save_and_reraise_exception():"},{"line_number":2298,"context_line":"                LOG.error(\u0027Failed to deallocate network for instance. \u0027"},{"line_number":2299,"context_line":"                          \u0027Error: %s\u0027, ex, instance\u003dinstance)"},{"line_number":2300,"context_line":"                self._set_instance_obj_error_state(context, instance)"},{"line_number":2301,"context_line":""}],"source_content_type":"text/x-python","patch_set":16,"id":"5f7c97a3_063f36b2","line":2298,"updated":"2018-07-30 19:52:08.000000000","message":"To avoid logging this every retry, can\u0027t we move the call to _deallocate_network into an inner method and put the decorator on that? Then if we timeout, the decorator will raise and we\u0027ll just log the exception once. I think that\u0027s what Jay is asking for.","commit_id":"f5056806f8e2dc8642a393443f3939e16e47a877"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"7f404ea57e4c9271ab3337390460d1cffdc3f32b","unresolved":false,"context_lines":[{"line_number":2295,"context_line":"            self._deallocate_network(context, instance, requested_networks)"},{"line_number":2296,"context_line":"        except Exception as ex:"},{"line_number":2297,"context_line":"            with excutils.save_and_reraise_exception():"},{"line_number":2298,"context_line":"                LOG.error(\u0027Failed to deallocate network for instance. \u0027"},{"line_number":2299,"context_line":"                          \u0027Error: %s\u0027, ex, instance\u003dinstance)"},{"line_number":2300,"context_line":"                self._set_instance_obj_error_state(context, instance)"},{"line_number":2301,"context_line":""}],"source_content_type":"text/x-python","patch_set":16,"id":"5f7c97a3_667b8a2a","line":2298,"in_reply_to":"5f7c97a3_063f36b2","updated":"2018-07-30 20:09:13.000000000","message":"e.g.:\n\ndef _try_deallocate_network(...):\n\n  @loopingcall.RetryDecorator(...)\n  def _deallocate_network_with_retries(...):\n      self._deallocate_network\n\n  try:\n     _deallocate_network_with_retries(...)\n  except Exception as ex:\n     with excutils.save_and_reraise_exception():\n        LOG.error(...)\n        self._set_instance_obj_error_state(context, instance)","commit_id":"f5056806f8e2dc8642a393443f3939e16e47a877"}],"nova/tests/unit/compute/test_compute_mgr.py":[{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"36ba915ae0a02d11b72603c1c7e4719504879d17","unresolved":false,"context_lines":[{"line_number":4381,"context_line":"            self.context, vm_state\u003dvm_states.ERROR,"},{"line_number":4382,"context_line":"            task_state\u003dtask_states.DELETING)"},{"line_number":4383,"context_line":"        self.compute.terminate_instance(self.context, instance, [])"},{"line_number":4384,"context_line":"        # No bdsm and reservations"},{"line_number":4385,"context_line":"        #self._delete_instance.assert_called_with(self.context, instance,[], [])"},{"line_number":4386,"context_line":"        self.assertEqual(mock_delete_instance.call_count, 1)"},{"line_number":4387,"context_line":""}],"source_content_type":"text/x-python","patch_set":1,"id":"9f6a8fd7_861a78c9","line":4384,"range":{"start_line":4384,"start_character":13,"end_line":4384,"end_character":17},"updated":"2018-04-21 13:56:28.000000000","message":"do you mean bdm? :)","commit_id":"592454b2b19e8cdebd5862ba0c92bfa227f15455"},{"author":{"_account_id":9708,"name":"Balazs Gibizer","display_name":"gibi","email":"gibizer@gmail.com","username":"gibi"},"change_message_id":"36ba915ae0a02d11b72603c1c7e4719504879d17","unresolved":false,"context_lines":[{"line_number":4383,"context_line":"        self.compute.terminate_instance(self.context, instance, [])"},{"line_number":4384,"context_line":"        # No bdsm and reservations"},{"line_number":4385,"context_line":"        #self._delete_instance.assert_called_with(self.context, instance,[], [])"},{"line_number":4386,"context_line":"        self.assertEqual(mock_delete_instance.call_count, 1)"},{"line_number":4387,"context_line":""},{"line_number":4388,"context_line":"    @mock.patch(\u0027nova.compute.utils.notify_about_instance_action\u0027)"},{"line_number":4389,"context_line":"    @mock.patch.object(nova.compute.manager.ComputeManager,"}],"source_content_type":"text/x-python","patch_set":1,"id":"9f6a8fd7_260b6c7d","line":4386,"updated":"2018-04-21 13:56:28.000000000","message":"if _delete_instance called only once then this is not a retry case.","commit_id":"592454b2b19e8cdebd5862ba0c92bfa227f15455"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"26729ab82dc358ff2fff584f2c3aa823fbc1efcf","unresolved":false,"context_lines":[{"line_number":4384,"context_line":"            keystone_exception.connection.ConnectFailure"},{"line_number":4385,"context_line":"        self.compute.terminate_instance(self.context, instance, [])"},{"line_number":4386,"context_line":"        # Three retries as per logic, with backoff time"},{"line_number":4387,"context_line":"        # hence unit test will take time to execute"},{"line_number":4388,"context_line":"        self.assertEqual(mock_delete_instance.call_count, 3)"},{"line_number":4389,"context_line":""},{"line_number":4390,"context_line":"    @mock.patch(\u0027nova.compute.manager.ComputeManager._delete_instance\u0027)"}],"source_content_type":"text/x-python","patch_set":3,"id":"5f7c97a3_dfd09ccc","line":4387,"range":{"start_line":4387,"start_character":16,"end_line":4387,"end_character":51},"updated":"2018-05-07 15:40:00.000000000","message":"Nope.  Mock sleep.","commit_id":"a83395dbb761f8b41b347fc3a3322367a05fea62"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"26729ab82dc358ff2fff584f2c3aa823fbc1efcf","unresolved":false,"context_lines":[{"line_number":4385,"context_line":"        self.compute.terminate_instance(self.context, instance, [])"},{"line_number":4386,"context_line":"        # Three retries as per logic, with backoff time"},{"line_number":4387,"context_line":"        # hence unit test will take time to execute"},{"line_number":4388,"context_line":"        self.assertEqual(mock_delete_instance.call_count, 3)"},{"line_number":4389,"context_line":""},{"line_number":4390,"context_line":"    @mock.patch(\u0027nova.compute.manager.ComputeManager._delete_instance\u0027)"},{"line_number":4391,"context_line":"    def test_no_retry_for_proper_terminate_instance(self,"}],"source_content_type":"text/x-python","patch_set":3,"id":"5f7c97a3_9f030443","line":4388,"range":{"start_line":4388,"start_character":25,"end_line":4388,"end_character":60},"updated":"2018-05-07 15:40:00.000000000","message":"You need a test for the condition I pointed out: namely what happens when you exceed your retries (as you\u0027ve done here).  The proper behavior is for this to raise in that circumstance.\n\nThen you should have a separate test where the side effect is one or two ConnectFailure and then a success of one variety or another.","commit_id":"a83395dbb761f8b41b347fc3a3322367a05fea62"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"3ac439de7599eda5f48b88edad08a36ce820ec1a","unresolved":false,"context_lines":[{"line_number":4382,"context_line":"            task_state\u003dtask_states.DELETING)"},{"line_number":4383,"context_line":"        mock_delete_instance.side_effect \u003d \\"},{"line_number":4384,"context_line":"            keystone_exception.connection.ConnectFailure"},{"line_number":4385,"context_line":"        try:"},{"line_number":4386,"context_line":"            self.compute.terminate_instance(self.context, instance, [])"},{"line_number":4387,"context_line":"            # Three retries as per logic, with backoff time"},{"line_number":4388,"context_line":"            # hence unit test will take time to execute"}],"source_content_type":"text/x-python","patch_set":4,"id":"5f7c97a3_4679202d","line":4385,"updated":"2018-05-07 20:16:14.000000000","message":"Still need to mock sleep here.","commit_id":"6ccd2f527873ba289be0aaeac9367ba811be6366"},{"author":{"_account_id":14070,"name":"Eric Fried","email":"openstack@fried.cc","username":"efried"},"change_message_id":"3ac439de7599eda5f48b88edad08a36ce820ec1a","unresolved":false,"context_lines":[{"line_number":4386,"context_line":"            self.compute.terminate_instance(self.context, instance, [])"},{"line_number":4387,"context_line":"            # Three retries as per logic, with backoff time"},{"line_number":4388,"context_line":"            # hence unit test will take time to execute"},{"line_number":4389,"context_line":"        except:"},{"line_number":4390,"context_line":"            self.assertEqual(mock_delete_instance.call_count, 3)"},{"line_number":4391,"context_line":""},{"line_number":4392,"context_line":"    @mock.patch(\u0027nova.compute.manager.ComputeManager._delete_instance\u0027)"}],"source_content_type":"text/x-python","patch_set":4,"id":"5f7c97a3_a68fd421","line":4389,"range":{"start_line":4389,"start_character":8,"end_line":4389,"end_character":14},"updated":"2018-05-07 20:16:14.000000000","message":"You should use assertRaises to assert that the expected exception (ConnectFailure) is raised here.  You can still keep your call_count assertion as well.","commit_id":"6ccd2f527873ba289be0aaeac9367ba811be6366"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"4284233e839edb422d142eace3a3be0b5be40e93","unresolved":false,"context_lines":[{"line_number":73,"context_line":"from nova.virt import hardware"},{"line_number":74,"context_line":"from nova.volume import cinder"},{"line_number":75,"context_line":""},{"line_number":76,"context_line":"try:"},{"line_number":77,"context_line":"    # py2.x"},{"line_number":78,"context_line":"    import threading"},{"line_number":79,"context_line":"    x \u003d threading._Event.wait"},{"line_number":80,"context_line":"    patch_func \u003d \u0027threading._Event.wait\u0027"},{"line_number":81,"context_line":"except Exception:"},{"line_number":82,"context_line":"    # py3.x"},{"line_number":83,"context_line":"    patch_func \u003d \u0027threading.Event.wait\u0027"},{"line_number":84,"context_line":""},{"line_number":85,"context_line":"CONF \u003d nova.conf.CONF"},{"line_number":86,"context_line":"fake_host_list \u003d [mock.sentinel.host1]"}],"source_content_type":"text/x-python","patch_set":11,"id":"5f7c97a3_6944281a","line":83,"range":{"start_line":76,"start_character":0,"end_line":83,"end_character":39},"updated":"2018-06-06 15:17:07.000000000","message":"Why was this necessary? Seems odd to require this kind of messing with low-level primitives.","commit_id":"83ccb5525e1aefcc3ce96ad40bbb114d560c3690"},{"author":{"_account_id":27482,"name":"MultipleCrashes","email":"inimitableharish@gmail.com","username":"MultipleCrashes"},"change_message_id":"e5269a15f8bb26790dc065a20c71a7b2b57c9564","unresolved":false,"context_lines":[{"line_number":73,"context_line":"from nova.virt import hardware"},{"line_number":74,"context_line":"from nova.volume import cinder"},{"line_number":75,"context_line":""},{"line_number":76,"context_line":"try:"},{"line_number":77,"context_line":"    # py2.x"},{"line_number":78,"context_line":"    import threading"},{"line_number":79,"context_line":"    x \u003d threading._Event.wait"},{"line_number":80,"context_line":"    patch_func \u003d \u0027threading._Event.wait\u0027"},{"line_number":81,"context_line":"except Exception:"},{"line_number":82,"context_line":"    # py3.x"},{"line_number":83,"context_line":"    patch_func \u003d \u0027threading.Event.wait\u0027"},{"line_number":84,"context_line":""},{"line_number":85,"context_line":"CONF \u003d nova.conf.CONF"},{"line_number":86,"context_line":"fake_host_list \u003d [mock.sentinel.host1]"}],"source_content_type":"text/x-python","patch_set":11,"id":"5f7c97a3_0310d1a7","line":83,"range":{"start_line":76,"start_character":0,"end_line":83,"end_character":39},"in_reply_to":"5f7c97a3_6944281a","updated":"2018-07-03 10:39:44.000000000","message":"Apparently the threading._Event of python2 has changed to threading.Event in python3.Since we need to mock the sleep time, which internally in this case uses threading._Event.wait , so in order to allow for faking the sleep time and not actually waiting for the unit test we are using this.Wonder, If there is an alternative way , since we would not want the unit tests to wait for the retry along with the delay.\n\n\nTried out a couple of ways on how the existing code mocks sleep to get rid of the sleep time.Couldn\u0027t find any better working alternative way of avoiding the actual wait time.","commit_id":"83ccb5525e1aefcc3ce96ad40bbb114d560c3690"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"0ae97bdb1f12a49a9b2d07293e3532f0810b9023","unresolved":false,"context_lines":[{"line_number":74,"context_line":"from nova.virt import hardware"},{"line_number":75,"context_line":"from nova.volume import cinder"},{"line_number":76,"context_line":""},{"line_number":77,"context_line":"try:"},{"line_number":78,"context_line":"    # py2.x"},{"line_number":79,"context_line":"    import threading"},{"line_number":80,"context_line":"    x \u003d threading._Event.wait"},{"line_number":81,"context_line":"    patch_func \u003d \u0027threading._Event.wait\u0027"},{"line_number":82,"context_line":"except Exception:"},{"line_number":83,"context_line":"    # py3.x"},{"line_number":84,"context_line":"    patch_func \u003d \u0027threading.Event.wait\u0027"},{"line_number":85,"context_line":""},{"line_number":86,"context_line":"CONF \u003d nova.conf.CONF"},{"line_number":87,"context_line":"fake_host_list \u003d [mock.sentinel.host1]"}],"source_content_type":"text/x-python","patch_set":13,"id":"5f7c97a3_58b0bddc","line":84,"range":{"start_line":77,"start_character":0,"end_line":84,"end_character":39},"updated":"2018-07-05 14:18:25.000000000","message":"I still don\u0027t think it\u0027s necessary to include this or even mock out anything to do with the threading.Event object.","commit_id":"80865cfd617234f40eb49938348e40f03afb9730"},{"author":{"_account_id":27482,"name":"MultipleCrashes","email":"inimitableharish@gmail.com","username":"MultipleCrashes"},"change_message_id":"03ebffc04112c3d98f5eea90afcc4f3e4062ce14","unresolved":false,"context_lines":[{"line_number":74,"context_line":"from nova.virt import hardware"},{"line_number":75,"context_line":"from nova.volume import cinder"},{"line_number":76,"context_line":""},{"line_number":77,"context_line":"try:"},{"line_number":78,"context_line":"    # py2.x"},{"line_number":79,"context_line":"    import threading"},{"line_number":80,"context_line":"    x \u003d threading._Event.wait"},{"line_number":81,"context_line":"    patch_func \u003d \u0027threading._Event.wait\u0027"},{"line_number":82,"context_line":"except Exception:"},{"line_number":83,"context_line":"    # py3.x"},{"line_number":84,"context_line":"    patch_func \u003d \u0027threading.Event.wait\u0027"},{"line_number":85,"context_line":""},{"line_number":86,"context_line":"CONF \u003d nova.conf.CONF"},{"line_number":87,"context_line":"fake_host_list \u003d [mock.sentinel.host1]"}],"source_content_type":"text/x-python","patch_set":13,"id":"5f7c97a3_0f29269b","line":84,"range":{"start_line":77,"start_character":0,"end_line":84,"end_character":39},"in_reply_to":"5f7c97a3_58b0bddc","updated":"2018-07-11 18:04:14.000000000","message":"Apparently RetryDector is not using the usual time.sleep to pause, rather it uses wait from Event in threading module, in order to fake/mock the wait time we need to have import for tox to run against python 2 and 3.","commit_id":"80865cfd617234f40eb49938348e40f03afb9730"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"0ae97bdb1f12a49a9b2d07293e3532f0810b9023","unresolved":false,"context_lines":[{"line_number":6070,"context_line":"    def test_try_deallocate_network_retry_direct(self,"},{"line_number":6071,"context_line":"                                                 deallocate_network_mock,"},{"line_number":6072,"context_line":"                                                 time_mock):"},{"line_number":6073,"context_line":"        # set ConnectFailure exception as side effect of deallocate n/w mock"},{"line_number":6074,"context_line":"        deallocate_network_mock.side_effect \u003d \\"},{"line_number":6075,"context_line":"            keystone_exception.connection.ConnectFailure"},{"line_number":6076,"context_line":"        req_networks \u003d objects.NetworkRequestList("}],"source_content_type":"text/x-python","patch_set":13,"id":"5f7c97a3_98bad5bd","line":6073,"range":{"start_line":6073,"start_character":8,"end_line":6073,"end_character":76},"updated":"2018-07-05 14:18:25.000000000","message":"this code comment isn\u0027t necessary since it\u0027s just repeating what the next code line does.","commit_id":"80865cfd617234f40eb49938348e40f03afb9730"},{"author":{"_account_id":27482,"name":"MultipleCrashes","email":"inimitableharish@gmail.com","username":"MultipleCrashes"},"change_message_id":"03ebffc04112c3d98f5eea90afcc4f3e4062ce14","unresolved":false,"context_lines":[{"line_number":6070,"context_line":"    def test_try_deallocate_network_retry_direct(self,"},{"line_number":6071,"context_line":"                                                 deallocate_network_mock,"},{"line_number":6072,"context_line":"                                                 time_mock):"},{"line_number":6073,"context_line":"        # set ConnectFailure exception as side effect of deallocate n/w mock"},{"line_number":6074,"context_line":"        deallocate_network_mock.side_effect \u003d \\"},{"line_number":6075,"context_line":"            keystone_exception.connection.ConnectFailure"},{"line_number":6076,"context_line":"        req_networks \u003d objects.NetworkRequestList("}],"source_content_type":"text/x-python","patch_set":13,"id":"5f7c97a3_2fd2e2fe","line":6073,"range":{"start_line":6073,"start_character":8,"end_line":6073,"end_character":76},"in_reply_to":"5f7c97a3_98bad5bd","updated":"2018-07-11 18:04:14.000000000","message":"will remove in next patch upload.","commit_id":"80865cfd617234f40eb49938348e40f03afb9730"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"772a1f5c5e7dbb12cdfd8d564761a1e80fe5a7dc","unresolved":false,"context_lines":[{"line_number":6065,"context_line":"                self.context, mock.sentinel.instance,"},{"line_number":6066,"context_line":"                requested_networks\u003drequested_networks)"},{"line_number":6067,"context_line":""},{"line_number":6068,"context_line":"    @mock.patch(patch_func)"},{"line_number":6069,"context_line":"    @mock.patch(\u0027nova.compute.manager.ComputeManager._deallocate_network\u0027)"},{"line_number":6070,"context_line":"    def test_try_deallocate_network_retry_direct(self,"},{"line_number":6071,"context_line":"                                                 deallocate_network_mock,"}],"source_content_type":"text/x-python","patch_set":14,"id":"5f7c97a3_551d1abc","line":6068,"range":{"start_line":6068,"start_character":0,"end_line":6068,"end_character":27},"updated":"2018-07-16 13:41:07.000000000","message":"I just pulled your patch locally. I removed this line and all of the stuff above from lines 77-84. Test passes just fine.\n\nPlease remove the import of the threading module along with the patch_func mock that I asked you to remove originally.\n\nThanks,\n\n-jay","commit_id":"85483215e1631d1b0ca37d364cd5496fe3fbefe2"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"cdb54019241503933b83b4c292a2848e97a35975","unresolved":false,"context_lines":[{"line_number":6065,"context_line":"                self.context, mock.sentinel.instance,"},{"line_number":6066,"context_line":"                requested_networks\u003drequested_networks)"},{"line_number":6067,"context_line":""},{"line_number":6068,"context_line":"    @mock.patch(patch_func)"},{"line_number":6069,"context_line":"    @mock.patch(\u0027nova.compute.manager.ComputeManager._deallocate_network\u0027)"},{"line_number":6070,"context_line":"    def test_try_deallocate_network_retry_direct(self,"},{"line_number":6071,"context_line":"                                                 deallocate_network_mock,"}],"source_content_type":"text/x-python","patch_set":14,"id":"5f7c97a3_985be753","line":6068,"range":{"start_line":6068,"start_character":0,"end_line":6068,"end_character":27},"in_reply_to":"5f7c97a3_38bdf366","updated":"2018-07-16 14:16:15.000000000","message":"Yes, the test will take a little while to run due to the retries. If you want to reduce that time, then simply mock out the @loopingcall.RetryDecorator with a noop one of your own.","commit_id":"85483215e1631d1b0ca37d364cd5496fe3fbefe2"},{"author":{"_account_id":27482,"name":"MultipleCrashes","email":"inimitableharish@gmail.com","username":"MultipleCrashes"},"change_message_id":"afec567e6da6483fe756f85edbe2af6a36496bcb","unresolved":false,"context_lines":[{"line_number":6065,"context_line":"                self.context, mock.sentinel.instance,"},{"line_number":6066,"context_line":"                requested_networks\u003drequested_networks)"},{"line_number":6067,"context_line":""},{"line_number":6068,"context_line":"    @mock.patch(patch_func)"},{"line_number":6069,"context_line":"    @mock.patch(\u0027nova.compute.manager.ComputeManager._deallocate_network\u0027)"},{"line_number":6070,"context_line":"    def test_try_deallocate_network_retry_direct(self,"},{"line_number":6071,"context_line":"                                                 deallocate_network_mock,"}],"source_content_type":"text/x-python","patch_set":14,"id":"5f7c97a3_38bdf366","line":6068,"range":{"start_line":6068,"start_character":0,"end_line":6068,"end_character":27},"in_reply_to":"5f7c97a3_551d1abc","updated":"2018-07-16 14:14:12.000000000","message":"Thanks Jay for quick review,\n\n Just wanted to check if on removal of mock path_func , the unit test is actually waits for the execution of this test in your dev environment.I could recall that although the test would pass because of its correctness , but it seems to wait for the retry delays.Will check it in my local env and make the change asap.\n\nRegards,\nHarish.","commit_id":"85483215e1631d1b0ca37d364cd5496fe3fbefe2"},{"author":{"_account_id":27482,"name":"MultipleCrashes","email":"inimitableharish@gmail.com","username":"MultipleCrashes"},"change_message_id":"be9bc2cedfcca08443913ef92132778009099ed9","unresolved":false,"context_lines":[{"line_number":6065,"context_line":"                self.context, mock.sentinel.instance,"},{"line_number":6066,"context_line":"                requested_networks\u003drequested_networks)"},{"line_number":6067,"context_line":""},{"line_number":6068,"context_line":"    @mock.patch(patch_func)"},{"line_number":6069,"context_line":"    @mock.patch(\u0027nova.compute.manager.ComputeManager._deallocate_network\u0027)"},{"line_number":6070,"context_line":"    def test_try_deallocate_network_retry_direct(self,"},{"line_number":6071,"context_line":"                                                 deallocate_network_mock,"}],"source_content_type":"text/x-python","patch_set":14,"id":"5f7c97a3_1d501cd5","line":6068,"range":{"start_line":6068,"start_character":0,"end_line":6068,"end_character":27},"in_reply_to":"5f7c97a3_985be753","updated":"2018-07-16 21:37:51.000000000","message":"Will probably remove the patch mock, since any other alternatives such as mocking of \u0027oslo_service.loopingcall.time.sleep\u0027 , \u0027\u0027oslo_service.eventlet.sleep\u0027, \u0027eventlet.greenthread.sleep does not seem to work\u0027.\nAlso verified that threading Events difference between py 2 and 3 which is the only way it seems to mock the sleep time here.\nFor now, will remove it in the next patch,in which case this takes around 60s.","commit_id":"85483215e1631d1b0ca37d364cd5496fe3fbefe2"},{"author":{"_account_id":6873,"name":"Matt Riedemann","email":"mriedem.os@gmail.com","username":"mriedem"},"change_message_id":"becfe5b6562af6e6ab0cb888f540893f337f3bc7","unresolved":false,"context_lines":[{"line_number":6061,"context_line":"    def test_try_deallocate_network_retry_direct(self,"},{"line_number":6062,"context_line":"                                                 deallocate_network_mock):"},{"line_number":6063,"context_line":"        deallocate_network_mock.side_effect \u003d \\"},{"line_number":6064,"context_line":"            keystone_exception.connection.ConnectFailure"},{"line_number":6065,"context_line":"        req_networks \u003d objects.NetworkRequestList("},{"line_number":6066,"context_line":"            objects\u003d[objects.NetworkRequest(network_id\u003d\u0027auto\u0027)])"},{"line_number":6067,"context_line":"        instance \u003d mock.MagicMock()"}],"source_content_type":"text/x-python","patch_set":16,"id":"5f7c97a3_31c14e82","line":6064,"updated":"2018-07-30 20:49:16.000000000","message":"Should also test that we don\u0027t retry on non-ConnectFailure exceptions.","commit_id":"f5056806f8e2dc8642a393443f3939e16e47a877"}]}