)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":30491,"name":"Radosław Piliszek","display_name":"Radek","email":"radek@piliszek.it","username":"yoctozepto","status":"self-employed techologist, collaborating mostly with 7bulls.com"},"change_message_id":"2125eb55b9e73d234bc4de2a280e9f938b2df3d6","unresolved":false,"context_lines":[{"line_number":14,"context_line":"If one unfinished notification is expired, just set its status to"},{"line_number":15,"context_line":"\u0027failed\u0027."},{"line_number":16,"context_line":""},{"line_number":17,"context_line":"Close-Bug: #1773765"},{"line_number":18,"context_line":"Change-Id: If49635639dd976aeec3ea73e702ad2636fcf1e0a"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":4,"id":"9f560f44_14a5499f","line":17,"range":{"start_line":17,"start_character":0,"end_line":17,"end_character":5},"updated":"2020-08-26 17:24:26.000000000","message":"Closes","commit_id":"3a4f782441f6bdcfb8ee49a393937267fc246c56"}],"masakari/conf/engine.py":[{"author":{"_account_id":1011,"name":"Tushar Patil","email":"tushar.vitthal.patil@gmail.com","username":"tpatil"},"change_message_id":"e37a4bc993eb58ae48e06884f159b0271a313a4a","unresolved":false,"context_lines":[{"line_number":97,"context_line":"                    \"generated_time, then it is considered that notification \""},{"line_number":98,"context_line":"                    \"is ignored by the messaging queue and will be processed \""},{"line_number":99,"context_line":"                    \"by \u0027process_unfinished_notifications\u0027 periodic task.\"),"},{"line_number":100,"context_line":"    cfg.IntOpt(\u0027check_expired_notifications_interval\u0027,"},{"line_number":101,"context_line":"               default\u003d600,"},{"line_number":102,"context_line":"               help\u003d\u0027Interval in seconds for checking running notifications.\u0027),"},{"line_number":103,"context_line":"    cfg.IntOpt(\u0027notifications_expired_interval\u0027,"}],"source_content_type":"text/x-python","patch_set":4,"id":"1f493fa4_10ab721d","line":100,"range":{"start_line":100,"start_character":16,"end_line":100,"end_character":52},"updated":"2020-04-20 23:01:45.000000000","message":"IMO, we can override init_host method and process notifications in NEW/ERROR statuses so that when masakari-engine is restarted, it will process notifications in NEW/ERROR statuses. I think with this implementation, there will be no need to add a new periodic task.\n\nAlso, we need to think what should be the default value of config options \"process_unfinished_notifications_interval\" and \"retry_notification_new_status_interval\". \n\nShould process_unfinished_notifications_interval \u003c retry_notification_new_status_interval?","commit_id":"3a4f782441f6bdcfb8ee49a393937267fc246c56"},{"author":{"_account_id":24501,"name":"sue","display_name":"suzhengwei","email":"sugar-2008@163.com","username":"suzhengwei"},"change_message_id":"68cf19ff1eb7fdc17f3eda80a82ef06078d0d027","unresolved":false,"context_lines":[{"line_number":97,"context_line":"                    \"generated_time, then it is considered that notification \""},{"line_number":98,"context_line":"                    \"is ignored by the messaging queue and will be processed \""},{"line_number":99,"context_line":"                    \"by \u0027process_unfinished_notifications\u0027 periodic task.\"),"},{"line_number":100,"context_line":"    cfg.IntOpt(\u0027check_expired_notifications_interval\u0027,"},{"line_number":101,"context_line":"               default\u003d600,"},{"line_number":102,"context_line":"               help\u003d\u0027Interval in seconds for checking running notifications.\u0027),"},{"line_number":103,"context_line":"    cfg.IntOpt(\u0027notifications_expired_interval\u0027,"}],"source_content_type":"text/x-python","patch_set":4,"id":"1f493fa4_67d2fd36","line":100,"range":{"start_line":100,"start_character":16,"end_line":100,"end_character":52},"in_reply_to":"1f493fa4_10ab721d","updated":"2020-04-21 01:17:52.000000000","message":"That’s one choice. But there still is one risk.  In the real condition, if one notification has been in NEW status for long time, it is not allowed to be processed again. Only notifitions is not older than \u0027retry_notification_new_status_interval\u0027 will be processed again. If it is older, it will remains all times.\nTo the notifications in \u0027ERROR\u0027 status, it will be update to \u0027FAILED\u0027 status. It\u0027s not a problem.\nTo the notifications in \u0027RUNNIG\u0027 status, there is no method to check if it is in processing. It will remain all times.\nI think they are both necessary, to check expired notifications and process unfinished notifications.","commit_id":"3a4f782441f6bdcfb8ee49a393937267fc246c56"},{"author":{"_account_id":24501,"name":"sue","display_name":"suzhengwei","email":"sugar-2008@163.com","username":"suzhengwei"},"change_message_id":"2aaa1926cec83f60259b1212fd52470ffd6f73f5","unresolved":false,"context_lines":[{"line_number":97,"context_line":"                    \"generated_time, then it is considered that notification \""},{"line_number":98,"context_line":"                    \"is ignored by the messaging queue and will be processed \""},{"line_number":99,"context_line":"                    \"by \u0027process_unfinished_notifications\u0027 periodic task.\"),"},{"line_number":100,"context_line":"    cfg.IntOpt(\u0027check_expired_notifications_interval\u0027,"},{"line_number":101,"context_line":"               default\u003d600,"},{"line_number":102,"context_line":"               help\u003d\u0027Interval in seconds for checking running notifications.\u0027),"},{"line_number":103,"context_line":"    cfg.IntOpt(\u0027notifications_expired_interval\u0027,"}],"source_content_type":"text/x-python","patch_set":4,"id":"1f493fa4_9a9e7c05","line":100,"range":{"start_line":100,"start_character":16,"end_line":100,"end_character":52},"in_reply_to":"1f493fa4_3a2c8898","updated":"2020-04-21 02:37:33.000000000","message":"Why one notification woud be in NEW status for long time, it is another feature I will import in V. In short, due to limit and circuit breaker when large scale hosts failure， not all notifications will be process at once, so there will be some notifications remain in NEW status for long time.\nTo notification in RUNNIG status, if there is one masakari-engine, it can be processed again when masakari-engine restarted again. but if there are more than two masakari-engine, how do we verify it is in processing or terminated abruptly?","commit_id":"3a4f782441f6bdcfb8ee49a393937267fc246c56"},{"author":{"_account_id":1011,"name":"Tushar Patil","email":"tushar.vitthal.patil@gmail.com","username":"tpatil"},"change_message_id":"9e58128da2f73d0ac038510730cb8ead181e27c6","unresolved":false,"context_lines":[{"line_number":97,"context_line":"                    \"generated_time, then it is considered that notification \""},{"line_number":98,"context_line":"                    \"is ignored by the messaging queue and will be processed \""},{"line_number":99,"context_line":"                    \"by \u0027process_unfinished_notifications\u0027 periodic task.\"),"},{"line_number":100,"context_line":"    cfg.IntOpt(\u0027check_expired_notifications_interval\u0027,"},{"line_number":101,"context_line":"               default\u003d600,"},{"line_number":102,"context_line":"               help\u003d\u0027Interval in seconds for checking running notifications.\u0027),"},{"line_number":103,"context_line":"    cfg.IntOpt(\u0027notifications_expired_interval\u0027,"}],"source_content_type":"text/x-python","patch_set":4,"id":"1f493fa4_3a2c8898","line":100,"range":{"start_line":100,"start_character":16,"end_line":100,"end_character":52},"in_reply_to":"1f493fa4_67d2fd36","updated":"2020-04-21 02:01:43.000000000","message":"\u003e if one notification has been in NEW status for long time,  it is not allowed to be processed again\nI\u0027m trying to understand in what circumstances, the notifications in NEW status will not be processed?\nIs it because, masakari-engine is down or rabbitMQ is down?\n\nIf a notification is set to \"RUNNING\" status, the next transition would be from RUNNING-\u003e(FINISHED or IGNORED or ERROR). Before setting it to either of these status, if the masakari-engine process is terminated  abruptly, then when it will be restarted again, we can process notifications which are in NEW, RUNNING, or ERROR statuses in the init_host method.","commit_id":"3a4f782441f6bdcfb8ee49a393937267fc246c56"},{"author":{"_account_id":1011,"name":"Tushar Patil","email":"tushar.vitthal.patil@gmail.com","username":"tpatil"},"change_message_id":"0d06bc29899f12d553b6e70d53d062e2ca828489","unresolved":false,"context_lines":[{"line_number":97,"context_line":"                    \"generated_time, then it is considered that notification \""},{"line_number":98,"context_line":"                    \"is ignored by the messaging queue and will be processed \""},{"line_number":99,"context_line":"                    \"by \u0027process_unfinished_notifications\u0027 periodic task.\"),"},{"line_number":100,"context_line":"    cfg.IntOpt(\u0027check_expired_notifications_interval\u0027,"},{"line_number":101,"context_line":"               default\u003d600,"},{"line_number":102,"context_line":"               help\u003d\u0027Interval in seconds for checking running notifications.\u0027),"},{"line_number":103,"context_line":"    cfg.IntOpt(\u0027notifications_expired_interval\u0027,"}],"source_content_type":"text/x-python","patch_set":4,"id":"ff570b3c_c96a4beb","line":100,"range":{"start_line":100,"start_character":16,"end_line":100,"end_character":52},"in_reply_to":"1f493fa4_9a9e7c05","updated":"2020-05-19 07:51:41.000000000","message":"It\u0027s not supported in the current design. If you run two masakari-engine services at the same time on two different hosts, then it would pickup same notifications and process it inside periodic task which could lead to more severe problems.","commit_id":"3a4f782441f6bdcfb8ee49a393937267fc246c56"},{"author":{"_account_id":24501,"name":"sue","display_name":"suzhengwei","email":"sugar-2008@163.com","username":"suzhengwei"},"change_message_id":"b745acb575d0ecf3fea10b640325e8e77a0f271d","unresolved":false,"context_lines":[{"line_number":97,"context_line":"                    \"generated_time, then it is considered that notification \""},{"line_number":98,"context_line":"                    \"is ignored by the messaging queue and will be processed \""},{"line_number":99,"context_line":"                    \"by \u0027process_unfinished_notifications\u0027 periodic task.\"),"},{"line_number":100,"context_line":"    cfg.IntOpt(\u0027check_expired_notifications_interval\u0027,"},{"line_number":101,"context_line":"               default\u003d600,"},{"line_number":102,"context_line":"               help\u003d\u0027Interval in seconds for checking running notifications.\u0027),"},{"line_number":103,"context_line":"    cfg.IntOpt(\u0027notifications_expired_interval\u0027,"}],"source_content_type":"text/x-python","patch_set":4,"id":"ff570b3c_dbc33288","line":100,"range":{"start_line":100,"start_character":16,"end_line":100,"end_character":52},"in_reply_to":"ff570b3c_c96a4beb","updated":"2020-05-19 08:26:49.000000000","message":"It\u0027s improper understanding. Like other project, the masakari-engine process the notification by rpc called from masakari-api. The rpc called will ensure only one masakari-engine will continously process the notification. \nEven if the rpc call was blocked, the status of the notification acts like a DB lock. Only one masakari-engine can occupy the \u0027NEW\u0027notification and update the status to \u0027RUNNING\u0027, so other masakari-engine won\u0027t process the \u0027RUNNING\u0027 notification again.\n\nThis is tested in my env, 3 masakari-api/engine on 3 control nodes.","commit_id":"3a4f782441f6bdcfb8ee49a393937267fc246c56"},{"author":{"_account_id":30491,"name":"Radosław Piliszek","display_name":"Radek","email":"radek@piliszek.it","username":"yoctozepto","status":"self-employed techologist, collaborating mostly with 7bulls.com"},"change_message_id":"2125eb55b9e73d234bc4de2a280e9f938b2df3d6","unresolved":false,"context_lines":[{"line_number":97,"context_line":"                    \"generated_time, then it is considered that notification \""},{"line_number":98,"context_line":"                    \"is ignored by the messaging queue and will be processed \""},{"line_number":99,"context_line":"                    \"by \u0027process_unfinished_notifications\u0027 periodic task.\"),"},{"line_number":100,"context_line":"    cfg.IntOpt(\u0027check_expired_notifications_interval\u0027,"},{"line_number":101,"context_line":"               default\u003d600,"},{"line_number":102,"context_line":"               help\u003d\u0027Interval in seconds for checking running notifications.\u0027),"},{"line_number":103,"context_line":"    cfg.IntOpt(\u0027notifications_expired_interval\u0027,"}],"source_content_type":"text/x-python","patch_set":4,"id":"9f560f44_14c26953","line":100,"range":{"start_line":100,"start_character":16,"end_line":100,"end_character":52},"in_reply_to":"ff570b3c_dbc33288","updated":"2020-08-26 17:24:26.000000000","message":"I generally observed behaviour reported by suzhengwei\u0027s so I agree with suzhengwei on that.\n\nBut I guess we could still mix the two ideas and check on init while considering the timeout, hmm? I guess we need a bit better design so that engines actually coordinate this between themselves (who\u0027s working on which notification). Due to the simple nature of this resource (the notification) we could probably get away without any heavy coordination via tooz and instead keep the necessary information in the database (as handling uniqueness is done by the message queue already).","commit_id":"3a4f782441f6bdcfb8ee49a393937267fc246c56"}],"masakari/engine/manager.py":[{"author":{"_account_id":1011,"name":"Tushar Patil","email":"tushar.vitthal.patil@gmail.com","username":"tpatil"},"change_message_id":"beef77f2dfdd2fc2074601016f9b5c73597867cc","unresolved":false,"context_lines":[{"line_number":372,"context_line":"        spacing\u003dCONF.check_expired_notifications_interval)"},{"line_number":373,"context_line":"    def _check_expired_notifications(self, context):"},{"line_number":374,"context_line":"        filters \u003d {"},{"line_number":375,"context_line":"            \u0027status\u0027: [fields.NotificationStatus.RUNNING,"},{"line_number":376,"context_line":"                       fields.NotificationStatus.ERROR,"},{"line_number":377,"context_line":"                       fields.NotificationStatus.NEW]"},{"line_number":378,"context_line":"        }"}],"source_content_type":"text/x-python","patch_set":4,"id":"1f493fa4_9a983c6b","line":375,"range":{"start_line":375,"start_character":49,"end_line":375,"end_character":56},"updated":"2020-04-21 02:16:58.000000000","message":"If a notification is under processing i..e RUNNING, and if this periodic task executes, then there are chances the status would be set to FAILED leading to unexpected result.","commit_id":"3a4f782441f6bdcfb8ee49a393937267fc246c56"}],"masakari/tests/unit/engine/test_engine_mgr.py":[{"author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"tag":"autogenerated:zuul:check","change_message_id":"38395226eeb14524072a9d8ea77214a242a0d10c","unresolved":false,"context_lines":[{"line_number":34,"context_line":""},{"line_number":35,"context_line":"NOW \u003d timeutils.utcnow().replace(microsecond\u003d0)"},{"line_number":36,"context_line":"EXPIRED_TIME \u003d timeutils.utcnow().replace(microsecond\u003d0) \\"},{"line_number":37,"context_line":"    - datetime.timedelta(seconds\u003dCONF.notifications_expired_interval)"},{"line_number":38,"context_line":""},{"line_number":39,"context_line":"def _get_vm_type_notification(status\u003d\"new\"):"},{"line_number":40,"context_line":"    return fakes.create_fake_notification("}],"source_content_type":"text/x-python","patch_set":2,"id":"3f4c43b2_0cd78325","line":37,"updated":"2020-04-17 03:04:20.000000000","message":"pep8: F821 undefined name \u0027datetime\u0027","commit_id":"eeca10d3d1564da7a62f9be6a85630310ac914e9"},{"author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"tag":"autogenerated:zuul:check","change_message_id":"38395226eeb14524072a9d8ea77214a242a0d10c","unresolved":false,"context_lines":[{"line_number":36,"context_line":"EXPIRED_TIME \u003d timeutils.utcnow().replace(microsecond\u003d0) \\"},{"line_number":37,"context_line":"    - datetime.timedelta(seconds\u003dCONF.notifications_expired_interval)"},{"line_number":38,"context_line":""},{"line_number":39,"context_line":"def _get_vm_type_notification(status\u003d\"new\"):"},{"line_number":40,"context_line":"    return fakes.create_fake_notification("},{"line_number":41,"context_line":"        type\u003d\"VM\", id\u003d1, payload\u003d{"},{"line_number":42,"context_line":"            \u0027event\u0027: \u0027LIFECYCLE\u0027, \u0027instance_uuid\u0027: uuidsentinel.fake_ins,"}],"source_content_type":"text/x-python","patch_set":2,"id":"3f4c43b2_6cde0740","line":39,"updated":"2020-04-17 03:04:20.000000000","message":"pep8: E302 expected 2 blank lines, found 1","commit_id":"eeca10d3d1564da7a62f9be6a85630310ac914e9"}]}