)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":9531,"name":"liuyulong","display_name":"LIU Yulong","email":"i@liuyulong.me","username":"LIU-Yulong"},"change_message_id":"3863864575a4922fb3cfdb24e923db82591b6c30","unresolved":false,"context_lines":[{"line_number":9,"context_line":"Sometime between liberty and pike, adding rules to SG\u0027s got"},{"line_number":10,"context_line":"slow, and slower with every rule. Streamline the rule create path,"},{"line_number":11,"context_line":"and get close to the old performance back."},{"line_number":12,"context_line":""},{"line_number":13,"context_line":"Change-Id: I34e41a128f28211f2e7ab814a2611ce22620fcf3"},{"line_number":14,"context_line":"Closes-bug: 1810563"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":9,"id":"dfd5e7cf_ff0ca243","line":12,"updated":"2019-01-07 09:51:10.000000000","message":"Nit nit, it\u0027s better to say something about why this fix helps. It is always welcomed for reviewers. Thanks.","commit_id":"932d9527fcf236cec73a70525597f7bff6298335"},{"author":{"_account_id":10980,"name":"Doug Wiegley","email":"dougwig@parkside.io","username":"dougw"},"change_message_id":"e1dbf5bccb5705972f089791ab1fa4288a6dfae6","unresolved":false,"context_lines":[{"line_number":9,"context_line":"Sometime between liberty and pike, adding rules to SG\u0027s got"},{"line_number":10,"context_line":"slow, and slower with every rule. Streamline the rule create path,"},{"line_number":11,"context_line":"and get close to the old performance back."},{"line_number":12,"context_line":""},{"line_number":13,"context_line":"Change-Id: I34e41a128f28211f2e7ab814a2611ce22620fcf3"},{"line_number":14,"context_line":"Closes-bug: 1810563"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":9,"id":"dfd5e7cf_0176df07","line":12,"in_reply_to":"dfd5e7cf_ff0ca243","updated":"2019-01-07 19:32:52.000000000","message":"Done","commit_id":"932d9527fcf236cec73a70525597f7bff6298335"},{"author":{"_account_id":27654,"name":"Hongbin Lu","email":"kira034@163.com","username":"hongbin.lu"},"change_message_id":"42033fb0f64e17cc0e9235b6fa15915c72ec0c0b","unresolved":false,"context_lines":[{"line_number":10,"context_line":"slow, and slower with every rule. Streamline the rule create path,"},{"line_number":11,"context_line":"and get close to the old performance back."},{"line_number":12,"context_line":""},{"line_number":13,"context_line":"Two performance fixes:"},{"line_number":14,"context_line":"1. Get rid of an n^2 duplicate check, using a hash table instead,"},{"line_number":15,"context_line":"on bulk creates. This is more memory intensive than the previous loop,"},{"line_number":16,"context_line":"but usable far past where the other becomes too slow to be useful."}],"source_content_type":"text/x-gerrit-commit-message","patch_set":15,"id":"dfd5e7cf_f767aa50","line":13,"range":{"start_line":13,"start_character":0,"end_line":13,"end_character":3},"updated":"2019-01-09 21:53:59.000000000","message":"If feasible, I would suggest to split this patches into two so that each patch can be debated and backported separately.","commit_id":"2d95136999908fc9423512505a42df6e45c194af"},{"author":{"_account_id":27654,"name":"Hongbin Lu","email":"kira034@163.com","username":"hongbin.lu"},"change_message_id":"a0491f3ab681bcd2df80a9b48a35274e81bfd1bf","unresolved":false,"context_lines":[{"line_number":10,"context_line":"slow, and slower with every rule. Streamline the rule create path,"},{"line_number":11,"context_line":"and get close to the old performance back."},{"line_number":12,"context_line":""},{"line_number":13,"context_line":"Two performance fixes:"},{"line_number":14,"context_line":"1. Get rid of an n^2 duplicate check, using a hash table instead,"},{"line_number":15,"context_line":"on bulk creates. This is more memory intensive than the previous loop,"},{"line_number":16,"context_line":"but usable far past where the other becomes too slow to be useful."}],"source_content_type":"text/x-gerrit-commit-message","patch_set":15,"id":"dfd5e7cf_c53c1ec1","line":13,"range":{"start_line":13,"start_character":0,"end_line":13,"end_character":3},"in_reply_to":"dfd5e7cf_7d4f1992","updated":"2019-01-10 16:02:21.000000000","message":"OK","commit_id":"2d95136999908fc9423512505a42df6e45c194af"},{"author":{"_account_id":10980,"name":"Doug Wiegley","email":"dougwig@parkside.io","username":"dougw"},"change_message_id":"ce42038dc4ff018c66cc7f377b8b073c9da3ac3e","unresolved":false,"context_lines":[{"line_number":10,"context_line":"slow, and slower with every rule. Streamline the rule create path,"},{"line_number":11,"context_line":"and get close to the old performance back."},{"line_number":12,"context_line":""},{"line_number":13,"context_line":"Two performance fixes:"},{"line_number":14,"context_line":"1. Get rid of an n^2 duplicate check, using a hash table instead,"},{"line_number":15,"context_line":"on bulk creates. This is more memory intensive than the previous loop,"},{"line_number":16,"context_line":"but usable far past where the other becomes too slow to be useful."}],"source_content_type":"text/x-gerrit-commit-message","patch_set":15,"id":"bfdaf3ff_f8e58c39","line":13,"range":{"start_line":13,"start_character":0,"end_line":13,"end_character":3},"in_reply_to":"dfd5e7cf_c53c1ec1","updated":"2019-01-11 21:19:13.000000000","message":"Actually, I agree with you. Since we can use object_exists in the write path, the rule create and sg index fixes can be separate patches.","commit_id":"2d95136999908fc9423512505a42df6e45c194af"},{"author":{"_account_id":10980,"name":"Doug Wiegley","email":"dougwig@parkside.io","username":"dougw"},"change_message_id":"fe7c38730588f17b872d8c6a0614164bd42832c5","unresolved":false,"context_lines":[{"line_number":10,"context_line":"slow, and slower with every rule. Streamline the rule create path,"},{"line_number":11,"context_line":"and get close to the old performance back."},{"line_number":12,"context_line":""},{"line_number":13,"context_line":"Two performance fixes:"},{"line_number":14,"context_line":"1. Get rid of an n^2 duplicate check, using a hash table instead,"},{"line_number":15,"context_line":"on bulk creates. This is more memory intensive than the previous loop,"},{"line_number":16,"context_line":"but usable far past where the other becomes too slow to be useful."}],"source_content_type":"text/x-gerrit-commit-message","patch_set":15,"id":"dfd5e7cf_7d4f1992","line":13,"range":{"start_line":13,"start_character":0,"end_line":13,"end_character":3},"in_reply_to":"dfd5e7cf_f767aa50","updated":"2019-01-10 00:37:12.000000000","message":"I could, but either change is useless without the other. Unless this patch is really un-reviewable as is, I\u0027d rather not add overhead.","commit_id":"2d95136999908fc9423512505a42df6e45c194af"}],"neutron/db/securitygroups_db.py":[{"author":{"_account_id":1131,"name":"Brian Haley","email":"haleyb.dev@gmail.com","username":"brian-haley"},"change_message_id":"591da1d066f9bf353d65c235f9fbb86678f8445c","unresolved":false,"context_lines":[{"line_number":214,"context_line":"        finally:"},{"line_number":215,"context_line":"            if tenant_id:"},{"line_number":216,"context_line":"                context.tenant_id \u003d tmp_context_tenant_id"},{"line_number":217,"context_line":"        return ret"},{"line_number":218,"context_line":""},{"line_number":219,"context_line":"    def _get_security_group_db(self, context, id):"},{"line_number":220,"context_line":"        try:"}],"source_content_type":"text/x-python","patch_set":2,"id":"dfd5e7cf_f165c8e2","line":217,"updated":"2019-01-04 22:37:32.000000000","message":"Obviously this method is duplicating most of get_security_group(), would be better if it could just be combined with that.","commit_id":"fb5d86232b91b75a7de528e4671ad92c8a4184bb"},{"author":{"_account_id":10980,"name":"Doug Wiegley","email":"dougwig@parkside.io","username":"dougw"},"change_message_id":"1f214be6e0dda789e0b73261d95af5663e3cd057","unresolved":false,"context_lines":[{"line_number":214,"context_line":"        finally:"},{"line_number":215,"context_line":"            if tenant_id:"},{"line_number":216,"context_line":"                context.tenant_id \u003d tmp_context_tenant_id"},{"line_number":217,"context_line":"        return ret"},{"line_number":218,"context_line":""},{"line_number":219,"context_line":"    def _get_security_group_db(self, context, id):"},{"line_number":220,"context_line":"        try:"}],"source_content_type":"text/x-python","patch_set":2,"id":"dfd5e7cf_5a4ce975","line":217,"in_reply_to":"dfd5e7cf_91531ce3","updated":"2019-01-05 05:06:20.000000000","message":"Done","commit_id":"fb5d86232b91b75a7de528e4671ad92c8a4184bb"},{"author":{"_account_id":10980,"name":"Doug Wiegley","email":"dougwig@parkside.io","username":"dougw"},"change_message_id":"b07e2cbee8d9f460dd76837d60964a2f4a067885","unresolved":false,"context_lines":[{"line_number":214,"context_line":"        finally:"},{"line_number":215,"context_line":"            if tenant_id:"},{"line_number":216,"context_line":"                context.tenant_id \u003d tmp_context_tenant_id"},{"line_number":217,"context_line":"        return ret"},{"line_number":218,"context_line":""},{"line_number":219,"context_line":"    def _get_security_group_db(self, context, id):"},{"line_number":220,"context_line":"        try:"}],"source_content_type":"text/x-python","patch_set":2,"id":"dfd5e7cf_91531ce3","line":217,"in_reply_to":"dfd5e7cf_f165c8e2","updated":"2019-01-04 22:45:22.000000000","message":"I had it that way, but it looked messy to me.  I\u0027ll put it back in one.","commit_id":"fb5d86232b91b75a7de528e4671ad92c8a4184bb"},{"author":{"_account_id":1131,"name":"Brian Haley","email":"haleyb.dev@gmail.com","username":"brian-haley"},"change_message_id":"591da1d066f9bf353d65c235f9fbb86678f8445c","unresolved":false,"context_lines":[{"line_number":219,"context_line":"    def _get_security_group_db(self, context, id):"},{"line_number":220,"context_line":"        try:"},{"line_number":221,"context_line":"            query \u003d self._model_query(context, sg_models.SecurityGroup)"},{"line_number":222,"context_line":"            sg \u003d query.filter(sg_models.SecurityGroup.id \u003d\u003d id).one()"},{"line_number":223,"context_line":""},{"line_number":224,"context_line":"        except exc.NoResultFound:"},{"line_number":225,"context_line":"            raise ext_sg.SecurityGroupNotFound(id\u003did)"}],"source_content_type":"text/x-python","patch_set":2,"id":"dfd5e7cf_5175b497","line":222,"updated":"2019-01-04 22:37:32.000000000","message":"This is also rolling back the OVO work, is _get_security_group() that slow?","commit_id":"fb5d86232b91b75a7de528e4671ad92c8a4184bb"},{"author":{"_account_id":10980,"name":"Doug Wiegley","email":"dougwig@parkside.io","username":"dougw"},"change_message_id":"3ba6552a5a4ed9b5592b09fc1384f8e9432a7ef9","unresolved":false,"context_lines":[{"line_number":219,"context_line":"    def _get_security_group_db(self, context, id):"},{"line_number":220,"context_line":"        try:"},{"line_number":221,"context_line":"            query \u003d self._model_query(context, sg_models.SecurityGroup)"},{"line_number":222,"context_line":"            sg \u003d query.filter(sg_models.SecurityGroup.id \u003d\u003d id).one()"},{"line_number":223,"context_line":""},{"line_number":224,"context_line":"        except exc.NoResultFound:"},{"line_number":225,"context_line":"            raise ext_sg.SecurityGroupNotFound(id\u003did)"}],"source_content_type":"text/x-python","patch_set":2,"id":"dfd5e7cf_ba46354f","line":222,"in_reply_to":"dfd5e7cf_115bac87","updated":"2019-01-05 05:09:03.000000000","message":"I\u0027m certainly open to that discussion. I figured a tiny db get might be ok, for a performance fix that should be backported.\n\nIt really looks like we\u0027re fetching the rules *twice* in get(!shallow) today. Once explicitly, and once via ORM implications. Let me know if this single reference is a really big issue...","commit_id":"fb5d86232b91b75a7de528e4671ad92c8a4184bb"},{"author":{"_account_id":10980,"name":"Doug Wiegley","email":"dougwig@parkside.io","username":"dougw"},"change_message_id":"b07e2cbee8d9f460dd76837d60964a2f4a067885","unresolved":false,"context_lines":[{"line_number":219,"context_line":"    def _get_security_group_db(self, context, id):"},{"line_number":220,"context_line":"        try:"},{"line_number":221,"context_line":"            query \u003d self._model_query(context, sg_models.SecurityGroup)"},{"line_number":222,"context_line":"            sg \u003d query.filter(sg_models.SecurityGroup.id \u003d\u003d id).one()"},{"line_number":223,"context_line":""},{"line_number":224,"context_line":"        except exc.NoResultFound:"},{"line_number":225,"context_line":"            raise ext_sg.SecurityGroupNotFound(id\u003did)"}],"source_content_type":"text/x-python","patch_set":2,"id":"dfd5e7cf_716478ba","line":222,"in_reply_to":"dfd5e7cf_5175b497","updated":"2019-01-04 22:45:22.000000000","message":"Yes. The alternative would be to put some lazy-ref semantics into the ovo objects. Can we get away with this one direct db call?","commit_id":"fb5d86232b91b75a7de528e4671ad92c8a4184bb"},{"author":{"_account_id":1131,"name":"Brian Haley","email":"haleyb.dev@gmail.com","username":"brian-haley"},"change_message_id":"ef0a586dc6d342a86f0ef055a907f62a7e8a9598","unresolved":false,"context_lines":[{"line_number":219,"context_line":"    def _get_security_group_db(self, context, id):"},{"line_number":220,"context_line":"        try:"},{"line_number":221,"context_line":"            query \u003d self._model_query(context, sg_models.SecurityGroup)"},{"line_number":222,"context_line":"            sg \u003d query.filter(sg_models.SecurityGroup.id \u003d\u003d id).one()"},{"line_number":223,"context_line":""},{"line_number":224,"context_line":"        except exc.NoResultFound:"},{"line_number":225,"context_line":"            raise ext_sg.SecurityGroupNotFound(id\u003did)"}],"source_content_type":"text/x-python","patch_set":2,"id":"dfd5e7cf_115bac87","line":222,"in_reply_to":"dfd5e7cf_51d29436","updated":"2019-01-04 23:07:36.000000000","message":"I\u0027m not super close to the OVO work, but if they\u0027ve introduced a slowdown then we should look into fixing it somehow.  I\u0027d be curious what someone more involved would say, or even what Mike Bayer might say.  Sometimes we write bad code and there\u0027s a better way...","commit_id":"fb5d86232b91b75a7de528e4671ad92c8a4184bb"},{"author":{"_account_id":10980,"name":"Doug Wiegley","email":"dougwig@parkside.io","username":"dougw"},"change_message_id":"93be4c96457749a872e2665a6b6ee673131da5ec","unresolved":false,"context_lines":[{"line_number":219,"context_line":"    def _get_security_group_db(self, context, id):"},{"line_number":220,"context_line":"        try:"},{"line_number":221,"context_line":"            query \u003d self._model_query(context, sg_models.SecurityGroup)"},{"line_number":222,"context_line":"            sg \u003d query.filter(sg_models.SecurityGroup.id \u003d\u003d id).one()"},{"line_number":223,"context_line":""},{"line_number":224,"context_line":"        except exc.NoResultFound:"},{"line_number":225,"context_line":"            raise ext_sg.SecurityGroupNotFound(id\u003did)"}],"source_content_type":"text/x-python","patch_set":2,"id":"dfd5e7cf_51d29436","line":222,"in_reply_to":"dfd5e7cf_716478ba","updated":"2019-01-04 22:48:52.000000000","message":"To answer your question more succintly, yes, it\u0027s really slow. I included a script for repro in the bug report. And none of the calls I changed affect RPC writes, so the overhead was unnecessary.","commit_id":"fb5d86232b91b75a7de528e4671ad92c8a4184bb"},{"author":{"_account_id":1131,"name":"Brian Haley","email":"haleyb.dev@gmail.com","username":"brian-haley"},"change_message_id":"591da1d066f9bf353d65c235f9fbb86678f8445c","unresolved":false,"context_lines":[{"line_number":430,"context_line":"            if validate:"},{"line_number":431,"context_line":"                self._check_for_duplicate_rules_in_db(context,"},{"line_number":432,"context_line":"                                                      security_group_rule)"},{"line_number":433,"context_line":"            db \u003d sg_models.SecurityGroupRule("},{"line_number":434,"context_line":"                id\u003d(rule_dict.get(\u0027id\u0027) or uuidutils.generate_uuid()),"},{"line_number":435,"context_line":"                tenant_id\u003drule_dict[\u0027tenant_id\u0027],"},{"line_number":436,"context_line":"                security_group_id\u003drule_dict[\u0027security_group_id\u0027],"}],"source_content_type":"text/x-python","patch_set":2,"id":"ffd0ebdf_e9d13259","line":433,"updated":"2019-01-04 22:37:32.000000000","message":"Yeah, we can\u0027t un-do the OVO work, we have to live with it.","commit_id":"fb5d86232b91b75a7de528e4671ad92c8a4184bb"},{"author":{"_account_id":10980,"name":"Doug Wiegley","email":"dougwig@parkside.io","username":"dougw"},"change_message_id":"1f214be6e0dda789e0b73261d95af5663e3cd057","unresolved":false,"context_lines":[{"line_number":430,"context_line":"            if validate:"},{"line_number":431,"context_line":"                self._check_for_duplicate_rules_in_db(context,"},{"line_number":432,"context_line":"                                                      security_group_rule)"},{"line_number":433,"context_line":"            db \u003d sg_models.SecurityGroupRule("},{"line_number":434,"context_line":"                id\u003d(rule_dict.get(\u0027id\u0027) or uuidutils.generate_uuid()),"},{"line_number":435,"context_line":"                tenant_id\u003drule_dict[\u0027tenant_id\u0027],"},{"line_number":436,"context_line":"                security_group_id\u003drule_dict[\u0027security_group_id\u0027],"}],"source_content_type":"text/x-python","patch_set":2,"id":"dfd5e7cf_7a51ad0e","line":433,"in_reply_to":"dfd5e7cf_11596cfd","updated":"2019-01-05 05:06:20.000000000","message":"Done","commit_id":"fb5d86232b91b75a7de528e4671ad92c8a4184bb"},{"author":{"_account_id":10980,"name":"Doug Wiegley","email":"dougwig@parkside.io","username":"dougw"},"change_message_id":"b07e2cbee8d9f460dd76837d60964a2f4a067885","unresolved":false,"context_lines":[{"line_number":430,"context_line":"            if validate:"},{"line_number":431,"context_line":"                self._check_for_duplicate_rules_in_db(context,"},{"line_number":432,"context_line":"                                                      security_group_rule)"},{"line_number":433,"context_line":"            db \u003d sg_models.SecurityGroupRule("},{"line_number":434,"context_line":"                id\u003d(rule_dict.get(\u0027id\u0027) or uuidutils.generate_uuid()),"},{"line_number":435,"context_line":"                tenant_id\u003drule_dict[\u0027tenant_id\u0027],"},{"line_number":436,"context_line":"                security_group_id\u003drule_dict[\u0027security_group_id\u0027],"}],"source_content_type":"text/x-python","patch_set":2,"id":"dfd5e7cf_11596cfd","line":433,"in_reply_to":"ffd0ebdf_e9d13259","updated":"2019-01-04 22:45:22.000000000","message":"Ok, will restore that here.","commit_id":"fb5d86232b91b75a7de528e4671ad92c8a4184bb"},{"author":{"_account_id":1131,"name":"Brian Haley","email":"haleyb.dev@gmail.com","username":"brian-haley"},"change_message_id":"591da1d066f9bf353d65c235f9fbb86678f8445c","unresolved":false,"context_lines":[{"line_number":630,"context_line":""},{"line_number":631,"context_line":"    def _check_for_duplicate_rules(self, context, security_group_rules):"},{"line_number":632,"context_line":"        if not cfg.CONF.security_group_duplicate_detection:"},{"line_number":633,"context_line":"            return"},{"line_number":634,"context_line":"        for i in security_group_rules:"},{"line_number":635,"context_line":"            found_self \u003d False"},{"line_number":636,"context_line":"            for j in security_group_rules:"}],"source_content_type":"text/x-python","patch_set":2,"id":"ffd0ebdf_69f8e2c3","line":633,"updated":"2019-01-04 22:37:32.000000000","message":"Yuck.","commit_id":"fb5d86232b91b75a7de528e4671ad92c8a4184bb"},{"author":{"_account_id":10980,"name":"Doug Wiegley","email":"dougwig@parkside.io","username":"dougw"},"change_message_id":"1f214be6e0dda789e0b73261d95af5663e3cd057","unresolved":false,"context_lines":[{"line_number":630,"context_line":""},{"line_number":631,"context_line":"    def _check_for_duplicate_rules(self, context, security_group_rules):"},{"line_number":632,"context_line":"        if not cfg.CONF.security_group_duplicate_detection:"},{"line_number":633,"context_line":"            return"},{"line_number":634,"context_line":"        for i in security_group_rules:"},{"line_number":635,"context_line":"            found_self \u003d False"},{"line_number":636,"context_line":"            for j in security_group_rules:"}],"source_content_type":"text/x-python","patch_set":2,"id":"dfd5e7cf_1a566108","line":633,"in_reply_to":"dfd5e7cf_511374b9","updated":"2019-01-05 05:06:20.000000000","message":"Rewrote the dup detection from 2xO(N^2) to O(N), so this is no longer needed.  Removed the option. I agree that it was gross.","commit_id":"fb5d86232b91b75a7de528e4671ad92c8a4184bb"},{"author":{"_account_id":1131,"name":"Brian Haley","email":"haleyb.dev@gmail.com","username":"brian-haley"},"change_message_id":"ef0a586dc6d342a86f0ef055a907f62a7e8a9598","unresolved":false,"context_lines":[{"line_number":630,"context_line":""},{"line_number":631,"context_line":"    def _check_for_duplicate_rules(self, context, security_group_rules):"},{"line_number":632,"context_line":"        if not cfg.CONF.security_group_duplicate_detection:"},{"line_number":633,"context_line":"            return"},{"line_number":634,"context_line":"        for i in security_group_rules:"},{"line_number":635,"context_line":"            found_self \u003d False"},{"line_number":636,"context_line":"            for j in security_group_rules:"}],"source_content_type":"text/x-python","patch_set":2,"id":"dfd5e7cf_511374b9","line":633,"in_reply_to":"dfd5e7cf_d172047d","updated":"2019-01-04 23:07:36.000000000","message":":O\u003d\n\nJust that adding a config option to control this is ugly.  I\u0027m assuming it\u0027s the \"I know what I\u0027m doing and won\u0027t add duplicates\" hack.\n\nSo this path looks like it\u0027s the bulk path, was that just the one you\u0027ve been testing so is the pain point?  I can\u0027t say I\u0027ve dug that deep late on a Friday :)","commit_id":"fb5d86232b91b75a7de528e4671ad92c8a4184bb"},{"author":{"_account_id":10980,"name":"Doug Wiegley","email":"dougwig@parkside.io","username":"dougw"},"change_message_id":"b07e2cbee8d9f460dd76837d60964a2f4a067885","unresolved":false,"context_lines":[{"line_number":630,"context_line":""},{"line_number":631,"context_line":"    def _check_for_duplicate_rules(self, context, security_group_rules):"},{"line_number":632,"context_line":"        if not cfg.CONF.security_group_duplicate_detection:"},{"line_number":633,"context_line":"            return"},{"line_number":634,"context_line":"        for i in security_group_rules:"},{"line_number":635,"context_line":"            found_self \u003d False"},{"line_number":636,"context_line":"            for j in security_group_rules:"}],"source_content_type":"text/x-python","patch_set":2,"id":"dfd5e7cf_d172047d","line":633,"in_reply_to":"ffd0ebdf_69f8e2c3","updated":"2019-01-04 22:45:22.000000000","message":"I need more than \"Yuck\" to work with.  :-)\n\nAnything that loads all the rules on each append is an absolute killer.","commit_id":"fb5d86232b91b75a7de528e4671ad92c8a4184bb"},{"author":{"_account_id":1131,"name":"Brian Haley","email":"haleyb.dev@gmail.com","username":"brian-haley"},"change_message_id":"b6f4213848e1e6f5f33daf52667601eac1e77562","unresolved":false,"context_lines":[{"line_number":592,"context_line":"            res[\u0027protocol\u0027] \u003d self._get_ip_proto_name_and_num(value)"},{"line_number":593,"context_line":"        return res"},{"line_number":594,"context_line":""},{"line_number":595,"context_line":"    def _rule_to_key(self, rule):"},{"line_number":596,"context_line":"        comparison_keys \u003d ["},{"line_number":597,"context_line":"            \u0027direction\u0027,"},{"line_number":598,"context_line":"            \u0027ethertype\u0027,"}],"source_content_type":"text/x-python","patch_set":9,"id":"dfd5e7cf_d31bd670","line":595,"updated":"2019-01-07 14:53:04.000000000","message":"nit: can be @staticmethod, but then original wasn\u0027t either","commit_id":"932d9527fcf236cec73a70525597f7bff6298335"},{"author":{"_account_id":10980,"name":"Doug Wiegley","email":"dougwig@parkside.io","username":"dougw"},"change_message_id":"e1dbf5bccb5705972f089791ab1fa4288a6dfae6","unresolved":false,"context_lines":[{"line_number":592,"context_line":"            res[\u0027protocol\u0027] \u003d self._get_ip_proto_name_and_num(value)"},{"line_number":593,"context_line":"        return res"},{"line_number":594,"context_line":""},{"line_number":595,"context_line":"    def _rule_to_key(self, rule):"},{"line_number":596,"context_line":"        comparison_keys \u003d ["},{"line_number":597,"context_line":"            \u0027direction\u0027,"},{"line_number":598,"context_line":"            \u0027ethertype\u0027,"}],"source_content_type":"text/x-python","patch_set":9,"id":"dfd5e7cf_61248b0e","line":595,"in_reply_to":"dfd5e7cf_d31bd670","updated":"2019-01-07 19:32:52.000000000","message":"Done","commit_id":"932d9527fcf236cec73a70525597f7bff6298335"},{"author":{"_account_id":1131,"name":"Brian Haley","email":"haleyb.dev@gmail.com","username":"brian-haley"},"change_message_id":"b6f4213848e1e6f5f33daf52667601eac1e77562","unresolved":false,"context_lines":[{"line_number":616,"context_line":""},{"line_number":617,"context_line":"        if not skip_db_check:"},{"line_number":618,"context_line":"            for i in security_group_rules:"},{"line_number":619,"context_line":"                self._check_for_duplicate_rules_in_db(context, i)"},{"line_number":620,"context_line":""},{"line_number":621,"context_line":"    def _check_for_duplicate_rules_in_db(self, context, security_group_rule):"},{"line_number":622,"context_line":"        # Check in database if rule exists"}],"source_content_type":"text/x-python","patch_set":9,"id":"dfd5e7cf_73454a17","line":619,"updated":"2019-01-07 14:53:04.000000000","message":"So I think I finally understand the reasoning for the two checks now that you\u0027ve updated the review, maybe it was already obvious.  We have to both check if one of the rules is a duplicate of an existing rule, and if the POST has a duplicate in itself, since that would create a duplicate.\n\nLooks like you\u0027re getting around that by grouping them all together, so it might just be we tweak the error raised on L614 since it has POST in it\u0027s name?\n\nAlso, there is only only one caller of _check_for_duplicate_rules so this code can actually go away.","commit_id":"932d9527fcf236cec73a70525597f7bff6298335"},{"author":{"_account_id":10980,"name":"Doug Wiegley","email":"dougwig@parkside.io","username":"dougw"},"change_message_id":"e1dbf5bccb5705972f089791ab1fa4288a6dfae6","unresolved":false,"context_lines":[{"line_number":616,"context_line":""},{"line_number":617,"context_line":"        if not skip_db_check:"},{"line_number":618,"context_line":"            for i in security_group_rules:"},{"line_number":619,"context_line":"                self._check_for_duplicate_rules_in_db(context, i)"},{"line_number":620,"context_line":""},{"line_number":621,"context_line":"    def _check_for_duplicate_rules_in_db(self, context, security_group_rule):"},{"line_number":622,"context_line":"        # Check in database if rule exists"}],"source_content_type":"text/x-python","patch_set":9,"id":"dfd5e7cf_2bc2e234","line":619,"in_reply_to":"dfd5e7cf_2b18a21b","updated":"2019-01-07 19:32:52.000000000","message":"Yeah, let me do that. It collapses a lot of stuff.","commit_id":"932d9527fcf236cec73a70525597f7bff6298335"},{"author":{"_account_id":10980,"name":"Doug Wiegley","email":"dougwig@parkside.io","username":"dougw"},"change_message_id":"5cc933293936067524feb2be7d8083c6e292731e","unresolved":false,"context_lines":[{"line_number":616,"context_line":""},{"line_number":617,"context_line":"        if not skip_db_check:"},{"line_number":618,"context_line":"            for i in security_group_rules:"},{"line_number":619,"context_line":"                self._check_for_duplicate_rules_in_db(context, i)"},{"line_number":620,"context_line":""},{"line_number":621,"context_line":"    def _check_for_duplicate_rules_in_db(self, context, security_group_rule):"},{"line_number":622,"context_line":"        # Check in database if rule exists"}],"source_content_type":"text/x-python","patch_set":9,"id":"dfd5e7cf_8b274e65","line":619,"in_reply_to":"dfd5e7cf_73454a17","updated":"2019-01-07 17:11:22.000000000","message":"It\u0027s called twice, isn\u0027t it?  Once in the batch case, and once (If validate) in the single case.  The single case still needs to make the db call, unless we do a similar pre-load there.","commit_id":"932d9527fcf236cec73a70525597f7bff6298335"},{"author":{"_account_id":10980,"name":"Doug Wiegley","email":"dougwig@parkside.io","username":"dougw"},"change_message_id":"e1dbf5bccb5705972f089791ab1fa4288a6dfae6","unresolved":false,"context_lines":[{"line_number":616,"context_line":""},{"line_number":617,"context_line":"        if not skip_db_check:"},{"line_number":618,"context_line":"            for i in security_group_rules:"},{"line_number":619,"context_line":"                self._check_for_duplicate_rules_in_db(context, i)"},{"line_number":620,"context_line":""},{"line_number":621,"context_line":"    def _check_for_duplicate_rules_in_db(self, context, security_group_rule):"},{"line_number":622,"context_line":"        # Check in database if rule exists"}],"source_content_type":"text/x-python","patch_set":9,"id":"dfd5e7cf_a187d3d7","line":619,"in_reply_to":"dfd5e7cf_73454a17","updated":"2019-01-07 19:32:52.000000000","message":"Was able to keep the original errors in both.  Also found a bug in how we did the comparison check in the new rule case compared to the db case, and fixed that, too.","commit_id":"932d9527fcf236cec73a70525597f7bff6298335"},{"author":{"_account_id":10980,"name":"Doug Wiegley","email":"dougwig@parkside.io","username":"dougw"},"change_message_id":"e1dbf5bccb5705972f089791ab1fa4288a6dfae6","unresolved":false,"context_lines":[{"line_number":616,"context_line":""},{"line_number":617,"context_line":"        if not skip_db_check:"},{"line_number":618,"context_line":"            for i in security_group_rules:"},{"line_number":619,"context_line":"                self._check_for_duplicate_rules_in_db(context, i)"},{"line_number":620,"context_line":""},{"line_number":621,"context_line":"    def _check_for_duplicate_rules_in_db(self, context, security_group_rule):"},{"line_number":622,"context_line":"        # Check in database if rule exists"}],"source_content_type":"text/x-python","patch_set":9,"id":"dfd5e7cf_211e83ba","line":619,"in_reply_to":"dfd5e7cf_8b274e65","updated":"2019-01-07 19:32:52.000000000","message":"Done","commit_id":"932d9527fcf236cec73a70525597f7bff6298335"},{"author":{"_account_id":10980,"name":"Doug Wiegley","email":"dougwig@parkside.io","username":"dougw"},"change_message_id":"6bb7a71868a97d272ff268d2a0208f3ab8524193","unresolved":false,"context_lines":[{"line_number":616,"context_line":""},{"line_number":617,"context_line":"        if not skip_db_check:"},{"line_number":618,"context_line":"            for i in security_group_rules:"},{"line_number":619,"context_line":"                self._check_for_duplicate_rules_in_db(context, i)"},{"line_number":620,"context_line":""},{"line_number":621,"context_line":"    def _check_for_duplicate_rules_in_db(self, context, security_group_rule):"},{"line_number":622,"context_line":"        # Check in database if rule exists"}],"source_content_type":"text/x-python","patch_set":9,"id":"dfd5e7cf_2b18a21b","line":619,"in_reply_to":"dfd5e7cf_8b274e65","updated":"2019-01-07 17:12:04.000000000","message":"Let me reprase: unless you\u0027d like me to add a similar pre-load in the non-bulk case, too.","commit_id":"932d9527fcf236cec73a70525597f7bff6298335"},{"author":{"_account_id":1131,"name":"Brian Haley","email":"haleyb.dev@gmail.com","username":"brian-haley"},"change_message_id":"c6de220ec49a9eba14dd8a62498580c2a5982e4a","unresolved":false,"context_lines":[{"line_number":592,"context_line":"    def _rule_to_key(self, rule):"},{"line_number":593,"context_line":"        def _normalize_rule_value(key, value):"},{"line_number":594,"context_line":"            if key \u003d\u003d \u0027remote_ip_prefix\u0027:"},{"line_number":595,"context_line":"                all_address \u003d [\u00270.0.0.0/0\u0027, \u0027::/0\u0027, None]"},{"line_number":596,"context_line":"                if value in all_address:"},{"line_number":597,"context_line":"                    return \u0027+\u0027"},{"line_number":598,"context_line":"            elif key \u003d\u003d \u0027protocol\u0027:"}],"source_content_type":"text/x-python","patch_set":12,"id":"dfd5e7cf_8d475d1e","line":595,"updated":"2019-01-08 01:49:44.000000000","message":"There\u0027s constants for these \"any\" addresses, but we can fix it up later - constants.IPv4_ANY/IPv6_ANY - I can change it later since it\u0027s the same in _validate_duplicate_ip_prefix()","commit_id":"0a815dee6157db56ec68240c3aeecacea1433b04"},{"author":{"_account_id":10980,"name":"Doug Wiegley","email":"dougwig@parkside.io","username":"dougw"},"change_message_id":"a3039d2fc57bdf5a54bddcf5d82fc69d0ad5c5c0","unresolved":false,"context_lines":[{"line_number":592,"context_line":"    def _rule_to_key(self, rule):"},{"line_number":593,"context_line":"        def _normalize_rule_value(key, value):"},{"line_number":594,"context_line":"            if key \u003d\u003d \u0027remote_ip_prefix\u0027:"},{"line_number":595,"context_line":"                all_address \u003d [\u00270.0.0.0/0\u0027, \u0027::/0\u0027, None]"},{"line_number":596,"context_line":"                if value in all_address:"},{"line_number":597,"context_line":"                    return \u0027+\u0027"},{"line_number":598,"context_line":"            elif key \u003d\u003d \u0027protocol\u0027:"}],"source_content_type":"text/x-python","patch_set":12,"id":"dfd5e7cf_d0f73ef5","line":595,"in_reply_to":"dfd5e7cf_8d475d1e","updated":"2019-01-08 03:13:46.000000000","message":"Ok, I was copying the previous code. I can either fix them, or I prefer the idea of fixing the entire file as an entire patch (I\u0027m happy to submit that after, too.)","commit_id":"0a815dee6157db56ec68240c3aeecacea1433b04"},{"author":{"_account_id":1131,"name":"Brian Haley","email":"haleyb.dev@gmail.com","username":"brian-haley"},"change_message_id":"c6de220ec49a9eba14dd8a62498580c2a5982e4a","unresolved":false,"context_lines":[{"line_number":596,"context_line":"                if value in all_address:"},{"line_number":597,"context_line":"                    return \u0027+\u0027"},{"line_number":598,"context_line":"            elif key \u003d\u003d \u0027protocol\u0027:"},{"line_number":599,"context_line":"                return str(self._get_ip_proto_name_and_num(value))"},{"line_number":600,"context_line":"            elif value is None:"},{"line_number":601,"context_line":"                return \u0027+\u0027"},{"line_number":602,"context_line":"            return str(value)"}],"source_content_type":"text/x-python","patch_set":12,"id":"dfd5e7cf_8d1e7d0a","line":599,"updated":"2019-01-08 01:49:44.000000000","message":"if protocol is None this will return \u0027None\u0027, I\u0027m assuming that\u0027s correct?","commit_id":"0a815dee6157db56ec68240c3aeecacea1433b04"},{"author":{"_account_id":10980,"name":"Doug Wiegley","email":"dougwig@parkside.io","username":"dougw"},"change_message_id":"a3039d2fc57bdf5a54bddcf5d82fc69d0ad5c5c0","unresolved":false,"context_lines":[{"line_number":596,"context_line":"                if value in all_address:"},{"line_number":597,"context_line":"                    return \u0027+\u0027"},{"line_number":598,"context_line":"            elif key \u003d\u003d \u0027protocol\u0027:"},{"line_number":599,"context_line":"                return str(self._get_ip_proto_name_and_num(value))"},{"line_number":600,"context_line":"            elif value is None:"},{"line_number":601,"context_line":"                return \u0027+\u0027"},{"line_number":602,"context_line":"            return str(value)"}],"source_content_type":"text/x-python","patch_set":12,"id":"dfd5e7cf_90d9b684","line":599,"in_reply_to":"dfd5e7cf_8d1e7d0a","updated":"2019-01-08 03:13:46.000000000","message":"It is, but let me make it explicit and consistent.","commit_id":"0a815dee6157db56ec68240c3aeecacea1433b04"},{"author":{"_account_id":1131,"name":"Brian Haley","email":"haleyb.dev@gmail.com","username":"brian-haley"},"change_message_id":"c6de220ec49a9eba14dd8a62498580c2a5982e4a","unresolved":false,"context_lines":[{"line_number":630,"context_line":"        # when added."},{"line_number":631,"context_line":"        sg \u003d self.get_security_group(context, security_group_id)"},{"line_number":632,"context_line":"        if sg:"},{"line_number":633,"context_line":"            db_rules \u003d sg[\u0027security_group_rules\u0027]"},{"line_number":634,"context_line":"        else:"},{"line_number":635,"context_line":"            db_rules \u003d []"},{"line_number":636,"context_line":"        for i in db_rules:"}],"source_content_type":"text/x-python","patch_set":12,"id":"dfd5e7cf_6de6b919","line":633,"updated":"2019-01-08 01:49:44.000000000","message":"for could go here, potato/potatoe","commit_id":"0a815dee6157db56ec68240c3aeecacea1433b04"},{"author":{"_account_id":10980,"name":"Doug Wiegley","email":"dougwig@parkside.io","username":"dougw"},"change_message_id":"a3039d2fc57bdf5a54bddcf5d82fc69d0ad5c5c0","unresolved":false,"context_lines":[{"line_number":630,"context_line":"        # when added."},{"line_number":631,"context_line":"        sg \u003d self.get_security_group(context, security_group_id)"},{"line_number":632,"context_line":"        if sg:"},{"line_number":633,"context_line":"            db_rules \u003d sg[\u0027security_group_rules\u0027]"},{"line_number":634,"context_line":"        else:"},{"line_number":635,"context_line":"            db_rules \u003d []"},{"line_number":636,"context_line":"        for i in db_rules:"}],"source_content_type":"text/x-python","patch_set":12,"id":"dfd5e7cf_30e6ca46","line":633,"in_reply_to":"dfd5e7cf_6de6b919","updated":"2019-01-08 03:13:46.000000000","message":"Agree, I like that better. Artifact of shuffling, will move.","commit_id":"0a815dee6157db56ec68240c3aeecacea1433b04"},{"author":{"_account_id":16688,"name":"Rodolfo Alonso","email":"ralonsoh@redhat.com","username":"rodolfo-alonso-hernandez"},"change_message_id":"39f5c898a39ba43731a6fe1145e491d8162f4928","unresolved":false,"context_lines":[{"line_number":544,"context_line":"        sg \u003d self.get_security_group(context, security_group_id,"},{"line_number":545,"context_line":"                                     tenant_id\u003drule[\u0027tenant_id\u0027],"},{"line_number":546,"context_line":"                                     shallow\u003dTrue)"},{"line_number":547,"context_line":"        if not sg:"},{"line_number":548,"context_line":"            raise ext_sg.SecurityGroupNotFound(id\u003dsecurity_group_id)"},{"line_number":549,"context_line":""},{"line_number":550,"context_line":"        return security_group_id"}],"source_content_type":"text/x-python","patch_set":13,"id":"dfd5e7cf_6ca8dbda","line":547,"updated":"2019-01-09 10:11:06.000000000","message":"This check is not needed, \"_get_security_group\" (called from \"get_security_group\") will check it for you.","commit_id":"97625595ea3acd4626153f844cfefd2417f9a025"},{"author":{"_account_id":10980,"name":"Doug Wiegley","email":"dougwig@parkside.io","username":"dougw"},"change_message_id":"e1955b79aa9999b3551954cce5e24ab173a5ef6a","unresolved":false,"context_lines":[{"line_number":544,"context_line":"        sg \u003d self.get_security_group(context, security_group_id,"},{"line_number":545,"context_line":"                                     tenant_id\u003drule[\u0027tenant_id\u0027],"},{"line_number":546,"context_line":"                                     shallow\u003dTrue)"},{"line_number":547,"context_line":"        if not sg:"},{"line_number":548,"context_line":"            raise ext_sg.SecurityGroupNotFound(id\u003dsecurity_group_id)"},{"line_number":549,"context_line":""},{"line_number":550,"context_line":"        return security_group_id"}],"source_content_type":"text/x-python","patch_set":13,"id":"dfd5e7cf_b58a8318","line":547,"in_reply_to":"dfd5e7cf_6ca8dbda","updated":"2019-01-09 15:46:26.000000000","message":"Done","commit_id":"97625595ea3acd4626153f844cfefd2417f9a025"},{"author":{"_account_id":16688,"name":"Rodolfo Alonso","email":"ralonsoh@redhat.com","username":"rodolfo-alonso-hernandez"},"change_message_id":"39f5c898a39ba43731a6fe1145e491d8162f4928","unresolved":false,"context_lines":[{"line_number":622,"context_line":"            k \u003d self._rule_to_key(i[\u0027security_group_rule\u0027])"},{"line_number":623,"context_line":"            if k in h:"},{"line_number":624,"context_line":"                raise ext_sg.DuplicateSecurityGroupRuleInPost(rule\u003di)"},{"line_number":625,"context_line":"            h[k] \u003d True"},{"line_number":626,"context_line":""},{"line_number":627,"context_line":"        # Now, let\u0027s make sure none of the new rules conflict with"},{"line_number":628,"context_line":"        # existing rules; note that we do *not* store the db rules"}],"source_content_type":"text/x-python","patch_set":13,"id":"dfd5e7cf_c907791c","line":625,"range":{"start_line":625,"start_character":12,"end_line":625,"end_character":23},"updated":"2019-01-09 10:11:06.000000000","message":"nit: maybe here you can use a set, instead of a dict","commit_id":"97625595ea3acd4626153f844cfefd2417f9a025"},{"author":{"_account_id":10980,"name":"Doug Wiegley","email":"dougwig@parkside.io","username":"dougw"},"change_message_id":"e1955b79aa9999b3551954cce5e24ab173a5ef6a","unresolved":false,"context_lines":[{"line_number":622,"context_line":"            k \u003d self._rule_to_key(i[\u0027security_group_rule\u0027])"},{"line_number":623,"context_line":"            if k in h:"},{"line_number":624,"context_line":"                raise ext_sg.DuplicateSecurityGroupRuleInPost(rule\u003di)"},{"line_number":625,"context_line":"            h[k] \u003d True"},{"line_number":626,"context_line":""},{"line_number":627,"context_line":"        # Now, let\u0027s make sure none of the new rules conflict with"},{"line_number":628,"context_line":"        # existing rules; note that we do *not* store the db rules"}],"source_content_type":"text/x-python","patch_set":13,"id":"dfd5e7cf_5579d72c","line":625,"range":{"start_line":625,"start_character":12,"end_line":625,"end_character":23},"in_reply_to":"dfd5e7cf_c907791c","updated":"2019-01-09 15:46:26.000000000","message":"Ooh, this almost boiled down to a sexy comprehension, until I realized we needed the duplicate rule to raise the exception. So, switched, but less sexily than I\u0027d have liked.  :)","commit_id":"97625595ea3acd4626153f844cfefd2417f9a025"},{"author":{"_account_id":9531,"name":"liuyulong","display_name":"LIU Yulong","email":"i@liuyulong.me","username":"LIU-Yulong"},"change_message_id":"1abf1c48e8f4ca999c4f60982e4b51a1a0e9c80a","unresolved":false,"context_lines":[{"line_number":632,"context_line":"        if sg:"},{"line_number":633,"context_line":"            for i in sg[\u0027security_group_rules\u0027]:"},{"line_number":634,"context_line":"                k \u003d self._rule_to_key(i)"},{"line_number":635,"context_line":"                if k in h:"},{"line_number":636,"context_line":"                    raise ext_sg.SecurityGroupRuleExists(rule_id\u003di.get(\u0027id\u0027))"},{"line_number":637,"context_line":""},{"line_number":638,"context_line":"    def _validate_ip_prefix(self, rule):"}],"source_content_type":"text/x-python","patch_set":13,"id":"dfd5e7cf_2a540aca","line":635,"range":{"start_line":635,"start_character":16,"end_line":635,"end_character":26},"updated":"2019-01-08 14:48:56.000000000","message":"What happened if security group rule create API request is sent to two separated neutron-servers? This hash dict does not share among the neutron servers, right? And there is no unique constrain on the table:\nhttps://github.com/openstack/neutron/blob/master/neutron/db/models/securitygroup.py#L68-L97\nWe will have same rule add to DB? Correct me if I\u0027m wrong. Thanks.","commit_id":"97625595ea3acd4626153f844cfefd2417f9a025"},{"author":{"_account_id":10980,"name":"Doug Wiegley","email":"dougwig@parkside.io","username":"dougw"},"change_message_id":"4051bb22ea51c0a098847af3987bc5d3edddc571","unresolved":false,"context_lines":[{"line_number":632,"context_line":"        if sg:"},{"line_number":633,"context_line":"            for i in sg[\u0027security_group_rules\u0027]:"},{"line_number":634,"context_line":"                k \u003d self._rule_to_key(i)"},{"line_number":635,"context_line":"                if k in h:"},{"line_number":636,"context_line":"                    raise ext_sg.SecurityGroupRuleExists(rule_id\u003di.get(\u0027id\u0027))"},{"line_number":637,"context_line":""},{"line_number":638,"context_line":"    def _validate_ip_prefix(self, rule):"}],"source_content_type":"text/x-python","patch_set":13,"id":"dfd5e7cf_e5895526","line":635,"range":{"start_line":635,"start_character":16,"end_line":635,"end_character":26},"in_reply_to":"dfd5e7cf_2a540aca","updated":"2019-01-08 15:26:25.000000000","message":"Hi, and thanks for reviewing! This check all happens inside the write transaction, just like the previous for loops, so it\u0027ll stay consistent across processes.","commit_id":"97625595ea3acd4626153f844cfefd2417f9a025"},{"author":{"_account_id":27654,"name":"Hongbin Lu","email":"kira034@163.com","username":"hongbin.lu"},"change_message_id":"42033fb0f64e17cc0e9235b6fa15915c72ec0c0b","unresolved":false,"context_lines":[{"line_number":527,"context_line":"        if remote_group_id:"},{"line_number":528,"context_line":"            self.get_security_group(context, remote_group_id,"},{"line_number":529,"context_line":"                                    tenant_id\u003drule[\u0027tenant_id\u0027],"},{"line_number":530,"context_line":"                                    fields\u003d[\u0027id\u0027])"},{"line_number":531,"context_line":""},{"line_number":532,"context_line":"        security_group_id \u003d rule[\u0027security_group_id\u0027]"},{"line_number":533,"context_line":""}],"source_content_type":"text/x-python","patch_set":15,"id":"dfd5e7cf_77f3ba74","line":530,"range":{"start_line":530,"start_character":36,"end_line":530,"end_character":49},"updated":"2019-01-09 21:53:59.000000000","message":"Neutron OVO has optimization for this (declare: I am the author of the code): you can use \"sg_obj.SecurityGroup.get_values(context, \u0027id\u0027, tenant_id\u003drule[\u0027tenant_id\u0027], ...)\" instead. This will load single field (i.e. \u0027id\u0027) instead of the full ORM record.\n\nSee: https://review.openstack.org/#/c/610182/","commit_id":"2d95136999908fc9423512505a42df6e45c194af"},{"author":{"_account_id":10980,"name":"Doug Wiegley","email":"dougwig@parkside.io","username":"dougw"},"change_message_id":"047f723b91618ba2ba83046d9d97a38ac5cf8716","unresolved":false,"context_lines":[{"line_number":527,"context_line":"        if remote_group_id:"},{"line_number":528,"context_line":"            self.get_security_group(context, remote_group_id,"},{"line_number":529,"context_line":"                                    tenant_id\u003drule[\u0027tenant_id\u0027],"},{"line_number":530,"context_line":"                                    fields\u003d[\u0027id\u0027])"},{"line_number":531,"context_line":""},{"line_number":532,"context_line":"        security_group_id \u003d rule[\u0027security_group_id\u0027]"},{"line_number":533,"context_line":""}],"source_content_type":"text/x-python","patch_set":15,"id":"bfdaf3ff_bb0db791","line":530,"range":{"start_line":530,"start_character":36,"end_line":530,"end_character":49},"in_reply_to":"bfdaf3ff_60047e18","updated":"2019-01-13 00:22:28.000000000","message":"The speed advantage comes from not fetching the child objects. Right now, if you call GET security-groups and just ask for id\u0027s and name\u0027s, we will load every security group, and every security group rule.  Neutron has always loaded every column, which isn\u0027t really a big deal, except when you\u0027re then continuing to load 10,000 more child objects for no reason.\n\nBut, as you mentioned, this patch (starting with patchset 16, if solely focused on rule appending, and the GET problem above has moved into the second patch.","commit_id":"2d95136999908fc9423512505a42df6e45c194af"},{"author":{"_account_id":10980,"name":"Doug Wiegley","email":"dougwig@parkside.io","username":"dougw"},"change_message_id":"e09dfd4b74a9684baeb80e026f3a13dcb6826347","unresolved":false,"context_lines":[{"line_number":527,"context_line":"        if remote_group_id:"},{"line_number":528,"context_line":"            self.get_security_group(context, remote_group_id,"},{"line_number":529,"context_line":"                                    tenant_id\u003drule[\u0027tenant_id\u0027],"},{"line_number":530,"context_line":"                                    fields\u003d[\u0027id\u0027])"},{"line_number":531,"context_line":""},{"line_number":532,"context_line":"        security_group_id \u003d rule[\u0027security_group_id\u0027]"},{"line_number":533,"context_line":""}],"source_content_type":"text/x-python","patch_set":15,"id":"bfdaf3ff_f3c84d75","line":530,"range":{"start_line":530,"start_character":36,"end_line":530,"end_character":49},"in_reply_to":"bfdaf3ff_b3bba5e3","updated":"2019-01-11 22:53:08.000000000","message":"Ok.  I\u0027ve moved that over to review 630401, and I\u0027ll keep this one as just the rule create/append fixes.  I\u0027ll get that other one cleaned up and tested, and we can continue that discussion. Thanks!","commit_id":"2d95136999908fc9423512505a42df6e45c194af"},{"author":{"_account_id":27654,"name":"Hongbin Lu","email":"kira034@163.com","username":"hongbin.lu"},"change_message_id":"ef3de50ac50929e48444416ca9817214e8511349","unresolved":false,"context_lines":[{"line_number":527,"context_line":"        if remote_group_id:"},{"line_number":528,"context_line":"            self.get_security_group(context, remote_group_id,"},{"line_number":529,"context_line":"                                    tenant_id\u003drule[\u0027tenant_id\u0027],"},{"line_number":530,"context_line":"                                    fields\u003d[\u0027id\u0027])"},{"line_number":531,"context_line":""},{"line_number":532,"context_line":"        security_group_id \u003d rule[\u0027security_group_id\u0027]"},{"line_number":533,"context_line":""}],"source_content_type":"text/x-python","patch_set":15,"id":"bfdaf3ff_b3bba5e3","line":530,"range":{"start_line":530,"start_character":36,"end_line":530,"end_character":49},"in_reply_to":"bfdaf3ff_d851a810","updated":"2019-01-11 22:50:15.000000000","message":"It sounds like #2 is undesirable as you mentioned. I am not sure exactly what is #3 but it sounds like a revert of the the OVO effort (https://blueprints.launchpad.net/neutron/+spec/adopt-oslo-versioned-objects-for-db). Personally, I would vote for #1","commit_id":"2d95136999908fc9423512505a42df6e45c194af"},{"author":{"_account_id":16688,"name":"Rodolfo Alonso","email":"ralonsoh@redhat.com","username":"rodolfo-alonso-hernandez"},"change_message_id":"f331eb14587a36cbfa6eeb76c4bd725f99a67fc9","unresolved":false,"context_lines":[{"line_number":527,"context_line":"        if remote_group_id:"},{"line_number":528,"context_line":"            self.get_security_group(context, remote_group_id,"},{"line_number":529,"context_line":"                                    tenant_id\u003drule[\u0027tenant_id\u0027],"},{"line_number":530,"context_line":"                                    fields\u003d[\u0027id\u0027])"},{"line_number":531,"context_line":""},{"line_number":532,"context_line":"        security_group_id \u003d rule[\u0027security_group_id\u0027]"},{"line_number":533,"context_line":""}],"source_content_type":"text/x-python","patch_set":15,"id":"bfdaf3ff_60047e18","line":530,"range":{"start_line":530,"start_character":36,"end_line":530,"end_character":49},"in_reply_to":"bfdaf3ff_f3c84d75","updated":"2019-01-12 21:04:55.000000000","message":"Please, correct me if I\u0027m wrong. [1] is retrieving the full object from the DB and then filtering the needed fields. There is no speed advantage on this.\n\nIMO, the way the current patch is implemented is OK for me (apart from splitting the patch in two, rule creating and sg index).\n\n[1] https://review.openstack.org/#/c/630401/1/neutron/objects/base.py","commit_id":"2d95136999908fc9423512505a42df6e45c194af"},{"author":{"_account_id":10980,"name":"Doug Wiegley","email":"dougwig@parkside.io","username":"dougw"},"change_message_id":"fe7c38730588f17b872d8c6a0614164bd42832c5","unresolved":false,"context_lines":[{"line_number":527,"context_line":"        if remote_group_id:"},{"line_number":528,"context_line":"            self.get_security_group(context, remote_group_id,"},{"line_number":529,"context_line":"                                    tenant_id\u003drule[\u0027tenant_id\u0027],"},{"line_number":530,"context_line":"                                    fields\u003d[\u0027id\u0027])"},{"line_number":531,"context_line":""},{"line_number":532,"context_line":"        security_group_id \u003d rule[\u0027security_group_id\u0027]"},{"line_number":533,"context_line":""}],"source_content_type":"text/x-python","patch_set":15,"id":"dfd5e7cf_1d460d7e","line":530,"range":{"start_line":530,"start_character":36,"end_line":530,"end_character":49},"in_reply_to":"dfd5e7cf_3a2517ac","updated":"2019-01-10 00:37:12.000000000","message":"I\u0027ll try this one, thanks!","commit_id":"2d95136999908fc9423512505a42df6e45c194af"},{"author":{"_account_id":10980,"name":"Doug Wiegley","email":"dougwig@parkside.io","username":"dougw"},"change_message_id":"7b3f1bcf20ace33c2e5b878060e30578b6d40a3a","unresolved":false,"context_lines":[{"line_number":527,"context_line":"        if remote_group_id:"},{"line_number":528,"context_line":"            self.get_security_group(context, remote_group_id,"},{"line_number":529,"context_line":"                                    tenant_id\u003drule[\u0027tenant_id\u0027],"},{"line_number":530,"context_line":"                                    fields\u003d[\u0027id\u0027])"},{"line_number":531,"context_line":""},{"line_number":532,"context_line":"        security_group_id \u003d rule[\u0027security_group_id\u0027]"},{"line_number":533,"context_line":""}],"source_content_type":"text/x-python","patch_set":15,"id":"dfd5e7cf_b14fea32","line":530,"range":{"start_line":530,"start_character":36,"end_line":530,"end_character":49},"in_reply_to":"dfd5e7cf_5d31f50a","updated":"2019-01-10 06:19:50.000000000","message":"Ok, so what I could do, is change get_values() to accept a single string (as today), or a list of strings, and enhance it to fetch multiple columns at once. Then I can pass in the user defined fields, and it\u0027ll be a direct db fetch to return it. That\u0027ll end up de-ovo\u0027ing most of this file, though, for reads. And it\u0027ll be fast. If that\u0027s acceptable, I can switch to that...","commit_id":"2d95136999908fc9423512505a42df6e45c194af"},{"author":{"_account_id":10980,"name":"Doug Wiegley","email":"dougwig@parkside.io","username":"dougw"},"change_message_id":"fe7c38730588f17b872d8c6a0614164bd42832c5","unresolved":false,"context_lines":[{"line_number":527,"context_line":"        if remote_group_id:"},{"line_number":528,"context_line":"            self.get_security_group(context, remote_group_id,"},{"line_number":529,"context_line":"                                    tenant_id\u003drule[\u0027tenant_id\u0027],"},{"line_number":530,"context_line":"                                    fields\u003d[\u0027id\u0027])"},{"line_number":531,"context_line":""},{"line_number":532,"context_line":"        security_group_id \u003d rule[\u0027security_group_id\u0027]"},{"line_number":533,"context_line":""}],"source_content_type":"text/x-python","patch_set":15,"id":"dfd5e7cf_5d31f50a","line":530,"range":{"start_line":530,"start_character":36,"end_line":530,"end_character":49},"in_reply_to":"dfd5e7cf_77f3ba74","updated":"2019-01-10 00:37:12.000000000","message":"Both the list and get api operations also need this functionality, and they need to fetch multiple values, based on user input. So, either I modify get_values to return multiple columns, at which point it\u0027s just a thin wrapper around my very first patchset of going to sqla db directly, or we need something more ORM friendly, which is what I tried to do here. I\u0027d like some guidance from the team on which direction to take.","commit_id":"2d95136999908fc9423512505a42df6e45c194af"},{"author":{"_account_id":27654,"name":"Hongbin Lu","email":"kira034@163.com","username":"hongbin.lu"},"change_message_id":"dee21437d4740ca7422f27ff0fedc05ec8959740","unresolved":false,"context_lines":[{"line_number":527,"context_line":"        if remote_group_id:"},{"line_number":528,"context_line":"            self.get_security_group(context, remote_group_id,"},{"line_number":529,"context_line":"                                    tenant_id\u003drule[\u0027tenant_id\u0027],"},{"line_number":530,"context_line":"                                    fields\u003d[\u0027id\u0027])"},{"line_number":531,"context_line":""},{"line_number":532,"context_line":"        security_group_id \u003d rule[\u0027security_group_id\u0027]"},{"line_number":533,"context_line":""}],"source_content_type":"text/x-python","patch_set":15,"id":"dfd5e7cf_3a2517ac","line":530,"range":{"start_line":530,"start_character":36,"end_line":530,"end_character":49},"in_reply_to":"dfd5e7cf_77f3ba74","updated":"2019-01-09 22:02:39.000000000","message":"Perhaps there is an even simpler approach:\n\n    sg_obj.SecurityGroup.objects_exist(context, id\u003dremote_group_id, tenant_id\u003drule[\u0027tenant_id\u0027])","commit_id":"2d95136999908fc9423512505a42df6e45c194af"},{"author":{"_account_id":10980,"name":"Doug Wiegley","email":"dougwig@parkside.io","username":"dougw"},"change_message_id":"6171aef75f869297bbfd2e9cb1ae62d1cdb9f194","unresolved":false,"context_lines":[{"line_number":527,"context_line":"        if remote_group_id:"},{"line_number":528,"context_line":"            self.get_security_group(context, remote_group_id,"},{"line_number":529,"context_line":"                                    tenant_id\u003drule[\u0027tenant_id\u0027],"},{"line_number":530,"context_line":"                                    fields\u003d[\u0027id\u0027])"},{"line_number":531,"context_line":""},{"line_number":532,"context_line":"        security_group_id \u003d rule[\u0027security_group_id\u0027]"},{"line_number":533,"context_line":""}],"source_content_type":"text/x-python","patch_set":15,"id":"bfdaf3ff_d851a810","line":530,"range":{"start_line":530,"start_character":36,"end_line":530,"end_character":49},"in_reply_to":"dfd5e7cf_a56a3ab0","updated":"2019-01-11 21:08:23.000000000","message":"Ok, I modified get_values to accept/return multiple fields, and then added paging support. Then noticed that the two functions looked pretty similar at that point, so I combined them.  Then went and modified the securitygroups_db code to use get_values, which returned a list of dicts instead of objects. Changing those attribute references to dict references very quickly spiraled out of just that file, and all over neutron. I\u0027m guessing we don\u0027t want that.  get_values() seems useful now for places where you just want one piece of data, but not as a general purpose replacement for GET api calls.  Something like adding fields support to get_objects can handle that, like what is suggested in this patchset, or we have to change a lot.  It\u0027d be *vastly* simpler, and achieve the same result, to let those GET calls just go back to being the old sqla code.  So, what approach do we want to take?\n1. modify the objects ORM (what\u0027s in this patch now)\n2. modify get_values, or deal with the spiderweb mess\n3. go back to the old sqla code in securitygroups_db.py for reads\n\nThe reason that file is so heavily impacted is because a non-bounded object read pulls in all the security group rule objects as well. The other APIs don\u0027t have this child spewing issue.","commit_id":"2d95136999908fc9423512505a42df6e45c194af"},{"author":{"_account_id":27654,"name":"Hongbin Lu","email":"kira034@163.com","username":"hongbin.lu"},"change_message_id":"a0491f3ab681bcd2df80a9b48a35274e81bfd1bf","unresolved":false,"context_lines":[{"line_number":527,"context_line":"        if remote_group_id:"},{"line_number":528,"context_line":"            self.get_security_group(context, remote_group_id,"},{"line_number":529,"context_line":"                                    tenant_id\u003drule[\u0027tenant_id\u0027],"},{"line_number":530,"context_line":"                                    fields\u003d[\u0027id\u0027])"},{"line_number":531,"context_line":""},{"line_number":532,"context_line":"        security_group_id \u003d rule[\u0027security_group_id\u0027]"},{"line_number":533,"context_line":""}],"source_content_type":"text/x-python","patch_set":15,"id":"dfd5e7cf_a56a3ab0","line":530,"range":{"start_line":530,"start_character":36,"end_line":530,"end_character":49},"in_reply_to":"dfd5e7cf_b14fea32","updated":"2019-01-10 16:02:21.000000000","message":"Sounds good to me.","commit_id":"2d95136999908fc9423512505a42df6e45c194af"},{"author":{"_account_id":27654,"name":"Hongbin Lu","email":"kira034@163.com","username":"hongbin.lu"},"change_message_id":"892208acbce9dfb8b5b8f7e56db791c05bbda419","unresolved":false,"context_lines":[{"line_number":583,"context_line":"            if key \u003d\u003d \u0027remote_ip_prefix\u0027:"},{"line_number":584,"context_line":"                all_address \u003d [\u00270.0.0.0/0\u0027, \u0027::/0\u0027, None]"},{"line_number":585,"context_line":"                if value in all_address:"},{"line_number":586,"context_line":"                    return \u0027+\u0027"},{"line_number":587,"context_line":"            elif value is None:"},{"line_number":588,"context_line":"                return \u0027+\u0027"},{"line_number":589,"context_line":"            elif key \u003d\u003d \u0027protocol\u0027:"}],"source_content_type":"text/x-python","patch_set":16,"id":"bfdaf3ff_e4e12d2c","line":586,"range":{"start_line":586,"start_character":27,"end_line":586,"end_character":30},"updated":"2019-01-15 20:36:36.000000000","message":"nit: I think it is better to document what this special character is for. Either adding a short comment on top or define a variable with meaningful name should suffice.","commit_id":"fd42dca1235b9103415393f83736ffacbc4e59ce"},{"author":{"_account_id":10980,"name":"Doug Wiegley","email":"dougwig@parkside.io","username":"dougw"},"change_message_id":"c3e0a2ae3f6247604f5be018c7c3223fd8e7723b","unresolved":false,"context_lines":[{"line_number":583,"context_line":"            if key \u003d\u003d \u0027remote_ip_prefix\u0027:"},{"line_number":584,"context_line":"                all_address \u003d [\u00270.0.0.0/0\u0027, \u0027::/0\u0027, None]"},{"line_number":585,"context_line":"                if value in all_address:"},{"line_number":586,"context_line":"                    return \u0027+\u0027"},{"line_number":587,"context_line":"            elif value is None:"},{"line_number":588,"context_line":"                return \u0027+\u0027"},{"line_number":589,"context_line":"            elif key \u003d\u003d \u0027protocol\u0027:"}],"source_content_type":"text/x-python","patch_set":16,"id":"bfdaf3ff_b74835cc","line":586,"range":{"start_line":586,"start_character":27,"end_line":586,"end_character":30},"in_reply_to":"bfdaf3ff_772e4d44","updated":"2019-01-15 21:52:14.000000000","message":"Done","commit_id":"fd42dca1235b9103415393f83736ffacbc4e59ce"},{"author":{"_account_id":10980,"name":"Doug Wiegley","email":"dougwig@parkside.io","username":"dougw"},"change_message_id":"4ac5bfbe46fb7195359e4d70c75e132583f0bb6b","unresolved":false,"context_lines":[{"line_number":583,"context_line":"            if key \u003d\u003d \u0027remote_ip_prefix\u0027:"},{"line_number":584,"context_line":"                all_address \u003d [\u00270.0.0.0/0\u0027, \u0027::/0\u0027, None]"},{"line_number":585,"context_line":"                if value in all_address:"},{"line_number":586,"context_line":"                    return \u0027+\u0027"},{"line_number":587,"context_line":"            elif value is None:"},{"line_number":588,"context_line":"                return \u0027+\u0027"},{"line_number":589,"context_line":"            elif key \u003d\u003d \u0027protocol\u0027:"}],"source_content_type":"text/x-python","patch_set":16,"id":"bfdaf3ff_772e4d44","line":586,"range":{"start_line":586,"start_character":27,"end_line":586,"end_character":30},"in_reply_to":"bfdaf3ff_e4e12d2c","updated":"2019-01-15 21:40:36.000000000","message":"I\u0027ll make that a little clearer.  the + is just a no-op; fewer bytes than str(None)","commit_id":"fd42dca1235b9103415393f83736ffacbc4e59ce"},{"author":{"_account_id":27654,"name":"Hongbin Lu","email":"kira034@163.com","username":"hongbin.lu"},"change_message_id":"892208acbce9dfb8b5b8f7e56db791c05bbda419","unresolved":false,"context_lines":[{"line_number":585,"context_line":"                if value in all_address:"},{"line_number":586,"context_line":"                    return \u0027+\u0027"},{"line_number":587,"context_line":"            elif value is None:"},{"line_number":588,"context_line":"                return \u0027+\u0027"},{"line_number":589,"context_line":"            elif key \u003d\u003d \u0027protocol\u0027:"},{"line_number":590,"context_line":"                return str(self._get_ip_proto_name_and_num(value))"},{"line_number":591,"context_line":"            return str(value)"}],"source_content_type":"text/x-python","patch_set":16,"id":"bfdaf3ff_c43509dc","line":588,"range":{"start_line":588,"start_character":23,"end_line":588,"end_character":26},"updated":"2019-01-15 20:36:36.000000000","message":"IMHO, it is a bit confusing to reuse \u0027+\u0027 to denote both cases. Perhaps choosing another special character would be better.","commit_id":"fd42dca1235b9103415393f83736ffacbc4e59ce"},{"author":{"_account_id":10980,"name":"Doug Wiegley","email":"dougwig@parkside.io","username":"dougw"},"change_message_id":"c3e0a2ae3f6247604f5be018c7c3223fd8e7723b","unresolved":false,"context_lines":[{"line_number":585,"context_line":"                if value in all_address:"},{"line_number":586,"context_line":"                    return \u0027+\u0027"},{"line_number":587,"context_line":"            elif value is None:"},{"line_number":588,"context_line":"                return \u0027+\u0027"},{"line_number":589,"context_line":"            elif key \u003d\u003d \u0027protocol\u0027:"},{"line_number":590,"context_line":"                return str(self._get_ip_proto_name_and_num(value))"},{"line_number":591,"context_line":"            return str(value)"}],"source_content_type":"text/x-python","patch_set":16,"id":"bfdaf3ff_d74379ae","line":588,"range":{"start_line":588,"start_character":23,"end_line":588,"end_character":26},"in_reply_to":"bfdaf3ff_c43509dc","updated":"2019-01-15 21:52:14.000000000","message":"Done","commit_id":"fd42dca1235b9103415393f83736ffacbc4e59ce"},{"author":{"_account_id":27654,"name":"Hongbin Lu","email":"kira034@163.com","username":"hongbin.lu"},"change_message_id":"892208acbce9dfb8b5b8f7e56db791c05bbda419","unresolved":false,"context_lines":[{"line_number":590,"context_line":"                return str(self._get_ip_proto_name_and_num(value))"},{"line_number":591,"context_line":"            return str(value)"},{"line_number":592,"context_line":""},{"line_number":593,"context_line":"        comparison_keys \u003d ["},{"line_number":594,"context_line":"            \u0027direction\u0027,"},{"line_number":595,"context_line":"            \u0027ethertype\u0027,"},{"line_number":596,"context_line":"            \u0027port_range_max\u0027,"}],"source_content_type":"text/x-python","patch_set":16,"id":"bfdaf3ff_0413115e","line":593,"range":{"start_line":593,"start_character":8,"end_line":593,"end_character":23},"updated":"2019-01-15 20:36:36.000000000","message":"It looks the \u0027project_id\u0027 is not included. Is it intentional to exclude \u0027project_id\u0027 as a comparison key? Just want to make sure.","commit_id":"fd42dca1235b9103415393f83736ffacbc4e59ce"},{"author":{"_account_id":10980,"name":"Doug Wiegley","email":"dougwig@parkside.io","username":"dougw"},"change_message_id":"4ac5bfbe46fb7195359e4d70c75e132583f0bb6b","unresolved":false,"context_lines":[{"line_number":590,"context_line":"                return str(self._get_ip_proto_name_and_num(value))"},{"line_number":591,"context_line":"            return str(value)"},{"line_number":592,"context_line":""},{"line_number":593,"context_line":"        comparison_keys \u003d ["},{"line_number":594,"context_line":"            \u0027direction\u0027,"},{"line_number":595,"context_line":"            \u0027ethertype\u0027,"},{"line_number":596,"context_line":"            \u0027port_range_max\u0027,"}],"source_content_type":"text/x-python","patch_set":16,"id":"bfdaf3ff_3728c532","line":593,"range":{"start_line":593,"start_character":8,"end_line":593,"end_character":23},"in_reply_to":"bfdaf3ff_0413115e","updated":"2019-01-15 21:40:36.000000000","message":"Not needed, since it\u0027s verified above this function.","commit_id":"fd42dca1235b9103415393f83736ffacbc4e59ce"},{"author":{"_account_id":27654,"name":"Hongbin Lu","email":"kira034@163.com","username":"hongbin.lu"},"change_message_id":"892208acbce9dfb8b5b8f7e56db791c05bbda419","unresolved":false,"context_lines":[{"line_number":600,"context_line":"            \u0027remote_ip_prefix\u0027,"},{"line_number":601,"context_line":"            \u0027security_group_id\u0027"},{"line_number":602,"context_line":"        ]"},{"line_number":603,"context_line":"        return \u0027_\u0027.join([_normalize_rule_value(x, rule.get(x))"},{"line_number":604,"context_line":"                         for x in comparison_keys])"},{"line_number":605,"context_line":""},{"line_number":606,"context_line":"    def _check_for_duplicate_rules(self, context, security_group_id,"},{"line_number":607,"context_line":"                                   new_security_group_rules):"}],"source_content_type":"text/x-python","patch_set":16,"id":"bfdaf3ff_41410b0b","line":604,"range":{"start_line":603,"start_character":8,"end_line":604,"end_character":51},"updated":"2019-01-15 20:36:36.000000000","message":"I think this approach should work and good enough.\n\nJust want to point out an alternative approach, which is to convert the \"rule\" object to a frozenset (https://www.programiz.com/python-programming/methods/built-in/frozenset) that can also be used as a key.","commit_id":"fd42dca1235b9103415393f83736ffacbc4e59ce"},{"author":{"_account_id":10980,"name":"Doug Wiegley","email":"dougwig@parkside.io","username":"dougw"},"change_message_id":"4ac5bfbe46fb7195359e4d70c75e132583f0bb6b","unresolved":false,"context_lines":[{"line_number":600,"context_line":"            \u0027remote_ip_prefix\u0027,"},{"line_number":601,"context_line":"            \u0027security_group_id\u0027"},{"line_number":602,"context_line":"        ]"},{"line_number":603,"context_line":"        return \u0027_\u0027.join([_normalize_rule_value(x, rule.get(x))"},{"line_number":604,"context_line":"                         for x in comparison_keys])"},{"line_number":605,"context_line":""},{"line_number":606,"context_line":"    def _check_for_duplicate_rules(self, context, security_group_id,"},{"line_number":607,"context_line":"                                   new_security_group_rules):"}],"source_content_type":"text/x-python","patch_set":16,"id":"bfdaf3ff_f721dd4e","line":604,"range":{"start_line":603,"start_character":8,"end_line":604,"end_character":51},"in_reply_to":"bfdaf3ff_41410b0b","updated":"2019-01-15 21:40:36.000000000","message":"That\u0027s neat! I initially had something similar, but since some of the fields have to be normalized/transformed, that won\u0027t quite work in this case. But thank you!","commit_id":"fd42dca1235b9103415393f83736ffacbc4e59ce"},{"author":{"_account_id":27654,"name":"Hongbin Lu","email":"kira034@163.com","username":"hongbin.lu"},"change_message_id":"892208acbce9dfb8b5b8f7e56db791c05bbda419","unresolved":false,"context_lines":[{"line_number":606,"context_line":"    def _check_for_duplicate_rules(self, context, security_group_id,"},{"line_number":607,"context_line":"                                   new_security_group_rules):"},{"line_number":608,"context_line":"        # First up, check for any duplicates in the new rules."},{"line_number":609,"context_line":"        h \u003d set()"},{"line_number":610,"context_line":"        for i in new_security_group_rules:"},{"line_number":611,"context_line":"            k \u003d self._rule_to_key(i[\u0027security_group_rule\u0027])"},{"line_number":612,"context_line":"            if k in h:"}],"source_content_type":"text/x-python","patch_set":16,"id":"bfdaf3ff_0fcc8fba","line":609,"range":{"start_line":609,"start_character":8,"end_line":609,"end_character":9},"updated":"2019-01-15 20:36:36.000000000","message":"For me, it is a bit hard to read code with single letter variable... Perhaps choose a longer name.","commit_id":"fd42dca1235b9103415393f83736ffacbc4e59ce"},{"author":{"_account_id":10980,"name":"Doug Wiegley","email":"dougwig@parkside.io","username":"dougw"},"change_message_id":"4ac5bfbe46fb7195359e4d70c75e132583f0bb6b","unresolved":false,"context_lines":[{"line_number":606,"context_line":"    def _check_for_duplicate_rules(self, context, security_group_id,"},{"line_number":607,"context_line":"                                   new_security_group_rules):"},{"line_number":608,"context_line":"        # First up, check for any duplicates in the new rules."},{"line_number":609,"context_line":"        h \u003d set()"},{"line_number":610,"context_line":"        for i in new_security_group_rules:"},{"line_number":611,"context_line":"            k \u003d self._rule_to_key(i[\u0027security_group_rule\u0027])"},{"line_number":612,"context_line":"            if k in h:"}],"source_content_type":"text/x-python","patch_set":16,"id":"bfdaf3ff_171d810e","line":609,"range":{"start_line":609,"start_character":8,"end_line":609,"end_character":9},"in_reply_to":"bfdaf3ff_0fcc8fba","updated":"2019-01-15 21:40:36.000000000","message":"Ok","commit_id":"fd42dca1235b9103415393f83736ffacbc4e59ce"},{"author":{"_account_id":10980,"name":"Doug Wiegley","email":"dougwig@parkside.io","username":"dougw"},"change_message_id":"c3e0a2ae3f6247604f5be018c7c3223fd8e7723b","unresolved":false,"context_lines":[{"line_number":606,"context_line":"    def _check_for_duplicate_rules(self, context, security_group_id,"},{"line_number":607,"context_line":"                                   new_security_group_rules):"},{"line_number":608,"context_line":"        # First up, check for any duplicates in the new rules."},{"line_number":609,"context_line":"        h \u003d set()"},{"line_number":610,"context_line":"        for i in new_security_group_rules:"},{"line_number":611,"context_line":"            k \u003d self._rule_to_key(i[\u0027security_group_rule\u0027])"},{"line_number":612,"context_line":"            if k in h:"}],"source_content_type":"text/x-python","patch_set":16,"id":"bfdaf3ff_77140da1","line":609,"range":{"start_line":609,"start_character":8,"end_line":609,"end_character":9},"in_reply_to":"bfdaf3ff_171d810e","updated":"2019-01-15 21:52:14.000000000","message":"Done","commit_id":"fd42dca1235b9103415393f83736ffacbc4e59ce"},{"author":{"_account_id":9531,"name":"liuyulong","display_name":"LIU Yulong","email":"i@liuyulong.me","username":"LIU-Yulong"},"change_message_id":"1eb49cd379fd6ce128bc4b6cced461247d899cab","unresolved":false,"context_lines":[{"line_number":624,"context_line":"        if sg:"},{"line_number":625,"context_line":"            for i in sg[\u0027security_group_rules\u0027]:"},{"line_number":626,"context_line":"                rule_key \u003d self._rule_to_key(i)"},{"line_number":627,"context_line":"                if rule_key in new_rules_set:"},{"line_number":628,"context_line":"                    raise ext_sg.SecurityGroupRuleExists(rule_id\u003di.get(\u0027id\u0027))"},{"line_number":629,"context_line":""},{"line_number":630,"context_line":"    def _validate_ip_prefix(self, rule):"}],"source_content_type":"text/x-python","patch_set":17,"id":"bfdaf3ff_ca9e6b53","line":627,"range":{"start_line":627,"start_character":16,"end_line":627,"end_character":45},"updated":"2019-01-16 08:43:10.000000000","message":"Again this check here:\nI\u0027ve tested this patch with a simple script:\nhttp://paste.openstack.org/show/742711/ \nCreate 100 rules, and each rule will be send to multiple neutron-server twice.\n\nThen we have some basic results:\n(1) not 100 rules was created after running the script. 100% reproducable.\n(2) It still takes too long time to run, about 300s+ for 100 rules. Aslo obviously.","commit_id":"6ac5423e6527eed187c21cb845ecb911809947a8"},{"author":{"_account_id":10980,"name":"Doug Wiegley","email":"dougwig@parkside.io","username":"dougw"},"change_message_id":"d34cd955a8300e8a71284010ea90acd6f43d11c1","unresolved":false,"context_lines":[{"line_number":624,"context_line":"        if sg:"},{"line_number":625,"context_line":"            for i in sg[\u0027security_group_rules\u0027]:"},{"line_number":626,"context_line":"                rule_key \u003d self._rule_to_key(i)"},{"line_number":627,"context_line":"                if rule_key in new_rules_set:"},{"line_number":628,"context_line":"                    raise ext_sg.SecurityGroupRuleExists(rule_id\u003di.get(\u0027id\u0027))"},{"line_number":629,"context_line":""},{"line_number":630,"context_line":"    def _validate_ip_prefix(self, rule):"}],"source_content_type":"text/x-python","patch_set":17,"id":"bfdaf3ff_b5b0f844","line":627,"range":{"start_line":627,"start_character":16,"end_line":627,"end_character":45},"in_reply_to":"bfdaf3ff_7457005a","updated":"2019-01-18 20:57:27.000000000","message":"Hi Liu. Looking at your backtrace, it is indeed a db retry timeout. If your test only did 2 or 4 of the same security group id at once (times 100), instead of 200 of the same call against the same security group, I think the error wouldn\u0027t happen.\n\nIf we want to address the massively parallel case against a single security group, which I think it\u0027s an unlikely use case, can we address that in another patch? It might even need something neutron-wide.\n\nObviously something changed with the timing internally, that you\u0027re seeing this with this change, and not before, though I think the gains in the normal cases more than make up for it. Do you disagree?","commit_id":"6ac5423e6527eed187c21cb845ecb911809947a8"},{"author":{"_account_id":9531,"name":"liuyulong","display_name":"LIU Yulong","email":"i@liuyulong.me","username":"LIU-Yulong"},"change_message_id":"bbc1c42a5af9034d81c58bc689991464f069e92f","unresolved":false,"context_lines":[{"line_number":624,"context_line":"        if sg:"},{"line_number":625,"context_line":"            for i in sg[\u0027security_group_rules\u0027]:"},{"line_number":626,"context_line":"                rule_key \u003d self._rule_to_key(i)"},{"line_number":627,"context_line":"                if rule_key in new_rules_set:"},{"line_number":628,"context_line":"                    raise ext_sg.SecurityGroupRuleExists(rule_id\u003di.get(\u0027id\u0027))"},{"line_number":629,"context_line":""},{"line_number":630,"context_line":"    def _validate_ip_prefix(self, rule):"}],"source_content_type":"text/x-python","patch_set":17,"id":"bfdaf3ff_8c6dc605","line":627,"range":{"start_line":627,"start_character":16,"end_line":627,"end_character":45},"in_reply_to":"bfdaf3ff_8156015c","updated":"2019-01-17 11:31:50.000000000","message":"Hi, thanks for addressing my comment. IMHO, it may have regression of this patch mainly for these test results.\nStart two neutron-servers listen on port 9697 and 9698 (or setup two separated neutron-servers running on different hosts).\nThen everytime run that 100 rules create script, 100 rules will be created without this patch (before).\nBut with this patch (after), 100 rules were not fully created.\n\nAnd, for that \u0027run time\u0027, I did not mean the script execute time, sorry for the inaccurate.\nThe \u0027run time\u0027 is between script running time and last creating rule POST response status LOG. Such as:\n\"POST /v2.0/security-group-rules HTTP/1.1\" status: 409 (failed)\n\"POST /v2.0/security-group-rules HTTP/1.1\" status: 201 (succeed)\nOr with debug LOG level, it can be the first POST request log to last POST response log.\nOr record the time until no new rule was added. Yes, the \u0027run time\u0027 turns longer than before.\n\nSo, IMO, maybe you can double-check/confirm if such behavior is OK in such scenario testing with this fix.\n\nAnd since this patch is fixing the performace issue, and the current code is vary from the patch set 1-14, so is there any performance results after this fix? It still has the following improvement?\nhttps://bugs.launchpad.net/neutron/+bug/1810563/comments/3\nThanks.","commit_id":"6ac5423e6527eed187c21cb845ecb911809947a8"},{"author":{"_account_id":10980,"name":"Doug Wiegley","email":"dougwig@parkside.io","username":"dougw"},"change_message_id":"5a23a0531b64251c7e51fde4d5d27c69375e9661","unresolved":false,"context_lines":[{"line_number":624,"context_line":"        if sg:"},{"line_number":625,"context_line":"            for i in sg[\u0027security_group_rules\u0027]:"},{"line_number":626,"context_line":"                rule_key \u003d self._rule_to_key(i)"},{"line_number":627,"context_line":"                if rule_key in new_rules_set:"},{"line_number":628,"context_line":"                    raise ext_sg.SecurityGroupRuleExists(rule_id\u003di.get(\u0027id\u0027))"},{"line_number":629,"context_line":""},{"line_number":630,"context_line":"    def _validate_ip_prefix(self, rule):"}],"source_content_type":"text/x-python","patch_set":17,"id":"bfdaf3ff_7457005a","line":627,"range":{"start_line":627,"start_character":16,"end_line":627,"end_character":45},"in_reply_to":"bfdaf3ff_8c6dc605","updated":"2019-01-17 16:25:21.000000000","message":"Hi, and thanks for the further updates.  For my test runs, I modified your script with a \"wait\" at the end, so that the script runtime would be the time the last POST finished. So those times are the \"finish\" times. I also modified it so that I could see if I got back any returns other than success or conflict, and I didn\u0027t.  Do you have any exceptions in your server logs, or db retry errors?  I\u0027m trying to find out why you\u0027re seeing some of the requests failing.  Fewer than 100 is better than more; if it\u0027s just db retries timing out, I think that\u0027s ok. If it\u0027s something else, we need to figure out what it is.\n\nRe-running my earlier tests on this patchset, I get:\n\n1000 rule create, all at once:\nreal\t1m24.909s\n\n2000 rule create, in batches of 50:\nLoop test took 155 seconds\n\nThose results are similar to the first patchset. Slower than appending some db rows should be, but on par with Liberty again, and usable.","commit_id":"6ac5423e6527eed187c21cb845ecb911809947a8"},{"author":{"_account_id":10980,"name":"Doug Wiegley","email":"dougwig@parkside.io","username":"dougw"},"change_message_id":"113186288fd0fd768667acc9385bd48587f75e34","unresolved":false,"context_lines":[{"line_number":624,"context_line":"        if sg:"},{"line_number":625,"context_line":"            for i in sg[\u0027security_group_rules\u0027]:"},{"line_number":626,"context_line":"                rule_key \u003d self._rule_to_key(i)"},{"line_number":627,"context_line":"                if rule_key in new_rules_set:"},{"line_number":628,"context_line":"                    raise ext_sg.SecurityGroupRuleExists(rule_id\u003di.get(\u0027id\u0027))"},{"line_number":629,"context_line":""},{"line_number":630,"context_line":"    def _validate_ip_prefix(self, rule):"}],"source_content_type":"text/x-python","patch_set":17,"id":"bfdaf3ff_8156015c","line":627,"range":{"start_line":627,"start_character":16,"end_line":627,"end_character":45},"in_reply_to":"bfdaf3ff_ab3c46bf","updated":"2019-01-16 16:07:55.000000000","message":"I\u0027d be very interested in seeing your before/after results.  Using your script, I get:\n\nBEFORE change:\n\nreal\t2m53.370s\nuser\t0m4.958s\nsys\t0m2.341s\n\nstack@ubuntu1804:~$ openstack security group show dw-test-3 \u003e /tmp/z\nstack@ubuntu1804:~$ grep 4000 /tmp/z | wc -l\n100\n\nAFTER change:\n\nreal\t2m56.318s\nuser\t0m4.368s\nsys\t0m1.228s\n\nstack@ubuntu1804:~$ openstack security group show dw-test-3 \u003e /tmp/z\nstack@ubuntu1804:~$ grep 4000 /tmp/z | wc -l\n100\n\nAlso, I expect that a lot of that time is spent on the GET/read problem, which is going in in another fix. But as far as I can see, I\u0027m not making the conflicing SG uuid case worse.","commit_id":"6ac5423e6527eed187c21cb845ecb911809947a8"},{"author":{"_account_id":9531,"name":"liuyulong","display_name":"LIU Yulong","email":"i@liuyulong.me","username":"LIU-Yulong"},"change_message_id":"8d5e44c4c635de402057c74a4b803afc83a8d27c","unresolved":false,"context_lines":[{"line_number":624,"context_line":"        if sg:"},{"line_number":625,"context_line":"            for i in sg[\u0027security_group_rules\u0027]:"},{"line_number":626,"context_line":"                rule_key \u003d self._rule_to_key(i)"},{"line_number":627,"context_line":"                if rule_key in new_rules_set:"},{"line_number":628,"context_line":"                    raise ext_sg.SecurityGroupRuleExists(rule_id\u003di.get(\u0027id\u0027))"},{"line_number":629,"context_line":""},{"line_number":630,"context_line":"    def _validate_ip_prefix(self, rule):"}],"source_content_type":"text/x-python","patch_set":17,"id":"9fdfeff1_8091569c","line":627,"range":{"start_line":627,"start_character":16,"end_line":627,"end_character":45},"in_reply_to":"bfdaf3ff_b5b0f844","updated":"2019-01-22 15:58:07.000000000","message":"No, this change look good to me. I think we can get this in for now. But it is also worthy to get more test if we have some more volunteers.","commit_id":"6ac5423e6527eed187c21cb845ecb911809947a8"},{"author":{"_account_id":10980,"name":"Doug Wiegley","email":"dougwig@parkside.io","username":"dougw"},"change_message_id":"2e69118372c116ac780f493c4b19b34bc593da70","unresolved":false,"context_lines":[{"line_number":624,"context_line":"        if sg:"},{"line_number":625,"context_line":"            for i in sg[\u0027security_group_rules\u0027]:"},{"line_number":626,"context_line":"                rule_key \u003d self._rule_to_key(i)"},{"line_number":627,"context_line":"                if rule_key in new_rules_set:"},{"line_number":628,"context_line":"                    raise ext_sg.SecurityGroupRuleExists(rule_id\u003di.get(\u0027id\u0027))"},{"line_number":629,"context_line":""},{"line_number":630,"context_line":"    def _validate_ip_prefix(self, rule):"}],"source_content_type":"text/x-python","patch_set":17,"id":"bfdaf3ff_f9b6a7fa","line":627,"range":{"start_line":627,"start_character":16,"end_line":627,"end_character":45},"in_reply_to":"bfdaf3ff_ca9e6b53","updated":"2019-01-16 13:46:22.000000000","message":"Did you run that without the change? We’re the results different?","commit_id":"6ac5423e6527eed187c21cb845ecb911809947a8"},{"author":{"_account_id":10980,"name":"Doug Wiegley","email":"dougwig@parkside.io","username":"dougw"},"change_message_id":"b4e36e219cdf6c8ba50fd6f970c4db40b056972f","unresolved":false,"context_lines":[{"line_number":624,"context_line":"        if sg:"},{"line_number":625,"context_line":"            for i in sg[\u0027security_group_rules\u0027]:"},{"line_number":626,"context_line":"                rule_key \u003d self._rule_to_key(i)"},{"line_number":627,"context_line":"                if rule_key in new_rules_set:"},{"line_number":628,"context_line":"                    raise ext_sg.SecurityGroupRuleExists(rule_id\u003di.get(\u0027id\u0027))"},{"line_number":629,"context_line":""},{"line_number":630,"context_line":"    def _validate_ip_prefix(self, rule):"}],"source_content_type":"text/x-python","patch_set":17,"id":"bfdaf3ff_ab3c46bf","line":627,"range":{"start_line":627,"start_character":16,"end_line":627,"end_character":45},"in_reply_to":"bfdaf3ff_f9b6a7fa","updated":"2019-01-16 14:27:04.000000000","message":"The reason I ask, is because as long as the \"less than 100\" problem also means there were no duplicate rules, then like the db retry timeout was exceeded with that much contention on a single SG, which is likely not worse than before. But the use case of not having contention on the same SG (the common use case), is a lot better here, so perhaps we could focus on those issues in separate changes?   If, however, this is causing a regression, then that\u0027s a different story.","commit_id":"6ac5423e6527eed187c21cb845ecb911809947a8"},{"author":{"_account_id":10267,"name":"venkata anil","email":"anilvenkata@redhat.com","username":"anil"},"change_message_id":"5595d34c447595eceeee8bc25bd26a07e23f9c5d","unresolved":false,"context_lines":[{"line_number":594,"context_line":"                        raise ext_sg.DuplicateSecurityGroupRuleInPost(rule\u003di)"},{"line_number":595,"context_line":"                    found_self \u003d True"},{"line_number":596,"context_line":""},{"line_number":597,"context_line":"            self._check_for_duplicate_rules_in_db(context, i)"},{"line_number":598,"context_line":""},{"line_number":599,"context_line":"    def _check_for_duplicate_rules_in_db(self, context, security_group_rule):"},{"line_number":600,"context_line":"        # Check in database if rule exists"}],"source_content_type":"text/x-python","patch_set":18,"id":"9fdfeff1_679ee808","side":"PARENT","line":597,"updated":"2019-01-21 07:38:59.000000000","message":"Even in Juno release code, I see we are calling DB operation \"get_security_group_rules\" for each security group rule. But Juno is calling self._check_for_duplicate_rules inside context.session.begin rather than inside with db_api.CONTEXT_WRITER.using. May be because of this writer lock, we are seeing DB retry issues and longer time?\n\n\nJuno code base -\n        with context.session.begin(subtransactions\u003dTrue):\n            if not self.get_security_group(context, security_group_id):\n                raise ext_sg.SecurityGroupNotFound(id\u003dsecurity_group_id)\n\n            self._check_for_duplicate_rules(context, r)\n\n\n\nour latest code -\n        with db_api.CONTEXT_WRITER.using(context):\n            if not self.get_security_group(context, security_group_id):\n                raise ext_sg.SecurityGroupNotFound(id\u003dsecurity_group_id)\n\n            self._check_for_duplicate_rules(context, rules)","commit_id":"284afcf24f2dfd3af20a803ada79c69205249f25"},{"author":{"_account_id":9531,"name":"liuyulong","display_name":"LIU Yulong","email":"i@liuyulong.me","username":"LIU-Yulong"},"change_message_id":"8d5e44c4c635de402057c74a4b803afc83a8d27c","unresolved":false,"context_lines":[{"line_number":594,"context_line":"                        raise ext_sg.DuplicateSecurityGroupRuleInPost(rule\u003di)"},{"line_number":595,"context_line":"                    found_self \u003d True"},{"line_number":596,"context_line":""},{"line_number":597,"context_line":"            self._check_for_duplicate_rules_in_db(context, i)"},{"line_number":598,"context_line":""},{"line_number":599,"context_line":"    def _check_for_duplicate_rules_in_db(self, context, security_group_rule):"},{"line_number":600,"context_line":"        # Check in database if rule exists"}],"source_content_type":"text/x-python","patch_set":18,"id":"9fdfeff1_e0ab82d3","side":"PARENT","line":597,"in_reply_to":"9fdfeff1_113ce891","updated":"2019-01-22 15:58:07.000000000","message":"Any results from stable/queens or stable/rocky, we also have such issue?","commit_id":"284afcf24f2dfd3af20a803ada79c69205249f25"},{"author":{"_account_id":10980,"name":"Doug Wiegley","email":"dougwig@parkside.io","username":"dougw"},"change_message_id":"4ebbb91baf66bcf398554d6ca536afbd18e0e711","unresolved":false,"context_lines":[{"line_number":594,"context_line":"                        raise ext_sg.DuplicateSecurityGroupRuleInPost(rule\u003di)"},{"line_number":595,"context_line":"                    found_self \u003d True"},{"line_number":596,"context_line":""},{"line_number":597,"context_line":"            self._check_for_duplicate_rules_in_db(context, i)"},{"line_number":598,"context_line":""},{"line_number":599,"context_line":"    def _check_for_duplicate_rules_in_db(self, context, security_group_rule):"},{"line_number":600,"context_line":"        # Check in database if rule exists"}],"source_content_type":"text/x-python","patch_set":18,"id":"9fdfeff1_113ce891","side":"PARENT","line":597,"in_reply_to":"9fdfeff1_679ee808","updated":"2019-01-21 23:16:28.000000000","message":"If you take master, and swap it back to using sqla objects directly, it gets fast again.  It\u0027s a combo of the new objects, which are *much* slower loading child objects, along with what you\u0027ve outlined above.","commit_id":"284afcf24f2dfd3af20a803ada79c69205249f25"}]}
