)]}'
{"specs/neutron-extensions-integration-tests.rst":[{"author":{"_account_id":1921,"name":"Andrea Frittoli","email":"andrea.frittoli@gmail.com","username":"andrea-frittoli"},"change_message_id":"53251991e4555b747cfba71d399d7341a762fcd6","unresolved":false,"context_lines":[{"line_number":17,"context_line":"Problem description"},{"line_number":18,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":19,"context_line":""},{"line_number":20,"context_line":"Currently we have a low coverage of Neutron extensions, such like LBaaS, FWaaS, VPNaaS, DVR. There\u0027s no almost scenario tests for them and no any integration tests that involve two or more these features together. This blueprint is supposed to improve this situation and increase coverage by creating more tests for these extensions."},{"line_number":21,"context_line":""},{"line_number":22,"context_line":""},{"line_number":23,"context_line":"Proposed change"}],"source_content_type":"text/x-rst","patch_set":2,"id":"5a890539_a73963a5","line":20,"range":{"start_line":20,"start_character":0,"end_line":20,"end_character":33},"updated":"2014-12-05 17:42:50.000000000","message":"nit: it would be nice to have line breaks similar to what we use for the python code.","commit_id":"ac4610d7d26e98c4d1fbbacfd16e5b6df903e317"},{"author":{"_account_id":10969,"name":"Shnaidman Sagi (Sergey)","display_name":"Shnaidman Sagi","email":"sshnaidm@redhat.com","username":"sergsh"},"change_message_id":"b72ff4f5348079a77dcd49d541ddac7470ac36b4","unresolved":false,"context_lines":[{"line_number":17,"context_line":"Problem description"},{"line_number":18,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":19,"context_line":""},{"line_number":20,"context_line":"Currently we have a low coverage of Neutron extensions, such like LBaaS, FWaaS, VPNaaS, DVR. There\u0027s no almost scenario tests for them and no any integration tests that involve two or more these features together. This blueprint is supposed to improve this situation and increase coverage by creating more tests for these extensions."},{"line_number":21,"context_line":""},{"line_number":22,"context_line":""},{"line_number":23,"context_line":"Proposed change"}],"source_content_type":"text/x-rst","patch_set":2,"id":"3a961159_a1e761e3","line":20,"in_reply_to":"5a890539_a73963a5","updated":"2014-12-17 14:23:52.000000000","message":"Done","commit_id":"ac4610d7d26e98c4d1fbbacfd16e5b6df903e317"},{"author":{"_account_id":1921,"name":"Andrea Frittoli","email":"andrea.frittoli@gmail.com","username":"andrea-frittoli"},"change_message_id":"53251991e4555b747cfba71d399d7341a762fcd6","unresolved":false,"context_lines":[{"line_number":23,"context_line":"Proposed change"},{"line_number":24,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":25,"context_line":""},{"line_number":26,"context_line":"This blueprint contains the following steps to be implemented:"},{"line_number":27,"context_line":""},{"line_number":28,"context_line":" #. LBaaS scenario tests - add more, there\u0027s only one. We need to cover all load balancing strategies that are supported."},{"line_number":29,"context_line":" #. FWaaS scenario tests"}],"source_content_type":"text/x-rst","patch_set":2,"id":"5a890539_c723cf56","line":26,"range":{"start_line":26,"start_character":0,"end_line":26,"end_character":62},"updated":"2014-12-05 17:42:50.000000000","message":"How does this compare with the agreed plan to implement more in-tree functional tests? Can the test below be a combination of tempest tests and neutron in-tree functional tests?","commit_id":"ac4610d7d26e98c4d1fbbacfd16e5b6df903e317"},{"author":{"_account_id":10969,"name":"Shnaidman Sagi (Sergey)","display_name":"Shnaidman Sagi","email":"sshnaidm@redhat.com","username":"sergsh"},"change_message_id":"b72ff4f5348079a77dcd49d541ddac7470ac36b4","unresolved":false,"context_lines":[{"line_number":23,"context_line":"Proposed change"},{"line_number":24,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":25,"context_line":""},{"line_number":26,"context_line":"This blueprint contains the following steps to be implemented:"},{"line_number":27,"context_line":""},{"line_number":28,"context_line":" #. LBaaS scenario tests - add more, there\u0027s only one. We need to cover all load balancing strategies that are supported."},{"line_number":29,"context_line":" #. FWaaS scenario tests"}],"source_content_type":"text/x-rst","patch_set":2,"id":"3a961159_61dd7914","line":26,"in_reply_to":"5a890539_c723cf56","updated":"2014-12-17 14:23:52.000000000","message":"Done","commit_id":"ac4610d7d26e98c4d1fbbacfd16e5b6df903e317"},{"author":{"_account_id":1192,"name":"David Kranz","email":"dkranz@redhat.com","username":"david-kranz"},"change_message_id":"9a22c63922c3ae7125c3341bd206c469b15c9d05","unresolved":false,"context_lines":[{"line_number":23,"context_line":"Proposed change"},{"line_number":24,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":25,"context_line":""},{"line_number":26,"context_line":"This blueprint contains the following steps to be implemented:"},{"line_number":27,"context_line":""},{"line_number":28,"context_line":" #. LBaaS scenario tests - add more, there\u0027s only one. We need to cover all load balancing strategies that are supported."},{"line_number":29,"context_line":" #. FWaaS scenario tests"}],"source_content_type":"text/x-rst","patch_set":2,"id":"5a890539_f8feaa38","line":26,"in_reply_to":"5a890539_c723cf56","updated":"2014-12-05 17:53:28.000000000","message":"I agree and this needs more clarification. Although these are indeed scenario tests with more than one \"service\", they are all part of neutron. There might also be different configurations needed since changes to devstack are mentioned. This sounds like it might belong in neutron so requesting feedback from the neutron team.","commit_id":"ac4610d7d26e98c4d1fbbacfd16e5b6df903e317"},{"author":{"_account_id":10969,"name":"Shnaidman Sagi (Sergey)","display_name":"Shnaidman Sagi","email":"sshnaidm@redhat.com","username":"sergsh"},"change_message_id":"b72ff4f5348079a77dcd49d541ddac7470ac36b4","unresolved":false,"context_lines":[{"line_number":23,"context_line":"Proposed change"},{"line_number":24,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":25,"context_line":""},{"line_number":26,"context_line":"This blueprint contains the following steps to be implemented:"},{"line_number":27,"context_line":""},{"line_number":28,"context_line":" #. LBaaS scenario tests - add more, there\u0027s only one. We need to cover all load balancing strategies that are supported."},{"line_number":29,"context_line":" #. FWaaS scenario tests"}],"source_content_type":"text/x-rst","patch_set":2,"id":"3a961159_81e0a5cd","line":26,"in_reply_to":"5a890539_f8feaa38","updated":"2014-12-17 14:23:52.000000000","message":"Done","commit_id":"ac4610d7d26e98c4d1fbbacfd16e5b6df903e317"},{"author":{"_account_id":1921,"name":"Andrea Frittoli","email":"andrea.frittoli@gmail.com","username":"andrea-frittoli"},"change_message_id":"53251991e4555b747cfba71d399d7341a762fcd6","unresolved":false,"context_lines":[{"line_number":29,"context_line":" #. FWaaS scenario tests"},{"line_number":30,"context_line":" #. VPNaaS scenario tests"},{"line_number":31,"context_line":" #. DVR scenario tests"},{"line_number":32,"context_line":" #. Intergation cases with 2 components: VPN + LBaaS, FWaaS + LBaaS, DVR + LBaaS, VPN + FWaaS, DVR with FWaaS, security groups with FWaaS, VPN with security groups, FWaaS + DVR, LBaaS + DVR, VPNaaS + DVR."},{"line_number":33,"context_line":" #. Integration cases with 3+ components: VPN + LBaaS + FWaaS, VPN + LBaaS + FWaaS + DVR"},{"line_number":34,"context_line":" #. IPv6 support for all extensions above"},{"line_number":35,"context_line":" #. Investigate what are changes to tempest, devstack, Cirros, etc should be introduced for testing. Results and ideas are submitted to etherpad: https://etherpad.openstack.org/p/neutron-extensions-integration-tests"}],"source_content_type":"text/x-rst","patch_set":2,"id":"5a890539_b85502c4","line":32,"updated":"2014-12-05 17:42:50.000000000","message":"Integration","commit_id":"ac4610d7d26e98c4d1fbbacfd16e5b6df903e317"},{"author":{"_account_id":10969,"name":"Shnaidman Sagi (Sergey)","display_name":"Shnaidman Sagi","email":"sshnaidm@redhat.com","username":"sergsh"},"change_message_id":"b72ff4f5348079a77dcd49d541ddac7470ac36b4","unresolved":false,"context_lines":[{"line_number":29,"context_line":" #. FWaaS scenario tests"},{"line_number":30,"context_line":" #. VPNaaS scenario tests"},{"line_number":31,"context_line":" #. DVR scenario tests"},{"line_number":32,"context_line":" #. Intergation cases with 2 components: VPN + LBaaS, FWaaS + LBaaS, DVR + LBaaS, VPN + FWaaS, DVR with FWaaS, security groups with FWaaS, VPN with security groups, FWaaS + DVR, LBaaS + DVR, VPNaaS + DVR."},{"line_number":33,"context_line":" #. Integration cases with 3+ components: VPN + LBaaS + FWaaS, VPN + LBaaS + FWaaS + DVR"},{"line_number":34,"context_line":" #. IPv6 support for all extensions above"},{"line_number":35,"context_line":" #. Investigate what are changes to tempest, devstack, Cirros, etc should be introduced for testing. Results and ideas are submitted to etherpad: https://etherpad.openstack.org/p/neutron-extensions-integration-tests"}],"source_content_type":"text/x-rst","patch_set":2,"id":"3a961159_0bc5b63c","line":32,"in_reply_to":"5a890539_b85502c4","updated":"2014-12-17 14:23:52.000000000","message":"Done","commit_id":"ac4610d7d26e98c4d1fbbacfd16e5b6df903e317"},{"author":{"_account_id":1192,"name":"David Kranz","email":"dkranz@redhat.com","username":"david-kranz"},"change_message_id":"177555ce7f63a017ce344239dd1fdafa1a897a31","unresolved":false,"context_lines":[{"line_number":49,"context_line":"-------------------"},{"line_number":50,"context_line":""},{"line_number":51,"context_line":"All tests above require involvement of virtual machines with Nova, so it can be"},{"line_number":52,"context_line":"implemented only in Tempest tree. Everything which is related to Neutron only"},{"line_number":53,"context_line":"is out of scope of this specs."},{"line_number":54,"context_line":"We need real servers testing:"},{"line_number":55,"context_line":""}],"source_content_type":"text/x-rst","patch_set":7,"id":"3a961159_cb5fdc0d","line":52,"updated":"2014-12-19 14:47:58.000000000","message":"This statement is what needs to be clarified, and not just for neutron. The assumption here is that any tests that require a \"real\", i.e. some services with devstack, cloud must go in tempest. IMO that was not a requirement of the proposal to move functional testing to projects. There is no reason a gate job running tests in neutron cannot use devstack. So the question for the neutron team is whether neutron will have in-project tests like this or not. \n\nAnother problem with putting these kinds of tests, and similar levels of tests from other projects, into tempest is that all of tempest is run as a gate job. It does not scale to have thorough functional tests of every project running as a gate job. For this reason I think it makes more sense to have \"smoke\" variants of these tests in tempest and the thorough set of tests in neutron but I would like to hear from the neutron team.","commit_id":"193346f8ca55a264764c9f575f1aa1e3d2fab698"},{"author":{"_account_id":2035,"name":"Maru Newby","email":"marun@redhat.com","username":"maru"},"change_message_id":"bbb1fd50c3e9c17337b1a46d44510850ed1d7319","unresolved":false,"context_lines":[{"line_number":49,"context_line":"-------------------"},{"line_number":50,"context_line":""},{"line_number":51,"context_line":"All tests above require involvement of virtual machines with Nova, so it can be"},{"line_number":52,"context_line":"implemented only in Tempest tree. Everything which is related to Neutron only"},{"line_number":53,"context_line":"is out of scope of this specs."},{"line_number":54,"context_line":"We need real servers testing:"},{"line_number":55,"context_line":""}],"source_content_type":"text/x-rst","patch_set":7,"id":"3a961159_cc51a063","line":52,"in_reply_to":"3a961159_cb5fdc0d","updated":"2014-12-21 06:56:23.000000000","message":"I agree with David, and the main point I would make is that we need to start looking at in-tree functional testing as the precondition for doing higher-level integration testing that involves other services.  Most of the scenarios proposed in the previous section lack adequate functional coverage, and without knowing what we can cover at a lower-level of abstraction, there is no point in writing integration tests that are more expensive to maintain and execute.\n\nSo, the starting point should be separate efforts to implement functional testing using fake vm\u0027s (namespaces) in each of the advanced services repo (fw, vpn, lb) and in neutron (dvr).  Only when we have adequate coverage independently should we be looking at writing tests that validate the interaction of these features let alone interaction with other openstack services like Nova.","commit_id":"193346f8ca55a264764c9f575f1aa1e3d2fab698"},{"author":{"_account_id":10969,"name":"Shnaidman Sagi (Sergey)","display_name":"Shnaidman Sagi","email":"sshnaidm@redhat.com","username":"sergsh"},"change_message_id":"91461689e0834e2263f9f6f71570b04406317a43","unresolved":false,"context_lines":[{"line_number":49,"context_line":"-------------------"},{"line_number":50,"context_line":""},{"line_number":51,"context_line":"All tests above require involvement of virtual machines with Nova, so it can be"},{"line_number":52,"context_line":"implemented only in Tempest tree. Everything which is related to Neutron only"},{"line_number":53,"context_line":"is out of scope of this specs."},{"line_number":54,"context_line":"We need real servers testing:"},{"line_number":55,"context_line":""}],"source_content_type":"text/x-rst","patch_set":7,"id":"3a961159_03944789","line":52,"in_reply_to":"3a961159_cb5fdc0d","updated":"2014-12-22 12:52:22.000000000","message":"Thanks, David. I asked help form Neutron guys for this topic.\n\nI\u0027d like only to clarify my point about neutron tree, I didn\u0027t mean anything about devstack, but about including other services in these tests. Today it\u0027s nova, tomorrow it could be cinder, designate or anything else. I don\u0027t know any possibility to include these services in neutron tests, so IMO it\u0027s just impossible to implement them there. And this is one of reason to call these tests INTEGRATION - integration not with neutron only, but with other services of Openstack. I hope I\u0027ve explained my self, feel free to correct me or ask additional clarifying.","commit_id":"193346f8ca55a264764c9f575f1aa1e3d2fab698"},{"author":{"_account_id":10969,"name":"Shnaidman Sagi (Sergey)","display_name":"Shnaidman Sagi","email":"sshnaidm@redhat.com","username":"sergsh"},"change_message_id":"91461689e0834e2263f9f6f71570b04406317a43","unresolved":false,"context_lines":[{"line_number":49,"context_line":"-------------------"},{"line_number":50,"context_line":""},{"line_number":51,"context_line":"All tests above require involvement of virtual machines with Nova, so it can be"},{"line_number":52,"context_line":"implemented only in Tempest tree. Everything which is related to Neutron only"},{"line_number":53,"context_line":"is out of scope of this specs."},{"line_number":54,"context_line":"We need real servers testing:"},{"line_number":55,"context_line":""}],"source_content_type":"text/x-rst","patch_set":7,"id":"3a961159_033d272b","line":52,"in_reply_to":"3a961159_cc51a063","updated":"2014-12-22 12:52:22.000000000","message":"Thanks for review, Maru. \n\nI can not agree with you, unfortunately, about functional coverage. \n\nFirst of all, these components has a more or less coverage in api network tests, please look at tempest.api.network folder in existing tempest. \n\nSecondly, if they require more functional coverage - it\u0027s out of scope of this blueprint and should not affect in any way integration testing. There is no (and should not be) any reason-sequence relation between two different sorts of testing. Like blackbox testing could be done fine without any whitebox testing, system testing could be done fine without (or before) any component testing, it\u0027s completely different and not related to each other sorts of testing.\n\nI completely agree with you that we should increase coverage of functional tests of these components, however I\u0027d like to do it in separate blueprint when process of moving functional test to neutron will be clear.","commit_id":"193346f8ca55a264764c9f575f1aa1e3d2fab698"}]}