)]}'
{"specs/queens/python-build-install-simplification.rst":[{"author":{"_account_id":17068,"name":"Jean-Philippe Evrard","email":"openstack@a.spamming.party","username":"evrardjp"},"change_message_id":"2954af910104351e4c46a8b6e668126528e22c14","unresolved":false,"context_lines":[{"line_number":29,"context_line":"individually for every host that needs the service."},{"line_number":30,"context_line":""},{"line_number":31,"context_line":"The ``py_pkgs`` lookup, which pulls together the information used by the"},{"line_number":32,"context_line":"build process is a complex piece of code which is not well understood. It"},{"line_number":33,"context_line":"is a black box in terms of what it does, its output is very verbose and"},{"line_number":34,"context_line":"it makes some decisions about the information it reads and outputs which"},{"line_number":35,"context_line":"are not well understood or documented. The subsequent jinja in the"}],"source_content_type":"text/x-rst","patch_set":1,"id":"7f287b81_0c243aae","line":32,"range":{"start_line":32,"start_character":14,"end_line":32,"end_character":69},"updated":"2017-09-07 15:49:35.000000000","message":"This is an opinion and should be removed.","commit_id":"550382eb285ed848fddb76b6d6126957ebfcb7fb"},{"author":{"_account_id":6816,"name":"Jesse Pretorius","email":"jesse@odyssey4.me","username":"jesse-pretorius"},"change_message_id":"87d561a5779c37d814a2dea72083fd0d6355483c","unresolved":false,"context_lines":[{"line_number":29,"context_line":"individually for every host that needs the service."},{"line_number":30,"context_line":""},{"line_number":31,"context_line":"The ``py_pkgs`` lookup, which pulls together the information used by the"},{"line_number":32,"context_line":"build process is a complex piece of code which is not well understood. It"},{"line_number":33,"context_line":"is a black box in terms of what it does, its output is very verbose and"},{"line_number":34,"context_line":"it makes some decisions about the information it reads and outputs which"},{"line_number":35,"context_line":"are not well understood or documented. The subsequent jinja in the"}],"source_content_type":"text/x-rst","patch_set":1,"id":"5f2577a7_340ce8eb","line":32,"range":{"start_line":32,"start_character":14,"end_line":32,"end_character":69},"in_reply_to":"7f287b81_0c243aae","updated":"2017-09-21 06:43:22.000000000","message":"Done.","commit_id":"550382eb285ed848fddb76b6d6126957ebfcb7fb"},{"author":{"_account_id":17068,"name":"Jean-Philippe Evrard","email":"openstack@a.spamming.party","username":"evrardjp"},"change_message_id":"2954af910104351e4c46a8b6e668126528e22c14","unresolved":false,"context_lines":[{"line_number":30,"context_line":""},{"line_number":31,"context_line":"The ``py_pkgs`` lookup, which pulls together the information used by the"},{"line_number":32,"context_line":"build process is a complex piece of code which is not well understood. It"},{"line_number":33,"context_line":"is a black box in terms of what it does, its output is very verbose and"},{"line_number":34,"context_line":"it makes some decisions about the information it reads and outputs which"},{"line_number":35,"context_line":"are not well understood or documented. The subsequent jinja in the"},{"line_number":36,"context_line":"repo-build role which processes it is tough to work through and not easily"}],"source_content_type":"text/x-rst","patch_set":1,"id":"7f287b81_4c7c8282","line":33,"range":{"start_line":33,"start_character":41,"end_line":33,"end_character":67},"updated":"2017-09-07 15:49:35.000000000","message":"py_pkg lookup has no output.\nWe decided to have a debug task to show its content for ease of debugging for new contributors. We could probably keep that only when -vv is used for example","commit_id":"550382eb285ed848fddb76b6d6126957ebfcb7fb"},{"author":{"_account_id":17068,"name":"Jean-Philippe Evrard","email":"openstack@a.spamming.party","username":"evrardjp"},"change_message_id":"2954af910104351e4c46a8b6e668126528e22c14","unresolved":false,"context_lines":[{"line_number":30,"context_line":""},{"line_number":31,"context_line":"The ``py_pkgs`` lookup, which pulls together the information used by the"},{"line_number":32,"context_line":"build process is a complex piece of code which is not well understood. It"},{"line_number":33,"context_line":"is a black box in terms of what it does, its output is very verbose and"},{"line_number":34,"context_line":"it makes some decisions about the information it reads and outputs which"},{"line_number":35,"context_line":"are not well understood or documented. The subsequent jinja in the"},{"line_number":36,"context_line":"repo-build role which processes it is tough to work through and not easily"}],"source_content_type":"text/x-rst","patch_set":1,"id":"7f287b81_ec41ee3a","line":33,"range":{"start_line":33,"start_character":5,"end_line":33,"end_character":39},"updated":"2017-09-07 15:49:35.000000000","message":"you can probably add here that due to its \"black box\" nature, people don\u0027t touch it and therefore could become technical debt because nobody looks after the code.","commit_id":"550382eb285ed848fddb76b6d6126957ebfcb7fb"},{"author":{"_account_id":6816,"name":"Jesse Pretorius","email":"jesse@odyssey4.me","username":"jesse-pretorius"},"change_message_id":"87d561a5779c37d814a2dea72083fd0d6355483c","unresolved":false,"context_lines":[{"line_number":30,"context_line":""},{"line_number":31,"context_line":"The ``py_pkgs`` lookup, which pulls together the information used by the"},{"line_number":32,"context_line":"build process is a complex piece of code which is not well understood. It"},{"line_number":33,"context_line":"is a black box in terms of what it does, its output is very verbose and"},{"line_number":34,"context_line":"it makes some decisions about the information it reads and outputs which"},{"line_number":35,"context_line":"are not well understood or documented. The subsequent jinja in the"},{"line_number":36,"context_line":"repo-build role which processes it is tough to work through and not easily"}],"source_content_type":"text/x-rst","patch_set":1,"id":"5f2577a7_346748c0","line":33,"range":{"start_line":33,"start_character":41,"end_line":33,"end_character":67},"in_reply_to":"7f287b81_4c7c8282","updated":"2017-09-21 06:43:22.000000000","message":"Rephrased.","commit_id":"550382eb285ed848fddb76b6d6126957ebfcb7fb"},{"author":{"_account_id":6816,"name":"Jesse Pretorius","email":"jesse@odyssey4.me","username":"jesse-pretorius"},"change_message_id":"87d561a5779c37d814a2dea72083fd0d6355483c","unresolved":false,"context_lines":[{"line_number":30,"context_line":""},{"line_number":31,"context_line":"The ``py_pkgs`` lookup, which pulls together the information used by the"},{"line_number":32,"context_line":"build process is a complex piece of code which is not well understood. It"},{"line_number":33,"context_line":"is a black box in terms of what it does, its output is very verbose and"},{"line_number":34,"context_line":"it makes some decisions about the information it reads and outputs which"},{"line_number":35,"context_line":"are not well understood or documented. The subsequent jinja in the"},{"line_number":36,"context_line":"repo-build role which processes it is tough to work through and not easily"}],"source_content_type":"text/x-rst","patch_set":1,"id":"5f2577a7_1458ccf7","line":33,"range":{"start_line":33,"start_character":5,"end_line":33,"end_character":39},"in_reply_to":"7f287b81_ec41ee3a","updated":"2017-09-21 06:43:22.000000000","message":"Done.","commit_id":"550382eb285ed848fddb76b6d6126957ebfcb7fb"},{"author":{"_account_id":17068,"name":"Jean-Philippe Evrard","email":"openstack@a.spamming.party","username":"evrardjp"},"change_message_id":"2954af910104351e4c46a8b6e668126528e22c14","unresolved":false,"context_lines":[{"line_number":34,"context_line":"it makes some decisions about the information it reads and outputs which"},{"line_number":35,"context_line":"are not well understood or documented. The subsequent jinja in the"},{"line_number":36,"context_line":"repo-build role which processes it is tough to work through and not easily"},{"line_number":37,"context_line":"maintained."},{"line_number":38,"context_line":""},{"line_number":39,"context_line":"The way that git repositories are specified and parameters are provided"},{"line_number":40,"context_line":"to the build process does not scale very well. Each git repo requires at"}],"source_content_type":"text/x-rst","patch_set":1,"id":"7f287b81_ec98cec0","line":37,"updated":"2017-09-07 15:49:35.000000000","message":"Alternatively, we could write jinja filters to simplify its parsing, but then we are piling up black boxes, where we could go back to drawing board.","commit_id":"550382eb285ed848fddb76b6d6126957ebfcb7fb"},{"author":{"_account_id":6816,"name":"Jesse Pretorius","email":"jesse@odyssey4.me","username":"jesse-pretorius"},"change_message_id":"87d561a5779c37d814a2dea72083fd0d6355483c","unresolved":false,"context_lines":[{"line_number":34,"context_line":"it makes some decisions about the information it reads and outputs which"},{"line_number":35,"context_line":"are not well understood or documented. The subsequent jinja in the"},{"line_number":36,"context_line":"repo-build role which processes it is tough to work through and not easily"},{"line_number":37,"context_line":"maintained."},{"line_number":38,"context_line":""},{"line_number":39,"context_line":"The way that git repositories are specified and parameters are provided"},{"line_number":40,"context_line":"to the build process does not scale very well. Each git repo requires at"}],"source_content_type":"text/x-rst","patch_set":1,"id":"5f2577a7_94527cdd","line":37,"in_reply_to":"7f287b81_ec98cec0","updated":"2017-09-21 06:43:22.000000000","message":"Done.","commit_id":"550382eb285ed848fddb76b6d6126957ebfcb7fb"},{"author":{"_account_id":17068,"name":"Jean-Philippe Evrard","email":"openstack@a.spamming.party","username":"evrardjp"},"change_message_id":"2954af910104351e4c46a8b6e668126528e22c14","unresolved":false,"context_lines":[{"line_number":51,"context_line":""},{"line_number":52,"context_line":"The git clone process used to fetch the git sources in order to use when"},{"line_number":53,"context_line":"building is done asynchronously in order to improve the time to completion,"},{"line_number":54,"context_line":"however asynchronous tasks cannot be retried in Ansible, and the git clones"},{"line_number":55,"context_line":"commonly fail."},{"line_number":56,"context_line":""},{"line_number":57,"context_line":"When building wheels, pip has no way of resolving all dependencies up-front."}],"source_content_type":"text/x-rst","patch_set":1,"id":"7f287b81_4c348293","line":54,"range":{"start_line":54,"start_character":8,"end_line":54,"end_character":55},"updated":"2017-09-07 15:49:35.000000000","message":"... we are hitting a limitation of ansible: we cannot retry single elements of a list of asynchronous git cloning. (IIRC: Only the full process can be retried, and success rc will only be stored for the process)","commit_id":"550382eb285ed848fddb76b6d6126957ebfcb7fb"},{"author":{"_account_id":6816,"name":"Jesse Pretorius","email":"jesse@odyssey4.me","username":"jesse-pretorius"},"change_message_id":"87d561a5779c37d814a2dea72083fd0d6355483c","unresolved":false,"context_lines":[{"line_number":51,"context_line":""},{"line_number":52,"context_line":"The git clone process used to fetch the git sources in order to use when"},{"line_number":53,"context_line":"building is done asynchronously in order to improve the time to completion,"},{"line_number":54,"context_line":"however asynchronous tasks cannot be retried in Ansible, and the git clones"},{"line_number":55,"context_line":"commonly fail."},{"line_number":56,"context_line":""},{"line_number":57,"context_line":"When building wheels, pip has no way of resolving all dependencies up-front."}],"source_content_type":"text/x-rst","patch_set":1,"id":"5f2577a7_f4430012","line":54,"range":{"start_line":54,"start_character":8,"end_line":54,"end_character":55},"in_reply_to":"7f287b81_4c348293","updated":"2017-09-21 06:43:22.000000000","message":"Done.","commit_id":"550382eb285ed848fddb76b6d6126957ebfcb7fb"},{"author":{"_account_id":17068,"name":"Jean-Philippe Evrard","email":"openstack@a.spamming.party","username":"evrardjp"},"change_message_id":"2954af910104351e4c46a8b6e668126528e22c14","unresolved":false,"context_lines":[{"line_number":51,"context_line":""},{"line_number":52,"context_line":"The git clone process used to fetch the git sources in order to use when"},{"line_number":53,"context_line":"building is done asynchronously in order to improve the time to completion,"},{"line_number":54,"context_line":"however asynchronous tasks cannot be retried in Ansible, and the git clones"},{"line_number":55,"context_line":"commonly fail."},{"line_number":56,"context_line":""},{"line_number":57,"context_line":"When building wheels, pip has no way of resolving all dependencies up-front."},{"line_number":58,"context_line":"The only capability it has is to resolve the requirements for the current"}],"source_content_type":"text/x-rst","patch_set":1,"id":"7f287b81_0cdf9a3a","line":55,"range":{"start_line":54,"start_character":61,"end_line":55,"end_character":13},"updated":"2017-09-07 15:49:35.000000000","message":"we could rephrase that by writing:\nadding a more robust git cloning would require writing our own action module, which increases technical debt.","commit_id":"550382eb285ed848fddb76b6d6126957ebfcb7fb"},{"author":{"_account_id":6816,"name":"Jesse Pretorius","email":"jesse@odyssey4.me","username":"jesse-pretorius"},"change_message_id":"87d561a5779c37d814a2dea72083fd0d6355483c","unresolved":false,"context_lines":[{"line_number":51,"context_line":""},{"line_number":52,"context_line":"The git clone process used to fetch the git sources in order to use when"},{"line_number":53,"context_line":"building is done asynchronously in order to improve the time to completion,"},{"line_number":54,"context_line":"however asynchronous tasks cannot be retried in Ansible, and the git clones"},{"line_number":55,"context_line":"commonly fail."},{"line_number":56,"context_line":""},{"line_number":57,"context_line":"When building wheels, pip has no way of resolving all dependencies up-front."},{"line_number":58,"context_line":"The only capability it has is to resolve the requirements for the current"}],"source_content_type":"text/x-rst","patch_set":1,"id":"5f2577a7_143fac87","line":55,"range":{"start_line":54,"start_character":61,"end_line":55,"end_character":13},"in_reply_to":"7f287b81_0cdf9a3a","updated":"2017-09-21 06:43:22.000000000","message":"Done.","commit_id":"550382eb285ed848fddb76b6d6126957ebfcb7fb"},{"author":{"_account_id":17068,"name":"Jean-Philippe Evrard","email":"openstack@a.spamming.party","username":"evrardjp"},"change_message_id":"2954af910104351e4c46a8b6e668126528e22c14","unresolved":false,"context_lines":[{"line_number":54,"context_line":"however asynchronous tasks cannot be retried in Ansible, and the git clones"},{"line_number":55,"context_line":"commonly fail."},{"line_number":56,"context_line":""},{"line_number":57,"context_line":"When building wheels, pip has no way of resolving all dependencies up-front."},{"line_number":58,"context_line":"The only capability it has is to resolve the requirements for the current"},{"line_number":59,"context_line":"package. It then processes each package requirement in turn. To do so requires"},{"line_number":60,"context_line":"downloading the package and unpacking it to read the requirements. This is a"},{"line_number":61,"context_line":"sequential process and therefore takes a long time when processing packages"},{"line_number":62,"context_line":"with a lot of requirements as is typical for OpenStack projects."},{"line_number":63,"context_line":""},{"line_number":64,"context_line":"In Kilo the OpenStack requirements management process did not have the jobs"},{"line_number":65,"context_line":"which tested the co-installability of all OpenStack packages and produced the"},{"line_number":66,"context_line":"``upper-constraints.txt`` file as a manifest of which package versions worked"},{"line_number":67,"context_line":"together. We therefore needed to do our own processing of all python packages"},{"line_number":68,"context_line":"which would be installed by the roles and had to compile a set of requirements"},{"line_number":69,"context_line":"and constraints across them all for the purpose of building the wheels, and"},{"line_number":70,"context_line":"ensuring that the installed set were consistent for a build. When the"},{"line_number":71,"context_line":"OpenStack requirements repository started publishing the upper-constraints"},{"line_number":72,"context_line":"file we adopted it immediately to help keep builds more consistent. However,"},{"line_number":73,"context_line":"we still produce our own ``requirements_absolute_requirements.txt`` file"},{"line_number":74,"context_line":"which is used for all pip install tasks in order to ensure consistency and"},{"line_number":75,"context_line":"to ensure that the packages we built from git are used (instead of making"},{"line_number":76,"context_line":"the install in the role do the install from the git source, it installs from"},{"line_number":77,"context_line":"the wheel held in the repo server). However this is not practical any more"},{"line_number":78,"context_line":"as there are requirements for different services and needs which are not"},{"line_number":79,"context_line":"resolvable down to a common set - we need to be able to allow the installation"},{"line_number":80,"context_line":"of any version of packages and only apply constraints when needed."},{"line_number":81,"context_line":""},{"line_number":82,"context_line":"Some of the venvs we build do not adhere to the OpenStack requirements process"},{"line_number":83,"context_line":"and therefore sometimes cannot be built using the upper constraints file."}],"source_content_type":"text/x-rst","patch_set":1,"id":"7f287b81_2c8136fc","line":80,"range":{"start_line":57,"start_character":0,"end_line":80,"end_character":66},"updated":"2017-09-07 15:49:35.000000000","message":"Important, should go directly after paragraph 1, to link that with the reset of the python wheel building maybe?","commit_id":"550382eb285ed848fddb76b6d6126957ebfcb7fb"},{"author":{"_account_id":6816,"name":"Jesse Pretorius","email":"jesse@odyssey4.me","username":"jesse-pretorius"},"change_message_id":"87d561a5779c37d814a2dea72083fd0d6355483c","unresolved":false,"context_lines":[{"line_number":54,"context_line":"however asynchronous tasks cannot be retried in Ansible, and the git clones"},{"line_number":55,"context_line":"commonly fail."},{"line_number":56,"context_line":""},{"line_number":57,"context_line":"When building wheels, pip has no way of resolving all dependencies up-front."},{"line_number":58,"context_line":"The only capability it has is to resolve the requirements for the current"},{"line_number":59,"context_line":"package. It then processes each package requirement in turn. To do so requires"},{"line_number":60,"context_line":"downloading the package and unpacking it to read the requirements. This is a"},{"line_number":61,"context_line":"sequential process and therefore takes a long time when processing packages"},{"line_number":62,"context_line":"with a lot of requirements as is typical for OpenStack projects."},{"line_number":63,"context_line":""},{"line_number":64,"context_line":"In Kilo the OpenStack requirements management process did not have the jobs"},{"line_number":65,"context_line":"which tested the co-installability of all OpenStack packages and produced the"},{"line_number":66,"context_line":"``upper-constraints.txt`` file as a manifest of which package versions worked"},{"line_number":67,"context_line":"together. We therefore needed to do our own processing of all python packages"},{"line_number":68,"context_line":"which would be installed by the roles and had to compile a set of requirements"},{"line_number":69,"context_line":"and constraints across them all for the purpose of building the wheels, and"},{"line_number":70,"context_line":"ensuring that the installed set were consistent for a build. When the"},{"line_number":71,"context_line":"OpenStack requirements repository started publishing the upper-constraints"},{"line_number":72,"context_line":"file we adopted it immediately to help keep builds more consistent. However,"},{"line_number":73,"context_line":"we still produce our own ``requirements_absolute_requirements.txt`` file"},{"line_number":74,"context_line":"which is used for all pip install tasks in order to ensure consistency and"},{"line_number":75,"context_line":"to ensure that the packages we built from git are used (instead of making"},{"line_number":76,"context_line":"the install in the role do the install from the git source, it installs from"},{"line_number":77,"context_line":"the wheel held in the repo server). However this is not practical any more"},{"line_number":78,"context_line":"as there are requirements for different services and needs which are not"},{"line_number":79,"context_line":"resolvable down to a common set - we need to be able to allow the installation"},{"line_number":80,"context_line":"of any version of packages and only apply constraints when needed."},{"line_number":81,"context_line":""},{"line_number":82,"context_line":"Some of the venvs we build do not adhere to the OpenStack requirements process"},{"line_number":83,"context_line":"and therefore sometimes cannot be built using the upper constraints file."}],"source_content_type":"text/x-rst","patch_set":1,"id":"5f2577a7_5439b4a1","line":80,"range":{"start_line":57,"start_character":0,"end_line":80,"end_character":66},"in_reply_to":"7f287b81_2c8136fc","updated":"2017-09-21 06:43:22.000000000","message":"The first paragraph was more of a introduction, with each subsequent paragraph trying to tackle each step in the repo build process in sequence.","commit_id":"550382eb285ed848fddb76b6d6126957ebfcb7fb"},{"author":{"_account_id":17068,"name":"Jean-Philippe Evrard","email":"openstack@a.spamming.party","username":"evrardjp"},"change_message_id":"2954af910104351e4c46a8b6e668126528e22c14","unresolved":false,"context_lines":[{"line_number":99,"context_line":"for multiple architectures, then had to also split out multi-distro builds due"},{"line_number":100,"context_line":"do wheel/venvs references to C libraries being different for each distro due"},{"line_number":101,"context_line":"to the libraries available being different. Currently this is working, but it"},{"line_number":102,"context_line":"makes the repo build process much more complex and take a lot more time."},{"line_number":103,"context_line":""},{"line_number":104,"context_line":"In order to facilitate using the repo-server to respond to pip index queries,"},{"line_number":105,"context_line":"multiple directories and symlinks have been used to prepare the appropriate"}],"source_content_type":"text/x-rst","patch_set":1,"id":"7f287b81_ac1e46b8","line":102,"updated":"2017-09-07 15:49:35.000000000","message":"On top of that it multiplied the inventory nodes where this has to run, and sync afterwards.","commit_id":"550382eb285ed848fddb76b6d6126957ebfcb7fb"},{"author":{"_account_id":7353,"name":"Kevin Carter","email":"kevin@cloudnull.com","username":"cloudnull"},"change_message_id":"59e277b2ea7d3d574b1e3d4826c00c40192e1c5a","unresolved":false,"context_lines":[{"line_number":149,"context_line":"  necessary for offline deployments and for environments where the deployer"},{"line_number":150,"context_line":"  specifically opts-in to ensuring that everything is built."},{"line_number":151,"context_line":""},{"line_number":152,"context_line":"* Replace the current storage structure for wheels with a flat directory."},{"line_number":153,"context_line":"  This directory will be served via the pypi API provided by the very simple"},{"line_number":154,"context_line":"  `pypiserver`_ application."},{"line_number":155,"context_line":""},{"line_number":156,"context_line":"* Use nginx as a reverse proxy which responds to requests from pip by first"},{"line_number":157,"context_line":"  trying against the local pypiserver, then against tarballs.openstack.org"}],"source_content_type":"text/x-rst","patch_set":1,"id":"7f287b81_72acd0ec","line":154,"range":{"start_line":152,"start_character":0,"end_line":154,"end_character":28},"updated":"2017-09-06 19:07:45.000000000","message":"So long as this can work cross repo/distro/release this will be great!","commit_id":"550382eb285ed848fddb76b6d6126957ebfcb7fb"},{"author":{"_account_id":6816,"name":"Jesse Pretorius","email":"jesse@odyssey4.me","username":"jesse-pretorius"},"change_message_id":"116201424ebd4763924b536bd8d31cb4c405c47c","unresolved":false,"context_lines":[{"line_number":149,"context_line":"  necessary for offline deployments and for environments where the deployer"},{"line_number":150,"context_line":"  specifically opts-in to ensuring that everything is built."},{"line_number":151,"context_line":""},{"line_number":152,"context_line":"* Replace the current storage structure for wheels with a flat directory."},{"line_number":153,"context_line":"  This directory will be served via the pypi API provided by the very simple"},{"line_number":154,"context_line":"  `pypiserver`_ application."},{"line_number":155,"context_line":""},{"line_number":156,"context_line":"* Use nginx as a reverse proxy which responds to requests from pip by first"},{"line_number":157,"context_line":"  trying against the local pypiserver, then against tarballs.openstack.org"}],"source_content_type":"text/x-rst","patch_set":1,"id":"7f287b81_36b3c70c","line":154,"range":{"start_line":152,"start_character":0,"end_line":154,"end_character":28},"in_reply_to":"7f287b81_72acd0ec","updated":"2017-09-07 10:32:17.000000000","message":"Yeah, I *think* that only venvs need to be prepped per distro - not wheels. However, if we absolutely need to then I think we could present an index per distro, and therefore we have two folders and two instances of pypiserver. The pip installs would then refer to the index for that distro.\n\nI think we\u0027ll need to have a multi-arch and multi-distro test implemented to validate all this. Perhaps zuulv3 will make a multi-distro test easier to implement. A multi-arch test could perhaps be done using the multi-node AIO tooling.","commit_id":"550382eb285ed848fddb76b6d6126957ebfcb7fb"},{"author":{"_account_id":7353,"name":"Kevin Carter","email":"kevin@cloudnull.com","username":"cloudnull"},"change_message_id":"59e277b2ea7d3d574b1e3d4826c00c40192e1c5a","unresolved":false,"context_lines":[{"line_number":153,"context_line":"  This directory will be served via the pypi API provided by the very simple"},{"line_number":154,"context_line":"  `pypiserver`_ application."},{"line_number":155,"context_line":""},{"line_number":156,"context_line":"* Use nginx as a reverse proxy which responds to requests from pip by first"},{"line_number":157,"context_line":"  trying against the local pypiserver, then against tarballs.openstack.org"},{"line_number":158,"context_line":"  and then against pypi. This will allow nginx to cache all downloaded"},{"line_number":159,"context_line":"  packages (speeding up subsequent requests) without the repo server having"},{"line_number":160,"context_line":"  build them. It will also allow the package version to be whatever is"},{"line_number":161,"context_line":"  requested, rather than a fixed set. This allows different constraint sets"},{"line_number":162,"context_line":"  to be used when building venvs."},{"line_number":163,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"7f287b81_5232f4f3","line":160,"range":{"start_line":156,"start_character":2,"end_line":160,"end_character":13},"updated":"2017-09-06 19:07:45.000000000","message":"This is a great idea!","commit_id":"550382eb285ed848fddb76b6d6126957ebfcb7fb"},{"author":{"_account_id":7353,"name":"Kevin Carter","email":"kevin@cloudnull.com","username":"cloudnull"},"change_message_id":"59e277b2ea7d3d574b1e3d4826c00c40192e1c5a","unresolved":false,"context_lines":[{"line_number":157,"context_line":"  trying against the local pypiserver, then against tarballs.openstack.org"},{"line_number":158,"context_line":"  and then against pypi. This will allow nginx to cache all downloaded"},{"line_number":159,"context_line":"  packages (speeding up subsequent requests) without the repo server having"},{"line_number":160,"context_line":"  build them. It will also allow the package version to be whatever is"},{"line_number":161,"context_line":"  requested, rather than a fixed set. This allows different constraint sets"},{"line_number":162,"context_line":"  to be used when building venvs."},{"line_number":163,"context_line":""},{"line_number":164,"context_line":"* Implement changes to each role to handle the wheel building and venv"}],"source_content_type":"text/x-rst","patch_set":1,"id":"7f287b81_b258f830","line":161,"range":{"start_line":160,"start_character":14,"end_line":161,"end_character":37},"updated":"2017-09-06 19:07:45.000000000","message":"This is rather scary. I know we\u0027ll need to improve in terms of our flexibility but allowing package installations outside of a fixed release by default sounds like it could lead to a very slippery slope.","commit_id":"550382eb285ed848fddb76b6d6126957ebfcb7fb"},{"author":{"_account_id":6816,"name":"Jesse Pretorius","email":"jesse@odyssey4.me","username":"jesse-pretorius"},"change_message_id":"116201424ebd4763924b536bd8d31cb4c405c47c","unresolved":false,"context_lines":[{"line_number":157,"context_line":"  trying against the local pypiserver, then against tarballs.openstack.org"},{"line_number":158,"context_line":"  and then against pypi. This will allow nginx to cache all downloaded"},{"line_number":159,"context_line":"  packages (speeding up subsequent requests) without the repo server having"},{"line_number":160,"context_line":"  build them. It will also allow the package version to be whatever is"},{"line_number":161,"context_line":"  requested, rather than a fixed set. This allows different constraint sets"},{"line_number":162,"context_line":"  to be used when building venvs."},{"line_number":163,"context_line":""},{"line_number":164,"context_line":"* Implement changes to each role to handle the wheel building and venv"}],"source_content_type":"text/x-rst","patch_set":1,"id":"7f287b81_76d4af20","line":161,"range":{"start_line":160,"start_character":14,"end_line":161,"end_character":37},"in_reply_to":"7f287b81_b258f830","updated":"2017-09-07 10:32:17.000000000","message":"It is, but people are doing it. We don\u0027t test mixed releases - that\u0027s up to a deployer, not the project.\n\nI was kinda thinking that we could potentially publish artifacts per release which itemise each version of all distro and python packages used for the last successful build so that it would be easy to reference and potentially use to replicate a build environment.","commit_id":"550382eb285ed848fddb76b6d6126957ebfcb7fb"},{"author":{"_account_id":7353,"name":"Kevin Carter","email":"kevin@cloudnull.com","username":"cloudnull"},"change_message_id":"59e277b2ea7d3d574b1e3d4826c00c40192e1c5a","unresolved":false,"context_lines":[{"line_number":163,"context_line":""},{"line_number":164,"context_line":"* Implement changes to each role to handle the wheel building and venv"},{"line_number":165,"context_line":"  building, but do it in such a way that only the build can be executed"},{"line_number":166,"context_line":"  by using tags, setting a specific flag, or include_role and tasks_from."},{"line_number":167,"context_line":"  The specific dependencies can then be itemised in the role and the role"},{"line_number":168,"context_line":"  can be used for artifact preparation."},{"line_number":169,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"7f287b81_f22d6095","line":166,"range":{"start_line":166,"start_character":45,"end_line":166,"end_character":57},"updated":"2017-09-06 19:07:45.000000000","message":"I think the only clean way to do this is through the use of \"include_role\", otherwise we\u0027ll once again be on the path to mass duplication of code.","commit_id":"550382eb285ed848fddb76b6d6126957ebfcb7fb"},{"author":{"_account_id":7353,"name":"Kevin Carter","email":"kevin@cloudnull.com","username":"cloudnull"},"change_message_id":"59e277b2ea7d3d574b1e3d4826c00c40192e1c5a","unresolved":false,"context_lines":[{"line_number":167,"context_line":"  The specific dependencies can then be itemised in the role and the role"},{"line_number":168,"context_line":"  can be used for artifact preparation."},{"line_number":169,"context_line":""},{"line_number":170,"context_line":"* Remove all pip install activities from hosts, replacing them with the"},{"line_number":171,"context_line":"  use of distro packages exclusively for any python requirements on the"},{"line_number":172,"context_line":"  hosts. Any requirements which cannot be met via a distribution package"},{"line_number":173,"context_line":"  should be implemented in a virtualenv. This prevents system package"},{"line_number":174,"context_line":"  conflicts and will hopefully reduce the host package installation"},{"line_number":175,"context_line":"  requirements."},{"line_number":176,"context_line":""},{"line_number":177,"context_line":"* Implement a playbook which is optionally used to prepare pre-built venvs"},{"line_number":178,"context_line":"  for an environment as they are today."}],"source_content_type":"text/x-rst","patch_set":1,"id":"7f287b81_b20a381d","line":175,"range":{"start_line":170,"start_character":0,"end_line":175,"end_character":15},"updated":"2017-09-06 19:07:45.000000000","message":"this means we\u0027ll be linking our use of venvs with system packages (site-packages) which breaks our host system isolation; or at least allow for cross pollution. https://virtualenv.pypa.io/en/stable/userguide/#the-system-site-packages-option\n\n* Are we sure we want to do this?\n* Does going back to the package manager really buy us very much?\n* Could we potentially use giftwrap for some of this?","commit_id":"550382eb285ed848fddb76b6d6126957ebfcb7fb"},{"author":{"_account_id":6816,"name":"Jesse Pretorius","email":"jesse@odyssey4.me","username":"jesse-pretorius"},"change_message_id":"116201424ebd4763924b536bd8d31cb4c405c47c","unresolved":false,"context_lines":[{"line_number":167,"context_line":"  The specific dependencies can then be itemised in the role and the role"},{"line_number":168,"context_line":"  can be used for artifact preparation."},{"line_number":169,"context_line":""},{"line_number":170,"context_line":"* Remove all pip install activities from hosts, replacing them with the"},{"line_number":171,"context_line":"  use of distro packages exclusively for any python requirements on the"},{"line_number":172,"context_line":"  hosts. Any requirements which cannot be met via a distribution package"},{"line_number":173,"context_line":"  should be implemented in a virtualenv. This prevents system package"},{"line_number":174,"context_line":"  conflicts and will hopefully reduce the host package installation"},{"line_number":175,"context_line":"  requirements."},{"line_number":176,"context_line":""},{"line_number":177,"context_line":"* Implement a playbook which is optionally used to prepare pre-built venvs"},{"line_number":178,"context_line":"  for an environment as they are today."}],"source_content_type":"text/x-rst","patch_set":1,"id":"7f287b81_d6f6c3d7","line":175,"range":{"start_line":170,"start_character":0,"end_line":175,"end_character":15},"in_reply_to":"7f287b81_b20a381d","updated":"2017-09-07 10:32:17.000000000","message":"This is a bit if an incomplete statement, so I understand the confusion.\n\nEssentially the goal would be to implement all our python package bits into venvs exclusively, then try to only ever have python-virtualenv installed on the host as a distro package. The idea is to try and do that.\n\nEvery Ansible task which requires a host package will then be set to use the virtualenv that we ship to every host which contains the bits needed for Ansible.\n\nA note on Giftwrap - I definitely think it\u0027s worth discussing the option of building packages in the build stage, then consuming them for the deployment. Giftwrap does some clever things which we could use, but then we ruin our developer mode story. However, it could results in quite a bit of simplification in roles if we manage build requirements elsewhere and the roles could potentially then use distro package for openstack if the deployer wants to which might be a nice feature.","commit_id":"550382eb285ed848fddb76b6d6126957ebfcb7fb"},{"author":{"_account_id":17068,"name":"Jean-Philippe Evrard","email":"openstack@a.spamming.party","username":"evrardjp"},"change_message_id":"2954af910104351e4c46a8b6e668126528e22c14","unresolved":false,"context_lines":[{"line_number":174,"context_line":"  conflicts and will hopefully reduce the host package installation"},{"line_number":175,"context_line":"  requirements."},{"line_number":176,"context_line":""},{"line_number":177,"context_line":"* Implement a playbook which is optionally used to prepare pre-built venvs"},{"line_number":178,"context_line":"  for an environment as they are today."},{"line_number":179,"context_line":""},{"line_number":180,"context_line":"* Remove the complex git caching/staging process which exists today and"},{"line_number":181,"context_line":"  make the use of the repo server for git caching entirely optional."}],"source_content_type":"text/x-rst","patch_set":1,"id":"7f287b81_2525c4c2","line":178,"range":{"start_line":177,"start_character":0,"end_line":178,"end_character":39},"updated":"2017-09-07 15:49:35.000000000","message":"What would be great is that we explain too where this could trigger in the process.\n\nI think I\u0027d put that early in the deploy, and replace our current repo-build playbook with what\u0027s written two points below: the staging.","commit_id":"550382eb285ed848fddb76b6d6126957ebfcb7fb"},{"author":{"_account_id":6816,"name":"Jesse Pretorius","email":"jesse@odyssey4.me","username":"jesse-pretorius"},"change_message_id":"87d561a5779c37d814a2dea72083fd0d6355483c","unresolved":false,"context_lines":[{"line_number":174,"context_line":"  conflicts and will hopefully reduce the host package installation"},{"line_number":175,"context_line":"  requirements."},{"line_number":176,"context_line":""},{"line_number":177,"context_line":"* Implement a playbook which is optionally used to prepare pre-built venvs"},{"line_number":178,"context_line":"  for an environment as they are today."},{"line_number":179,"context_line":""},{"line_number":180,"context_line":"* Remove the complex git caching/staging process which exists today and"},{"line_number":181,"context_line":"  make the use of the repo server for git caching entirely optional."}],"source_content_type":"text/x-rst","patch_set":1,"id":"5f2577a7_7471500c","line":178,"range":{"start_line":177,"start_character":0,"end_line":178,"end_character":39},"in_reply_to":"7f287b81_2525c4c2","updated":"2017-09-21 06:43:22.000000000","message":"Done","commit_id":"550382eb285ed848fddb76b6d6126957ebfcb7fb"},{"author":{"_account_id":17068,"name":"Jean-Philippe Evrard","email":"openstack@a.spamming.party","username":"evrardjp"},"change_message_id":"2954af910104351e4c46a8b6e668126528e22c14","unresolved":false,"context_lines":[{"line_number":177,"context_line":"* Implement a playbook which is optionally used to prepare pre-built venvs"},{"line_number":178,"context_line":"  for an environment as they are today."},{"line_number":179,"context_line":""},{"line_number":180,"context_line":"* Remove the complex git caching/staging process which exists today and"},{"line_number":181,"context_line":"  make the use of the repo server for git caching entirely optional."},{"line_number":182,"context_line":"  Implement a replacement process which is simpler and more intuitive."},{"line_number":183,"context_line":""},{"line_number":184,"context_line":"* Implement a playbook which can be used to stage offline installs by"},{"line_number":185,"context_line":"  downloading all built artifacts (completed, perhaps by a CI job) to the"}],"source_content_type":"text/x-rst","patch_set":1,"id":"7f287b81_cb8a9162","line":182,"range":{"start_line":180,"start_character":1,"end_line":182,"end_character":70},"updated":"2017-09-07 15:49:35.000000000","message":"For deployers that would opt out of the repo building, I guess that the explicit git clone could be removed, as the role could pip install into the venv with the git link directly. Right? Is that what you meant?\n\nFor deployers that would opt-in for the repo building (whether partial or complete), would a git staging still be needed if we go for the \"using the role\" approach?\nI am not sure how do you see this panning out, in the repo build case (partial or full).","commit_id":"550382eb285ed848fddb76b6d6126957ebfcb7fb"},{"author":{"_account_id":6816,"name":"Jesse Pretorius","email":"jesse@odyssey4.me","username":"jesse-pretorius"},"change_message_id":"87d561a5779c37d814a2dea72083fd0d6355483c","unresolved":false,"context_lines":[{"line_number":177,"context_line":"* Implement a playbook which is optionally used to prepare pre-built venvs"},{"line_number":178,"context_line":"  for an environment as they are today."},{"line_number":179,"context_line":""},{"line_number":180,"context_line":"* Remove the complex git caching/staging process which exists today and"},{"line_number":181,"context_line":"  make the use of the repo server for git caching entirely optional."},{"line_number":182,"context_line":"  Implement a replacement process which is simpler and more intuitive."},{"line_number":183,"context_line":""},{"line_number":184,"context_line":"* Implement a playbook which can be used to stage offline installs by"},{"line_number":185,"context_line":"  downloading all built artifacts (completed, perhaps by a CI job) to the"}],"source_content_type":"text/x-rst","patch_set":1,"id":"5f2577a7_9474bcff","line":182,"range":{"start_line":180,"start_character":1,"end_line":182,"end_character":70},"in_reply_to":"7f287b81_cb8a9162","updated":"2017-09-21 06:43:22.000000000","message":"At this stage I\u0027m not exactly sure how this will pan out. It\u0027s an implementation detail which I can perhaps add here once we\u0027ve settled on a final solution.","commit_id":"550382eb285ed848fddb76b6d6126957ebfcb7fb"},{"author":{"_account_id":17068,"name":"Jean-Philippe Evrard","email":"openstack@a.spamming.party","username":"evrardjp"},"change_message_id":"2954af910104351e4c46a8b6e668126528e22c14","unresolved":false,"context_lines":[{"line_number":181,"context_line":"  make the use of the repo server for git caching entirely optional."},{"line_number":182,"context_line":"  Implement a replacement process which is simpler and more intuitive."},{"line_number":183,"context_line":""},{"line_number":184,"context_line":"* Implement a playbook which can be used to stage offline installs by"},{"line_number":185,"context_line":"  downloading all built artifacts (completed, perhaps by a CI job) to the"},{"line_number":186,"context_line":"  deployment host, then distributing them appropriately."},{"line_number":187,"context_line":""},{"line_number":188,"context_line":".. _pypiserver: https://pypiserver.readthedocs.io"},{"line_number":189,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"7f287b81_cb4df13b","line":186,"range":{"start_line":184,"start_character":0,"end_line":186,"end_character":56},"updated":"2017-09-07 15:49:35.000000000","message":"That would be to fill in the repo server with the repo build venvs right?\n\nOr also the wheels (for the record). \n\nIf a deployer choses to opt-in the repo build, but still do everything in one step (like we do now), would that step be taking care of spreading the repo build items to the repo server nodes?","commit_id":"550382eb285ed848fddb76b6d6126957ebfcb7fb"},{"author":{"_account_id":6816,"name":"Jesse Pretorius","email":"jesse@odyssey4.me","username":"jesse-pretorius"},"change_message_id":"87d561a5779c37d814a2dea72083fd0d6355483c","unresolved":false,"context_lines":[{"line_number":181,"context_line":"  make the use of the repo server for git caching entirely optional."},{"line_number":182,"context_line":"  Implement a replacement process which is simpler and more intuitive."},{"line_number":183,"context_line":""},{"line_number":184,"context_line":"* Implement a playbook which can be used to stage offline installs by"},{"line_number":185,"context_line":"  downloading all built artifacts (completed, perhaps by a CI job) to the"},{"line_number":186,"context_line":"  deployment host, then distributing them appropriately."},{"line_number":187,"context_line":""},{"line_number":188,"context_line":".. _pypiserver: https://pypiserver.readthedocs.io"},{"line_number":189,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"5f2577a7_349ce80d","line":186,"range":{"start_line":184,"start_character":0,"end_line":186,"end_character":56},"in_reply_to":"7f287b81_cb4df13b","updated":"2017-09-21 06:43:22.000000000","message":"At this stage I\u0027m not exactly sure how this will pan out. It\u0027s an implementation detail which I can perhaps add here once we\u0027ve settled on a final solution.","commit_id":"550382eb285ed848fddb76b6d6126957ebfcb7fb"},{"author":{"_account_id":6816,"name":"Jesse Pretorius","email":"jesse@odyssey4.me","username":"jesse-pretorius"},"change_message_id":"84c2f75655604fd1f7137afe90eb6d3765541c82","unresolved":false,"context_lines":[{"line_number":186,"context_line":"  deployment host, then distributing them appropriately."},{"line_number":187,"context_line":""},{"line_number":188,"context_line":".. _pypiserver: https://pypiserver.readthedocs.io"},{"line_number":189,"context_line":""},{"line_number":190,"context_line":"Alternatives"},{"line_number":191,"context_line":"------------"},{"line_number":192,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"7f287b81_a5f09468","line":189,"updated":"2017-09-07 15:08:04.000000000","message":"Add:\n\nSimplify the constraints management by implementing the use of --constraints in the following order (pip uses the first one it finds for the requirement):\n\n--constraint user-specified-constraints.txt\n--constraint openstack-ansible-pins.txt\n--constraint openstack-upper-constraints.txt\n\nThis would replace the current method which merges the various constraints into one file, requiring a fair amount of jinja magic because a single file cannot have two constraints and resolve successfully into a single result as we need in our current mechanism.\n\nWe\u0027d have to think about where the git-based builds fit into this. Perhaps they should be first (in user constraints) or perhaps they should be another layer (just before openstack-upper-constraints)?","commit_id":"550382eb285ed848fddb76b6d6126957ebfcb7fb"},{"author":{"_account_id":14805,"name":"Jimmy McCrory","email":"jimmy.mccrory@gmail.com","username":"jimmy-mccrory"},"change_message_id":"1110995f705ff40ccccebf4ee891812a3bda6101","unresolved":false,"context_lines":[{"line_number":195,"context_line":"* Implement a playbook which is optionally used to prepare pre-built venvs"},{"line_number":196,"context_line":"  for an environment as they are today. If a deployer wishes to prepare"},{"line_number":197,"context_line":"  the venvs in a build process, the playbook should be exercised in the"},{"line_number":198,"context_line":"  build process and should be executed on a designated \u0027build host\u0027 which"},{"line_number":199,"context_line":"  will make use of virtual machines to exercise the builds."},{"line_number":200,"context_line":""},{"line_number":201,"context_line":"* Remove the complex git caching/staging process which exists today and"},{"line_number":202,"context_line":"  make the use of the repo server for git caching entirely optional."}],"source_content_type":"text/x-rst","patch_set":3,"id":"9f5c4f37_c6aa1718","line":199,"range":{"start_line":198,"start_character":67,"end_line":199,"end_character":59},"updated":"2017-09-21 06:09:56.000000000","message":"Not sure where the virtual machines come in, and they\u0027re separate from the designated build host?","commit_id":"aae98f0bcb1890b8fc48e633239f1da2083e6799"},{"author":{"_account_id":6816,"name":"Jesse Pretorius","email":"jesse@odyssey4.me","username":"jesse-pretorius"},"change_message_id":"87d561a5779c37d814a2dea72083fd0d6355483c","unresolved":false,"context_lines":[{"line_number":195,"context_line":"* Implement a playbook which is optionally used to prepare pre-built venvs"},{"line_number":196,"context_line":"  for an environment as they are today. If a deployer wishes to prepare"},{"line_number":197,"context_line":"  the venvs in a build process, the playbook should be exercised in the"},{"line_number":198,"context_line":"  build process and should be executed on a designated \u0027build host\u0027 which"},{"line_number":199,"context_line":"  will make use of virtual machines to exercise the builds."},{"line_number":200,"context_line":""},{"line_number":201,"context_line":"* Remove the complex git caching/staging process which exists today and"},{"line_number":202,"context_line":"  make the use of the repo server for git caching entirely optional."}],"source_content_type":"text/x-rst","patch_set":3,"id":"9f5c4f37_c958e857","line":199,"range":{"start_line":198,"start_character":67,"end_line":199,"end_character":59},"in_reply_to":"9f5c4f37_c6aa1718","updated":"2017-09-21 06:43:22.000000000","message":"I\u0027ve made an edit to clarify this - hope that helps!","commit_id":"aae98f0bcb1890b8fc48e633239f1da2083e6799"},{"author":{"_account_id":14805,"name":"Jimmy McCrory","email":"jimmy.mccrory@gmail.com","username":"jimmy-mccrory"},"change_message_id":"55c9097abfb3c4c821c91a1523f5c9fe1dd4ae46","unresolved":false,"context_lines":[{"line_number":195,"context_line":"* Implement a playbook which is optionally used to prepare pre-built venvs"},{"line_number":196,"context_line":"  for an environment as they are today. If a deployer wishes to prepare"},{"line_number":197,"context_line":"  the venvs in a build process, the playbook should be exercised in the"},{"line_number":198,"context_line":"  build process and should be executed on a designated \u0027build host\u0027 which"},{"line_number":199,"context_line":"  will make use of virtual machines to exercise the builds."},{"line_number":200,"context_line":""},{"line_number":201,"context_line":"* Remove the complex git caching/staging process which exists today and"},{"line_number":202,"context_line":"  make the use of the repo server for git caching entirely optional."}],"source_content_type":"text/x-rst","patch_set":3,"id":"7f515b1d_3cd0de50","line":199,"range":{"start_line":198,"start_character":67,"end_line":199,"end_character":59},"in_reply_to":"9f5c4f37_c958e857","updated":"2017-09-22 03:57:36.000000000","message":"Thanks, interesting approach.","commit_id":"aae98f0bcb1890b8fc48e633239f1da2083e6799"},{"author":{"_account_id":17068,"name":"Jean-Philippe Evrard","email":"openstack@a.spamming.party","username":"evrardjp"},"change_message_id":"5e8a4d0a51b51424969d21af4c5bf3080412ca73","unresolved":false,"context_lines":[{"line_number":56,"context_line":"building is done asynchronously in order to improve the time to completion,"},{"line_number":57,"context_line":"however individual asynchronous tasks cannot be retried in Ansible, and the"},{"line_number":58,"context_line":"git clones commonly fail. This is an Ansible limitation which we could work"},{"line_number":59,"context_line":"around by implementing our own action module, but this would increase the"},{"line_number":60,"context_line":"technical debt as we would have to constantly keep the module code updated"},{"line_number":61,"context_line":"as we update to later versions of Ansible."},{"line_number":62,"context_line":""}],"source_content_type":"text/x-rst","patch_set":4,"id":"7f515b1d_545a1c2a","line":59,"updated":"2017-09-25 13:58:31.000000000","message":"would likely increase","commit_id":"ac28e66ab472e16fad40313324d364a37e379bf7"},{"author":{"_account_id":17068,"name":"Jean-Philippe Evrard","email":"openstack@a.spamming.party","username":"evrardjp"},"change_message_id":"50f27b3f3509a568000cd3ad7383646dd201bc52","unresolved":false,"context_lines":[{"line_number":56,"context_line":"building is done asynchronously in order to improve the time to completion,"},{"line_number":57,"context_line":"however individual asynchronous tasks cannot be retried in Ansible, and the"},{"line_number":58,"context_line":"git clones commonly fail. This is an Ansible limitation which we could work"},{"line_number":59,"context_line":"around by implementing our own action module, but this would increase the"},{"line_number":60,"context_line":"technical debt as we would have to constantly keep the module code updated"},{"line_number":61,"context_line":"as we update to later versions of Ansible."},{"line_number":62,"context_line":""}],"source_content_type":"text/x-rst","patch_set":4,"id":"7f515b1d_7855ed1c","line":59,"in_reply_to":"7f515b1d_17514663","updated":"2017-09-26 09:09:24.000000000","message":"you can\u0027t say for certain that ansible will change their interfaces.\nWhat we can say is that it\u0027s probable to happen. \n\nBut that\u0027s detail, let\u0027s keep it at way.","commit_id":"ac28e66ab472e16fad40313324d364a37e379bf7"},{"author":{"_account_id":6816,"name":"Jesse Pretorius","email":"jesse@odyssey4.me","username":"jesse-pretorius"},"change_message_id":"b52e8b3b246782018422949ca70416a41ac08dbd","unresolved":false,"context_lines":[{"line_number":56,"context_line":"building is done asynchronously in order to improve the time to completion,"},{"line_number":57,"context_line":"however individual asynchronous tasks cannot be retried in Ansible, and the"},{"line_number":58,"context_line":"git clones commonly fail. This is an Ansible limitation which we could work"},{"line_number":59,"context_line":"around by implementing our own action module, but this would increase the"},{"line_number":60,"context_line":"technical debt as we would have to constantly keep the module code updated"},{"line_number":61,"context_line":"as we update to later versions of Ansible."},{"line_number":62,"context_line":""}],"source_content_type":"text/x-rst","patch_set":4,"id":"7f515b1d_17514663","line":59,"in_reply_to":"7f515b1d_545a1c2a","updated":"2017-09-25 14:29:24.000000000","message":"likely decreases certainty - I am certain it would increase technical debt","commit_id":"ac28e66ab472e16fad40313324d364a37e379bf7"},{"author":{"_account_id":17068,"name":"Jean-Philippe Evrard","email":"openstack@a.spamming.party","username":"evrardjp"},"change_message_id":"5e8a4d0a51b51424969d21af4c5bf3080412ca73","unresolved":false,"context_lines":[{"line_number":57,"context_line":"however individual asynchronous tasks cannot be retried in Ansible, and the"},{"line_number":58,"context_line":"git clones commonly fail. This is an Ansible limitation which we could work"},{"line_number":59,"context_line":"around by implementing our own action module, but this would increase the"},{"line_number":60,"context_line":"technical debt as we would have to constantly keep the module code updated"},{"line_number":61,"context_line":"as we update to later versions of Ansible."},{"line_number":62,"context_line":""},{"line_number":63,"context_line":"When building wheels, pip has no way of resolving all dependencies up-front."}],"source_content_type":"text/x-rst","patch_set":4,"id":"7f515b1d_34afa0e4","line":60,"range":{"start_line":60,"start_character":55,"end_line":60,"end_character":61},"updated":"2017-09-25 13:58:31.000000000","message":"we would have to constantly update the action plugin code if any user interface to the git module or ansible\u0027s action plugin interface changes.","commit_id":"ac28e66ab472e16fad40313324d364a37e379bf7"},{"author":{"_account_id":17068,"name":"Jean-Philippe Evrard","email":"openstack@a.spamming.party","username":"evrardjp"},"change_message_id":"50f27b3f3509a568000cd3ad7383646dd201bc52","unresolved":false,"context_lines":[{"line_number":57,"context_line":"however individual asynchronous tasks cannot be retried in Ansible, and the"},{"line_number":58,"context_line":"git clones commonly fail. This is an Ansible limitation which we could work"},{"line_number":59,"context_line":"around by implementing our own action module, but this would increase the"},{"line_number":60,"context_line":"technical debt as we would have to constantly keep the module code updated"},{"line_number":61,"context_line":"as we update to later versions of Ansible."},{"line_number":62,"context_line":""},{"line_number":63,"context_line":"When building wheels, pip has no way of resolving all dependencies up-front."}],"source_content_type":"text/x-rst","patch_set":4,"id":"7f515b1d_d82501c4","line":60,"range":{"start_line":60,"start_character":55,"end_line":60,"end_character":61},"in_reply_to":"7f515b1d_17d6e6c4","updated":"2017-09-26 09:09:24.000000000","message":"This was more for an explanation of the cause of technical debt, for making sure it\u0027s not feeling but science :p\nIf that\u0027s not readable, let\u0027s skip it. This conversation is good enough of a record.","commit_id":"ac28e66ab472e16fad40313324d364a37e379bf7"},{"author":{"_account_id":6816,"name":"Jesse Pretorius","email":"jesse@odyssey4.me","username":"jesse-pretorius"},"change_message_id":"b52e8b3b246782018422949ca70416a41ac08dbd","unresolved":false,"context_lines":[{"line_number":57,"context_line":"however individual asynchronous tasks cannot be retried in Ansible, and the"},{"line_number":58,"context_line":"git clones commonly fail. This is an Ansible limitation which we could work"},{"line_number":59,"context_line":"around by implementing our own action module, but this would increase the"},{"line_number":60,"context_line":"technical debt as we would have to constantly keep the module code updated"},{"line_number":61,"context_line":"as we update to later versions of Ansible."},{"line_number":62,"context_line":""},{"line_number":63,"context_line":"When building wheels, pip has no way of resolving all dependencies up-front."}],"source_content_type":"text/x-rst","patch_set":4,"id":"7f515b1d_17d6e6c4","line":60,"range":{"start_line":60,"start_character":55,"end_line":60,"end_character":61},"in_reply_to":"7f515b1d_34afa0e4","updated":"2017-09-25 14:29:24.000000000","message":"that is a very confusing sentence, and I don\u0027t believe that it adds any value above what\u0027s already there","commit_id":"ac28e66ab472e16fad40313324d364a37e379bf7"},{"author":{"_account_id":17068,"name":"Jean-Philippe Evrard","email":"openstack@a.spamming.party","username":"evrardjp"},"change_message_id":"5e8a4d0a51b51424969d21af4c5bf3080412ca73","unresolved":false,"context_lines":[{"line_number":120,"context_line":"~~~~~~~"},{"line_number":121,"context_line":"Once the wheels, venvs and other artifacts are built for an environment they"},{"line_number":122,"context_line":"are stored and synchronised between the repo containers using a combination of"},{"line_number":123,"context_line":"rsync and lsyncd. While this sync process is generally OK, it is commonly a"},{"line_number":124,"context_line":"cause for confusion and requires a complex troubleshooting process to figure"},{"line_number":125,"context_line":"out why packages are not present."},{"line_number":126,"context_line":""}],"source_content_type":"text/x-rst","patch_set":4,"id":"7f515b1d_143704e6","line":123,"updated":"2017-09-25 13:58:31.000000000","message":"\"While this sync process is generally OK,\"\n-\u003e\n\"While this sync process is generally OK, its asynchronous and background nature is commonly a cause for confusion...\"","commit_id":"ac28e66ab472e16fad40313324d364a37e379bf7"},{"author":{"_account_id":6816,"name":"Jesse Pretorius","email":"jesse@odyssey4.me","username":"jesse-pretorius"},"change_message_id":"b52e8b3b246782018422949ca70416a41ac08dbd","unresolved":false,"context_lines":[{"line_number":120,"context_line":"~~~~~~~"},{"line_number":121,"context_line":"Once the wheels, venvs and other artifacts are built for an environment they"},{"line_number":122,"context_line":"are stored and synchronised between the repo containers using a combination of"},{"line_number":123,"context_line":"rsync and lsyncd. While this sync process is generally OK, it is commonly a"},{"line_number":124,"context_line":"cause for confusion and requires a complex troubleshooting process to figure"},{"line_number":125,"context_line":"out why packages are not present."},{"line_number":126,"context_line":""}],"source_content_type":"text/x-rst","patch_set":4,"id":"7f515b1d_f7fd8a3a","line":123,"in_reply_to":"7f515b1d_143704e6","updated":"2017-09-25 14:29:24.000000000","message":"I could add that, but it\u0027s adding unnecessary detail IMO.","commit_id":"ac28e66ab472e16fad40313324d364a37e379bf7"},{"author":{"_account_id":17068,"name":"Jean-Philippe Evrard","email":"openstack@a.spamming.party","username":"evrardjp"},"change_message_id":"50f27b3f3509a568000cd3ad7383646dd201bc52","unresolved":false,"context_lines":[{"line_number":120,"context_line":"~~~~~~~"},{"line_number":121,"context_line":"Once the wheels, venvs and other artifacts are built for an environment they"},{"line_number":122,"context_line":"are stored and synchronised between the repo containers using a combination of"},{"line_number":123,"context_line":"rsync and lsyncd. While this sync process is generally OK, it is commonly a"},{"line_number":124,"context_line":"cause for confusion and requires a complex troubleshooting process to figure"},{"line_number":125,"context_line":"out why packages are not present."},{"line_number":126,"context_line":""}],"source_content_type":"text/x-rst","patch_set":4,"id":"7f515b1d_1815d94a","line":123,"in_reply_to":"7f515b1d_f7fd8a3a","updated":"2017-09-26 09:09:24.000000000","message":"ok.","commit_id":"ac28e66ab472e16fad40313324d364a37e379bf7"},{"author":{"_account_id":17068,"name":"Jean-Philippe Evrard","email":"openstack@a.spamming.party","username":"evrardjp"},"change_message_id":"5e8a4d0a51b51424969d21af4c5bf3080412ca73","unresolved":false,"context_lines":[{"line_number":161,"context_line":"  This directory will be served via the pypi API provided by the very simple"},{"line_number":162,"context_line":"  `pypiserver`_ application. If we need to continue to provide per-distro"},{"line_number":163,"context_line":"  or per-architecture wheels then we could implement distro/arch indexes"},{"line_number":164,"context_line":"  which are supplied by individual folders. However, it is unlikely that"},{"line_number":165,"context_line":"  this will be necessary."},{"line_number":166,"context_line":""},{"line_number":167,"context_line":"* Use nginx as a reverse proxy which responds to requests from pip by first"},{"line_number":168,"context_line":"  trying against the local pypiserver, then against tarballs.openstack.org"}],"source_content_type":"text/x-rst","patch_set":4,"id":"7f515b1d_34220054","line":165,"range":{"start_line":164,"start_character":44,"end_line":165,"end_character":25},"updated":"2017-09-25 13:58:31.000000000","message":"Why is this unlikely? It makes sense to me to have wheels per arch, unless they are universal.\n\nBut I understand that in this case the wheels would be built into a folder on a machine that\u0027s matching the architecture, and all the matching nodes of the same architecture could feed from those.\n\nWas that what you meant?","commit_id":"ac28e66ab472e16fad40313324d364a37e379bf7"},{"author":{"_account_id":6816,"name":"Jesse Pretorius","email":"jesse@odyssey4.me","username":"jesse-pretorius"},"change_message_id":"b52e8b3b246782018422949ca70416a41ac08dbd","unresolved":false,"context_lines":[{"line_number":161,"context_line":"  This directory will be served via the pypi API provided by the very simple"},{"line_number":162,"context_line":"  `pypiserver`_ application. If we need to continue to provide per-distro"},{"line_number":163,"context_line":"  or per-architecture wheels then we could implement distro/arch indexes"},{"line_number":164,"context_line":"  which are supplied by individual folders. However, it is unlikely that"},{"line_number":165,"context_line":"  this will be necessary."},{"line_number":166,"context_line":""},{"line_number":167,"context_line":"* Use nginx as a reverse proxy which responds to requests from pip by first"},{"line_number":168,"context_line":"  trying against the local pypiserver, then against tarballs.openstack.org"}],"source_content_type":"text/x-rst","patch_set":4,"id":"7f515b1d_b7bef262","line":165,"range":{"start_line":164,"start_character":44,"end_line":165,"end_character":25},"in_reply_to":"7f515b1d_34220054","updated":"2017-09-25 14:29:24.000000000","message":"What I mean is that *if* we have to have seperate folders for different distributions and/or architectures, then we will do that, but it is unlikely that it will be necessary.","commit_id":"ac28e66ab472e16fad40313324d364a37e379bf7"},{"author":{"_account_id":17068,"name":"Jean-Philippe Evrard","email":"openstack@a.spamming.party","username":"evrardjp"},"change_message_id":"50f27b3f3509a568000cd3ad7383646dd201bc52","unresolved":false,"context_lines":[{"line_number":161,"context_line":"  This directory will be served via the pypi API provided by the very simple"},{"line_number":162,"context_line":"  `pypiserver`_ application. If we need to continue to provide per-distro"},{"line_number":163,"context_line":"  or per-architecture wheels then we could implement distro/arch indexes"},{"line_number":164,"context_line":"  which are supplied by individual folders. However, it is unlikely that"},{"line_number":165,"context_line":"  this will be necessary."},{"line_number":166,"context_line":""},{"line_number":167,"context_line":"* Use nginx as a reverse proxy which responds to requests from pip by first"},{"line_number":168,"context_line":"  trying against the local pypiserver, then against tarballs.openstack.org"}],"source_content_type":"text/x-rst","patch_set":4,"id":"7f515b1d_d80e213a","line":165,"range":{"start_line":164,"start_character":44,"end_line":165,"end_character":25},"in_reply_to":"7f515b1d_b7bef262","updated":"2017-09-26 09:09:24.000000000","message":"clear enough with this comment, no need to update the spec.","commit_id":"ac28e66ab472e16fad40313324d364a37e379bf7"},{"author":{"_account_id":17068,"name":"Jean-Philippe Evrard","email":"openstack@a.spamming.party","username":"evrardjp"},"change_message_id":"5e8a4d0a51b51424969d21af4c5bf3080412ca73","unresolved":false,"context_lines":[{"line_number":164,"context_line":"  which are supplied by individual folders. However, it is unlikely that"},{"line_number":165,"context_line":"  this will be necessary."},{"line_number":166,"context_line":""},{"line_number":167,"context_line":"* Use nginx as a reverse proxy which responds to requests from pip by first"},{"line_number":168,"context_line":"  trying against the local pypiserver, then against tarballs.openstack.org"},{"line_number":169,"context_line":"  and then against pypi. This will allow nginx to cache all downloaded"},{"line_number":170,"context_line":"  packages (speeding up subsequent requests) without the repo server having"},{"line_number":171,"context_line":"  build them."},{"line_number":172,"context_line":""},{"line_number":173,"context_line":"* Implement changes to the roles to allow service-specific constraints to"},{"line_number":174,"context_line":"  be applied when building venvs. This allows us to publish the last known"}],"source_content_type":"text/x-rst","patch_set":4,"id":"7f515b1d_9492d44a","line":171,"range":{"start_line":167,"start_character":2,"end_line":171,"end_character":13},"updated":"2017-09-25 13:58:31.000000000","message":"I\u0027d rather see this as a default behavior, and if ONE var is set (like  offline_mode\u003dTrue), we would remove this behavior, and use a full local thing explained in paragraph 1. Is that what you are suggesting? \n\nHow would that work:\n- Producing all wheels on the deploy node for all the architectures\n- Publish them on an offline local repo server\n\nHave I understood you correctly?\n\nI\u0027d say yes, according to the 4th paragraph after this one, but I\u0027d like these two to be linked in some way.","commit_id":"ac28e66ab472e16fad40313324d364a37e379bf7"},{"author":{"_account_id":17068,"name":"Jean-Philippe Evrard","email":"openstack@a.spamming.party","username":"evrardjp"},"change_message_id":"50f27b3f3509a568000cd3ad7383646dd201bc52","unresolved":false,"context_lines":[{"line_number":164,"context_line":"  which are supplied by individual folders. However, it is unlikely that"},{"line_number":165,"context_line":"  this will be necessary."},{"line_number":166,"context_line":""},{"line_number":167,"context_line":"* Use nginx as a reverse proxy which responds to requests from pip by first"},{"line_number":168,"context_line":"  trying against the local pypiserver, then against tarballs.openstack.org"},{"line_number":169,"context_line":"  and then against pypi. This will allow nginx to cache all downloaded"},{"line_number":170,"context_line":"  packages (speeding up subsequent requests) without the repo server having"},{"line_number":171,"context_line":"  build them."},{"line_number":172,"context_line":""},{"line_number":173,"context_line":"* Implement changes to the roles to allow service-specific constraints to"},{"line_number":174,"context_line":"  be applied when building venvs. This allows us to publish the last known"}],"source_content_type":"text/x-rst","patch_set":4,"id":"7f515b1d_5b00eb01","line":171,"range":{"start_line":167,"start_character":2,"end_line":171,"end_character":13},"in_reply_to":"7f515b1d_573e9eda","updated":"2017-09-26 09:09:24.000000000","message":"ok. Let\u0027s see that in reviews.","commit_id":"ac28e66ab472e16fad40313324d364a37e379bf7"},{"author":{"_account_id":6816,"name":"Jesse Pretorius","email":"jesse@odyssey4.me","username":"jesse-pretorius"},"change_message_id":"b52e8b3b246782018422949ca70416a41ac08dbd","unresolved":false,"context_lines":[{"line_number":164,"context_line":"  which are supplied by individual folders. However, it is unlikely that"},{"line_number":165,"context_line":"  this will be necessary."},{"line_number":166,"context_line":""},{"line_number":167,"context_line":"* Use nginx as a reverse proxy which responds to requests from pip by first"},{"line_number":168,"context_line":"  trying against the local pypiserver, then against tarballs.openstack.org"},{"line_number":169,"context_line":"  and then against pypi. This will allow nginx to cache all downloaded"},{"line_number":170,"context_line":"  packages (speeding up subsequent requests) without the repo server having"},{"line_number":171,"context_line":"  build them."},{"line_number":172,"context_line":""},{"line_number":173,"context_line":"* Implement changes to the roles to allow service-specific constraints to"},{"line_number":174,"context_line":"  be applied when building venvs. This allows us to publish the last known"}],"source_content_type":"text/x-rst","patch_set":4,"id":"7f515b1d_573e9eda","line":171,"range":{"start_line":167,"start_character":2,"end_line":171,"end_character":13},"in_reply_to":"7f515b1d_9492d44a","updated":"2017-09-25 14:29:24.000000000","message":"No, it will always work like this. If the installation is offline then the upstream connections just will not work and all the wheels will have to be local, which means the build process will have to be implemented.\n\nAt this time this is all theory though, I\u0027m trying to state intent here instead of getting too much into implementation details.","commit_id":"ac28e66ab472e16fad40313324d364a37e379bf7"},{"author":{"_account_id":17068,"name":"Jean-Philippe Evrard","email":"openstack@a.spamming.party","username":"evrardjp"},"change_message_id":"5e8a4d0a51b51424969d21af4c5bf3080412ca73","unresolved":false,"context_lines":[{"line_number":171,"context_line":"  build them."},{"line_number":172,"context_line":""},{"line_number":173,"context_line":"* Implement changes to the roles to allow service-specific constraints to"},{"line_number":174,"context_line":"  be applied when building venvs. This allows us to publish the last known"},{"line_number":175,"context_line":"  good set of packages implemented in unconstrained venvs (eg: tempest,"},{"line_number":176,"context_line":"  rally, gnocchi) in OpenStack-CI to ensure consistency in deployments"},{"line_number":177,"context_line":"  when they\u0027re done in production."}],"source_content_type":"text/x-rst","patch_set":4,"id":"7f515b1d_94bbf4c2","line":174,"updated":"2017-09-25 13:58:31.000000000","message":"How is the link between applying new constraints and publishing the last known good set of packages?","commit_id":"ac28e66ab472e16fad40313324d364a37e379bf7"},{"author":{"_account_id":17068,"name":"Jean-Philippe Evrard","email":"openstack@a.spamming.party","username":"evrardjp"},"change_message_id":"50f27b3f3509a568000cd3ad7383646dd201bc52","unresolved":false,"context_lines":[{"line_number":171,"context_line":"  build them."},{"line_number":172,"context_line":""},{"line_number":173,"context_line":"* Implement changes to the roles to allow service-specific constraints to"},{"line_number":174,"context_line":"  be applied when building venvs. This allows us to publish the last known"},{"line_number":175,"context_line":"  good set of packages implemented in unconstrained venvs (eg: tempest,"},{"line_number":176,"context_line":"  rally, gnocchi) in OpenStack-CI to ensure consistency in deployments"},{"line_number":177,"context_line":"  when they\u0027re done in production."}],"source_content_type":"text/x-rst","patch_set":4,"id":"7f515b1d_7b184774","line":174,"in_reply_to":"7f515b1d_62cf96b0","updated":"2017-09-26 09:09:24.000000000","message":"There is no direct link between first and second sentence. There is at least an explanation about build artifacts that would record constraints of the pip packages used in a build and published somewhere?","commit_id":"ac28e66ab472e16fad40313324d364a37e379bf7"},{"author":{"_account_id":6816,"name":"Jesse Pretorius","email":"jesse@odyssey4.me","username":"jesse-pretorius"},"change_message_id":"b52e8b3b246782018422949ca70416a41ac08dbd","unresolved":false,"context_lines":[{"line_number":171,"context_line":"  build them."},{"line_number":172,"context_line":""},{"line_number":173,"context_line":"* Implement changes to the roles to allow service-specific constraints to"},{"line_number":174,"context_line":"  be applied when building venvs. This allows us to publish the last known"},{"line_number":175,"context_line":"  good set of packages implemented in unconstrained venvs (eg: tempest,"},{"line_number":176,"context_line":"  rally, gnocchi) in OpenStack-CI to ensure consistency in deployments"},{"line_number":177,"context_line":"  when they\u0027re done in production."}],"source_content_type":"text/x-rst","patch_set":4,"id":"7f515b1d_62cf96b0","line":174,"in_reply_to":"7f515b1d_94bbf4c2","updated":"2017-09-25 14:29:24.000000000","message":"I don\u0027t understand the question. Please clarify what you\u0027re asking here.","commit_id":"ac28e66ab472e16fad40313324d364a37e379bf7"},{"author":{"_account_id":17068,"name":"Jean-Philippe Evrard","email":"openstack@a.spamming.party","username":"evrardjp"},"change_message_id":"5e8a4d0a51b51424969d21af4c5bf3080412ca73","unresolved":false,"context_lines":[{"line_number":180,"context_line":"  building, but do it in such a way that only the build can be executed"},{"line_number":181,"context_line":"  by using tags, setting a specific flag, or include_role and tasks_from."},{"line_number":182,"context_line":"  The specific dependencies can then be itemised in the role and the role"},{"line_number":183,"context_line":"  can be used for artifact preparation."},{"line_number":184,"context_line":""},{"line_number":185,"context_line":"* Remove all pip install activities from hosts, replacing them with the"},{"line_number":186,"context_line":"  use of distro packages exclusively for any python requirements on the"}],"source_content_type":"text/x-rst","patch_set":4,"id":"7f515b1d_b406f085","line":183,"updated":"2017-09-25 13:58:31.000000000","message":"Or an alternative task file that is not on the main path, or conditionally on the main path.\n\nBut I guess that\u0027s what you meant with include_role and tasks_from.\n\n\nOn another topic:\nShould we care here about the delegation to a repo build node, that would, if empty, be the local node?","commit_id":"ac28e66ab472e16fad40313324d364a37e379bf7"},{"author":{"_account_id":17068,"name":"Jean-Philippe Evrard","email":"openstack@a.spamming.party","username":"evrardjp"},"change_message_id":"50f27b3f3509a568000cd3ad7383646dd201bc52","unresolved":false,"context_lines":[{"line_number":180,"context_line":"  building, but do it in such a way that only the build can be executed"},{"line_number":181,"context_line":"  by using tags, setting a specific flag, or include_role and tasks_from."},{"line_number":182,"context_line":"  The specific dependencies can then be itemised in the role and the role"},{"line_number":183,"context_line":"  can be used for artifact preparation."},{"line_number":184,"context_line":""},{"line_number":185,"context_line":"* Remove all pip install activities from hosts, replacing them with the"},{"line_number":186,"context_line":"  use of distro packages exclusively for any python requirements on the"}],"source_content_type":"text/x-rst","patch_set":4,"id":"7f515b1d_3b2c0f98","line":183,"in_reply_to":"7f515b1d_22f4fece","updated":"2017-09-26 09:09:24.000000000","message":"ok","commit_id":"ac28e66ab472e16fad40313324d364a37e379bf7"},{"author":{"_account_id":6816,"name":"Jesse Pretorius","email":"jesse@odyssey4.me","username":"jesse-pretorius"},"change_message_id":"b52e8b3b246782018422949ca70416a41ac08dbd","unresolved":false,"context_lines":[{"line_number":180,"context_line":"  building, but do it in such a way that only the build can be executed"},{"line_number":181,"context_line":"  by using tags, setting a specific flag, or include_role and tasks_from."},{"line_number":182,"context_line":"  The specific dependencies can then be itemised in the role and the role"},{"line_number":183,"context_line":"  can be used for artifact preparation."},{"line_number":184,"context_line":""},{"line_number":185,"context_line":"* Remove all pip install activities from hosts, replacing them with the"},{"line_number":186,"context_line":"  use of distro packages exclusively for any python requirements on the"}],"source_content_type":"text/x-rst","patch_set":4,"id":"7f515b1d_22f4fece","line":183,"in_reply_to":"7f515b1d_b406f085","updated":"2017-09-25 14:29:24.000000000","message":"These are implementation details which we\u0027ll need to flesh out in review. I don\u0027t think we need to continue speculating here where we don\u0027t have a practical sense of what works best.","commit_id":"ac28e66ab472e16fad40313324d364a37e379bf7"},{"author":{"_account_id":17068,"name":"Jean-Philippe Evrard","email":"openstack@a.spamming.party","username":"evrardjp"},"change_message_id":"5e8a4d0a51b51424969d21af4c5bf3080412ca73","unresolved":false,"context_lines":[{"line_number":200,"context_line":"  host to exercise the builds for the necessary distribution and architecture"},{"line_number":201,"context_line":"  combinations."},{"line_number":202,"context_line":""},{"line_number":203,"context_line":"* Remove the complex git caching/staging process which exists today and"},{"line_number":204,"context_line":"  make the use of the repo server for git caching entirely optional."},{"line_number":205,"context_line":"  Implement a replacement process which is simpler and more intuitive."},{"line_number":206,"context_line":""},{"line_number":207,"context_line":"* Implement a playbook which can be used to stage offline installs by"},{"line_number":208,"context_line":"  downloading all built artifacts (completed, perhaps by a CI job) to the"}],"source_content_type":"text/x-rst","patch_set":4,"id":"7f515b1d_3404404a","line":205,"range":{"start_line":203,"start_character":1,"end_line":205,"end_character":70},"updated":"2017-09-25 13:58:31.000000000","message":"Would this still be needed, if everything is fully built offline, or built just in time if online?","commit_id":"ac28e66ab472e16fad40313324d364a37e379bf7"},{"author":{"_account_id":6816,"name":"Jesse Pretorius","email":"jesse@odyssey4.me","username":"jesse-pretorius"},"change_message_id":"aecb69764a5165dbf16e2d640b93e5500cca50cf","unresolved":false,"context_lines":[{"line_number":200,"context_line":"  host to exercise the builds for the necessary distribution and architecture"},{"line_number":201,"context_line":"  combinations."},{"line_number":202,"context_line":""},{"line_number":203,"context_line":"* Remove the complex git caching/staging process which exists today and"},{"line_number":204,"context_line":"  make the use of the repo server for git caching entirely optional."},{"line_number":205,"context_line":"  Implement a replacement process which is simpler and more intuitive."},{"line_number":206,"context_line":""},{"line_number":207,"context_line":"* Implement a playbook which can be used to stage offline installs by"},{"line_number":208,"context_line":"  downloading all built artifacts (completed, perhaps by a CI job) to the"}],"source_content_type":"text/x-rst","patch_set":4,"id":"7f515b1d_81a71050","line":205,"range":{"start_line":203,"start_character":1,"end_line":205,"end_character":70},"in_reply_to":"7f515b1d_1b583330","updated":"2017-09-26 11:03:15.000000000","message":"OK, rethinking this - we won\u0027t *require* it, but if you\u0027re needing to do an offline install then we will still need it for novnc/spice. I don\u0027t think it\u0027ll be needed for anything else.","commit_id":"ac28e66ab472e16fad40313324d364a37e379bf7"},{"author":{"_account_id":6816,"name":"Jesse Pretorius","email":"jesse@odyssey4.me","username":"jesse-pretorius"},"change_message_id":"b52e8b3b246782018422949ca70416a41ac08dbd","unresolved":false,"context_lines":[{"line_number":200,"context_line":"  host to exercise the builds for the necessary distribution and architecture"},{"line_number":201,"context_line":"  combinations."},{"line_number":202,"context_line":""},{"line_number":203,"context_line":"* Remove the complex git caching/staging process which exists today and"},{"line_number":204,"context_line":"  make the use of the repo server for git caching entirely optional."},{"line_number":205,"context_line":"  Implement a replacement process which is simpler and more intuitive."},{"line_number":206,"context_line":""},{"line_number":207,"context_line":"* Implement a playbook which can be used to stage offline installs by"},{"line_number":208,"context_line":"  downloading all built artifacts (completed, perhaps by a CI job) to the"}],"source_content_type":"text/x-rst","patch_set":4,"id":"7f515b1d_c2bc0ade","line":205,"range":{"start_line":203,"start_character":1,"end_line":205,"end_character":70},"in_reply_to":"7f515b1d_3404404a","updated":"2017-09-25 14:29:24.000000000","message":"Some parts of it will likely be - for example, the novnc/spice repositories.","commit_id":"ac28e66ab472e16fad40313324d364a37e379bf7"},{"author":{"_account_id":17068,"name":"Jean-Philippe Evrard","email":"openstack@a.spamming.party","username":"evrardjp"},"change_message_id":"50f27b3f3509a568000cd3ad7383646dd201bc52","unresolved":false,"context_lines":[{"line_number":200,"context_line":"  host to exercise the builds for the necessary distribution and architecture"},{"line_number":201,"context_line":"  combinations."},{"line_number":202,"context_line":""},{"line_number":203,"context_line":"* Remove the complex git caching/staging process which exists today and"},{"line_number":204,"context_line":"  make the use of the repo server for git caching entirely optional."},{"line_number":205,"context_line":"  Implement a replacement process which is simpler and more intuitive."},{"line_number":206,"context_line":""},{"line_number":207,"context_line":"* Implement a playbook which can be used to stage offline installs by"},{"line_number":208,"context_line":"  downloading all built artifacts (completed, perhaps by a CI job) to the"}],"source_content_type":"text/x-rst","patch_set":4,"id":"7f515b1d_1b583330","line":205,"range":{"start_line":203,"start_character":1,"end_line":205,"end_character":70},"in_reply_to":"7f515b1d_c2bc0ade","updated":"2017-09-26 09:09:24.000000000","message":"Could you explain why?\nYou are just stating removal, and I didn\u0027t directly understand why we need to keep it.","commit_id":"ac28e66ab472e16fad40313324d364a37e379bf7"},{"author":{"_account_id":17068,"name":"Jean-Philippe Evrard","email":"openstack@a.spamming.party","username":"evrardjp"},"change_message_id":"5e8a4d0a51b51424969d21af4c5bf3080412ca73","unresolved":false,"context_lines":[{"line_number":210,"context_line":""},{"line_number":211,"context_line":"* Simplify the constraints management by implementing the use of"},{"line_number":212,"context_line":"  --constraints in the following order (pip uses the first one it finds for"},{"line_number":213,"context_line":"  the requirement):"},{"line_number":214,"context_line":""},{"line_number":215,"context_line":"  --constraint user-specified-constraints.txt"},{"line_number":216,"context_line":"  --constraint openstack-ansible-pins.txt"}],"source_content_type":"text/x-rst","patch_set":4,"id":"7f515b1d_342560c3","line":213,"updated":"2017-09-25 13:58:31.000000000","message":"I am not sure to understand this:\n\nif pip uses the first one as requirement, and the rest are the constraints we set, how would a package be properly installed?\n\nShouldn\u0027t this be -r \u003cproject\u003e/requirements.txt -c user-specified -c oa-pins -c upper-constraints?\n\nOr is the -r to be dropped too?\nI didn\u0027t really get the \"for the requirement\" part.","commit_id":"ac28e66ab472e16fad40313324d364a37e379bf7"},{"author":{"_account_id":17068,"name":"Jean-Philippe Evrard","email":"openstack@a.spamming.party","username":"evrardjp"},"change_message_id":"50f27b3f3509a568000cd3ad7383646dd201bc52","unresolved":false,"context_lines":[{"line_number":210,"context_line":""},{"line_number":211,"context_line":"* Simplify the constraints management by implementing the use of"},{"line_number":212,"context_line":"  --constraints in the following order (pip uses the first one it finds for"},{"line_number":213,"context_line":"  the requirement):"},{"line_number":214,"context_line":""},{"line_number":215,"context_line":"  --constraint user-specified-constraints.txt"},{"line_number":216,"context_line":"  --constraint openstack-ansible-pins.txt"}],"source_content_type":"text/x-rst","patch_set":4,"id":"7f515b1d_58647122","line":213,"in_reply_to":"7f515b1d_0278e227","updated":"2017-09-26 09:09:24.000000000","message":"I still don\u0027t understand the \"pip uses the first one it finds for the requirement\" in your sentence.","commit_id":"ac28e66ab472e16fad40313324d364a37e379bf7"},{"author":{"_account_id":6816,"name":"Jesse Pretorius","email":"jesse@odyssey4.me","username":"jesse-pretorius"},"change_message_id":"b52e8b3b246782018422949ca70416a41ac08dbd","unresolved":false,"context_lines":[{"line_number":210,"context_line":""},{"line_number":211,"context_line":"* Simplify the constraints management by implementing the use of"},{"line_number":212,"context_line":"  --constraints in the following order (pip uses the first one it finds for"},{"line_number":213,"context_line":"  the requirement):"},{"line_number":214,"context_line":""},{"line_number":215,"context_line":"  --constraint user-specified-constraints.txt"},{"line_number":216,"context_line":"  --constraint openstack-ansible-pins.txt"}],"source_content_type":"text/x-rst","patch_set":4,"id":"7f515b1d_0278e227","line":213,"in_reply_to":"7f515b1d_342560c3","updated":"2017-09-25 14:29:24.000000000","message":"This is only referring to the constraints parameters, not discussing the full set of pip parameters.","commit_id":"ac28e66ab472e16fad40313324d364a37e379bf7"},{"author":{"_account_id":6816,"name":"Jesse Pretorius","email":"jesse@odyssey4.me","username":"jesse-pretorius"},"change_message_id":"aecb69764a5165dbf16e2d640b93e5500cca50cf","unresolved":false,"context_lines":[{"line_number":210,"context_line":""},{"line_number":211,"context_line":"* Simplify the constraints management by implementing the use of"},{"line_number":212,"context_line":"  --constraints in the following order (pip uses the first one it finds for"},{"line_number":213,"context_line":"  the requirement):"},{"line_number":214,"context_line":""},{"line_number":215,"context_line":"  --constraint user-specified-constraints.txt"},{"line_number":216,"context_line":"  --constraint openstack-ansible-pins.txt"}],"source_content_type":"text/x-rst","patch_set":4,"id":"7f515b1d_a6810292","line":213,"in_reply_to":"7f515b1d_58647122","updated":"2017-09-26 11:03:15.000000000","message":"When doing the instruction:\n\npip install pyyaml -c constraintfile1.txt -c constraintfile2.txt -c constraintfile3.txt\n\npip will use the first constraint it finds for pyyaml (the requirement) in the constraint files given.","commit_id":"ac28e66ab472e16fad40313324d364a37e379bf7"},{"author":{"_account_id":17068,"name":"Jean-Philippe Evrard","email":"openstack@a.spamming.party","username":"evrardjp"},"change_message_id":"5e8a4d0a51b51424969d21af4c5bf3080412ca73","unresolved":false,"context_lines":[{"line_number":227,"context_line":"  packages will be changed into separate lists so that the runtime environment"},{"line_number":228,"context_line":"  is only installing the packages it needs."},{"line_number":229,"context_line":""},{"line_number":230,"context_line":"* Ensure that \u0027optional\u0027 pip packages are installed into the venv during the"},{"line_number":231,"context_line":"  build stage, rather than during the install stage."},{"line_number":232,"context_line":""},{"line_number":233,"context_line":".. _pypiserver: https://pypiserver.readthedocs.io"},{"line_number":234,"context_line":""}],"source_content_type":"text/x-rst","patch_set":4,"id":"7f515b1d_9702d608","line":231,"range":{"start_line":230,"start_character":2,"end_line":231,"end_character":52},"updated":"2017-09-25 13:58:31.000000000","message":"Why, and how,  can someone determine if the optional pip install has to be installed into the venv? Is that directly available as a var when executing the role?","commit_id":"ac28e66ab472e16fad40313324d364a37e379bf7"},{"author":{"_account_id":6816,"name":"Jesse Pretorius","email":"jesse@odyssey4.me","username":"jesse-pretorius"},"change_message_id":"b52e8b3b246782018422949ca70416a41ac08dbd","unresolved":false,"context_lines":[{"line_number":227,"context_line":"  packages will be changed into separate lists so that the runtime environment"},{"line_number":228,"context_line":"  is only installing the packages it needs."},{"line_number":229,"context_line":""},{"line_number":230,"context_line":"* Ensure that \u0027optional\u0027 pip packages are installed into the venv during the"},{"line_number":231,"context_line":"  build stage, rather than during the install stage."},{"line_number":232,"context_line":""},{"line_number":233,"context_line":".. _pypiserver: https://pypiserver.readthedocs.io"},{"line_number":234,"context_line":""}],"source_content_type":"text/x-rst","patch_set":4,"id":"7f515b1d_8299726f","line":231,"range":{"start_line":230,"start_character":2,"end_line":231,"end_character":52},"in_reply_to":"7f515b1d_9702d608","updated":"2017-09-25 14:29:24.000000000","message":"Yes, it\u0027s currently already done by the role.","commit_id":"ac28e66ab472e16fad40313324d364a37e379bf7"},{"author":{"_account_id":17068,"name":"Jean-Philippe Evrard","email":"openstack@a.spamming.party","username":"evrardjp"},"change_message_id":"5e8a4d0a51b51424969d21af4c5bf3080412ca73","unresolved":false,"context_lines":[{"line_number":265,"context_line":"------------------"},{"line_number":266,"context_line":""},{"line_number":267,"context_line":"The performance of the deployment should be improved due to the reduction in"},{"line_number":268,"context_line":"time aken to deploy with pre-built packages if a full set of artifacts are"},{"line_number":269,"context_line":"built."},{"line_number":270,"context_line":""},{"line_number":271,"context_line":"End user impact"}],"source_content_type":"text/x-rst","patch_set":4,"id":"7f515b1d_972bf680","line":268,"updated":"2017-09-25 13:58:31.000000000","message":"nit: time taken","commit_id":"ac28e66ab472e16fad40313324d364a37e379bf7"},{"author":{"_account_id":6816,"name":"Jesse Pretorius","email":"jesse@odyssey4.me","username":"jesse-pretorius"},"change_message_id":"b52e8b3b246782018422949ca70416a41ac08dbd","unresolved":false,"context_lines":[{"line_number":265,"context_line":"------------------"},{"line_number":266,"context_line":""},{"line_number":267,"context_line":"The performance of the deployment should be improved due to the reduction in"},{"line_number":268,"context_line":"time aken to deploy with pre-built packages if a full set of artifacts are"},{"line_number":269,"context_line":"built."},{"line_number":270,"context_line":""},{"line_number":271,"context_line":"End user impact"}],"source_content_type":"text/x-rst","patch_set":4,"id":"7f515b1d_42a37a9f","line":268,"in_reply_to":"7f515b1d_972bf680","updated":"2017-09-25 14:29:24.000000000","message":"Done","commit_id":"ac28e66ab472e16fad40313324d364a37e379bf7"},{"author":{"_account_id":17068,"name":"Jean-Philippe Evrard","email":"openstack@a.spamming.party","username":"evrardjp"},"change_message_id":"53766de9ab52c3ec808222daa6f869e1352a7a34","unresolved":false,"context_lines":[{"line_number":178,"context_line":"  solves a problem we have today where some projects (eg: tempest, rally,"},{"line_number":179,"context_line":"  gnocchi) have to be built unconstrained as they do not conform to the"},{"line_number":180,"context_line":"  global requirements process."},{"line_number":181,"context_line":""},{"line_number":182,"context_line":"* Implement changes to each role to handle the wheel building and venv"},{"line_number":183,"context_line":"  building, but do it in such a way that only the build can be executed"},{"line_number":184,"context_line":"  by using tags, setting a specific flag, or include_role and tasks_from."}],"source_content_type":"text/x-rst","patch_set":6,"id":"7f515b1d_bb659fe3","line":181,"updated":"2017-09-26 09:11:26.000000000","message":"Ok for me.","commit_id":"43b3f13f3ec4ffc9e797b4f88edf4763ca5b664d"},{"author":{"_account_id":17068,"name":"Jean-Philippe Evrard","email":"openstack@a.spamming.party","username":"evrardjp"},"change_message_id":"53766de9ab52c3ec808222daa6f869e1352a7a34","unresolved":false,"context_lines":[{"line_number":205,"context_line":""},{"line_number":206,"context_line":"* Remove the complex git caching/staging process which exists today and"},{"line_number":207,"context_line":"  make the use of the repo server for git caching entirely optional."},{"line_number":208,"context_line":"  Implement a replacement process which is simpler and more intuitive."},{"line_number":209,"context_line":""},{"line_number":210,"context_line":"* Implement a playbook which can be used to stage offline installs by"},{"line_number":211,"context_line":"  downloading all built artifacts (completed, perhaps by a CI job) to the"}],"source_content_type":"text/x-rst","patch_set":6,"id":"7f515b1d_dbaf3be1","line":208,"updated":"2017-09-26 09:11:26.000000000","message":"This part I don\u0027t get, as you are stating removal and optional parts. Not sure why it would be needed.","commit_id":"43b3f13f3ec4ffc9e797b4f88edf4763ca5b664d"}]}