)]}'
{"doc/source/admin/tuning.rst":[{"author":{"_account_id":11292,"name":"Arne Wiebalck","email":"Arne.Wiebalck@cern.ch","username":"wiebalck"},"change_message_id":"2f6a171426c1cba052a4b23e847e0357c3bf1b73","unresolved":false,"context_lines":[{"line_number":8,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":9,"context_line":""},{"line_number":10,"context_line":"Memory utilization is a difficult thing to tune in Ironic as largely we may"},{"line_number":11,"context_line":"be asked by API consumers to perform work which the underlying tools require"},{"line_number":12,"context_line":"large amounts of memory."},{"line_number":13,"context_line":""},{"line_number":14,"context_line":"The biggest example of this is image conversion. Images not in a raw format"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bf51134e_06375fb2","line":11,"range":{"start_line":11,"start_character":42,"end_line":11,"end_character":47},"updated":"2020-07-07 16:40:37.000000000","message":"for which?","commit_id":"769ad4b3305039ae6a936a2b6f689a6ef9782ad7"},{"author":{"_account_id":11292,"name":"Arne Wiebalck","email":"Arne.Wiebalck@cern.ch","username":"wiebalck"},"change_message_id":"2f6a171426c1cba052a4b23e847e0357c3bf1b73","unresolved":false,"context_lines":[{"line_number":13,"context_line":""},{"line_number":14,"context_line":"The biggest example of this is image conversion. Images not in a raw format"},{"line_number":15,"context_line":"need to be written out to disk (local files or remote in iscsi deploy) which"},{"line_number":16,"context_line":"requires the conversion process to generate an in-memory map to re-assmble"},{"line_number":17,"context_line":"the image contents into a coherent stream of data. This entire process also"},{"line_number":18,"context_line":"stresses the kernel buffers and cache."},{"line_number":19,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"bf51134e_66457b15","line":16,"range":{"start_line":16,"start_character":64,"end_line":16,"end_character":74},"updated":"2020-07-07 16:40:37.000000000","message":"re-assemble","commit_id":"769ad4b3305039ae6a936a2b6f689a6ef9782ad7"},{"author":{"_account_id":11292,"name":"Arne Wiebalck","email":"Arne.Wiebalck@cern.ch","username":"wiebalck"},"change_message_id":"2f6a171426c1cba052a4b23e847e0357c3bf1b73","unresolved":false,"context_lines":[{"line_number":20,"context_line":"This ultimately comes down to a trade-off of Memory versus Performance,"},{"line_number":21,"context_line":"similar to the trade off of Performance versus Cost."},{"line_number":22,"context_line":""},{"line_number":23,"context_line":"On a plus side, an idle ironic deployment does not need much in the way"},{"line_number":24,"context_line":"of memory. On the down side, a highly bursty environment where a large"},{"line_number":25,"context_line":"number of concurrent deployments may be requested should consider two"},{"line_number":26,"context_line":"aspects:"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bf51134e_9929109d","line":23,"range":{"start_line":23,"start_character":24,"end_line":23,"end_character":30},"updated":"2020-07-07 16:40:37.000000000","message":"Ironic","commit_id":"769ad4b3305039ae6a936a2b6f689a6ef9782ad7"},{"author":{"_account_id":11292,"name":"Arne Wiebalck","email":"Arne.Wiebalck@cern.ch","username":"wiebalck"},"change_message_id":"2f6a171426c1cba052a4b23e847e0357c3bf1b73","unresolved":false,"context_lines":[{"line_number":25,"context_line":"number of concurrent deployments may be requested should consider two"},{"line_number":26,"context_line":"aspects:"},{"line_number":27,"context_line":""},{"line_number":28,"context_line":"* How the ironic-api is seved? Will more processes be launched automatically?"},{"line_number":29,"context_line":"* Number of concurrent possible image conversions."},{"line_number":30,"context_line":""},{"line_number":31,"context_line":"API"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bf51134e_2612630a","line":28,"range":{"start_line":28,"start_character":24,"end_line":28,"end_character":29},"updated":"2020-07-07 16:40:37.000000000","message":"served? set up?","commit_id":"769ad4b3305039ae6a936a2b6f689a6ef9782ad7"},{"author":{"_account_id":11292,"name":"Arne Wiebalck","email":"Arne.Wiebalck@cern.ch","username":"wiebalck"},"change_message_id":"2f6a171426c1cba052a4b23e847e0357c3bf1b73","unresolved":false,"context_lines":[{"line_number":32,"context_line":"\u003d\u003d\u003d"},{"line_number":33,"context_line":""},{"line_number":34,"context_line":"Ironic\u0027s API should have a fairly stable memory footprint with activity,"},{"line_number":35,"context_line":"however depending on how the webserver is running the API, additioanl"},{"line_number":36,"context_line":"processes can be launched."},{"line_number":37,"context_line":""},{"line_number":38,"context_line":"Under normal conditions, as of July 2020, the ironic-api consumes"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bf51134e_a6fdd3b2","line":35,"range":{"start_line":35,"start_character":59,"end_line":35,"end_character":69},"updated":"2020-07-07 16:40:37.000000000","message":"additional","commit_id":"769ad4b3305039ae6a936a2b6f689a6ef9782ad7"},{"author":{"_account_id":11292,"name":"Arne Wiebalck","email":"Arne.Wiebalck@cern.ch","username":"wiebalck"},"change_message_id":"2f6a171426c1cba052a4b23e847e0357c3bf1b73","unresolved":false,"context_lines":[{"line_number":35,"context_line":"however depending on how the webserver is running the API, additioanl"},{"line_number":36,"context_line":"processes can be launched."},{"line_number":37,"context_line":""},{"line_number":38,"context_line":"Under normal conditions, as of July 2020, the ironic-api consumes"},{"line_number":39,"context_line":"approximately 270MB of memory per worker. Workers being separate processes."},{"line_number":40,"context_line":"Threads may be more useful, but again there are trade-offs with hosting"},{"line_number":41,"context_line":"HTTP based resources."}],"source_content_type":"text/x-rst","patch_set":1,"id":"bf51134e_3914444e","line":38,"range":{"start_line":38,"start_character":46,"end_line":38,"end_character":56},"updated":"2020-07-07 16:40:37.000000000","message":"``ironic-api``","commit_id":"769ad4b3305039ae6a936a2b6f689a6ef9782ad7"},{"author":{"_account_id":11292,"name":"Arne Wiebalck","email":"Arne.Wiebalck@cern.ch","username":"wiebalck"},"change_message_id":"2f6a171426c1cba052a4b23e847e0357c3bf1b73","unresolved":false,"context_lines":[{"line_number":40,"context_line":"Threads may be more useful, but again there are trade-offs with hosting"},{"line_number":41,"context_line":"HTTP based resources."},{"line_number":42,"context_line":""},{"line_number":43,"context_line":"When the webserver is launched by the API process directly, the default"},{"line_number":44,"context_line":"is four processes."},{"line_number":45,"context_line":""},{"line_number":46,"context_line":"When launching using uwsgi, this will entirely vary upon your configuration,"},{"line_number":47,"context_line":"but balancing workers/threads based upon your load and needs is highly"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bf51134e_dc01b606","line":44,"range":{"start_line":43,"start_character":0,"end_line":44,"end_character":18},"updated":"2020-07-07 16:40:37.000000000","message":"maybe add: \" ..., but can be configured as described here.\" (where \u0027here\u0027 is a link to our hopefully existing doc:-)","commit_id":"769ad4b3305039ae6a936a2b6f689a6ef9782ad7"},{"author":{"_account_id":11655,"name":"Julia Kreger","email":"juliaashleykreger@gmail.com","username":"jkreger","status":"Flying to the moon with a Jetpack!"},"change_message_id":"15bfcc0c5ec8f004f24203018e820f2536c00b8b","unresolved":false,"context_lines":[{"line_number":40,"context_line":"Threads may be more useful, but again there are trade-offs with hosting"},{"line_number":41,"context_line":"HTTP based resources."},{"line_number":42,"context_line":""},{"line_number":43,"context_line":"When the webserver is launched by the API process directly, the default"},{"line_number":44,"context_line":"is four processes."},{"line_number":45,"context_line":""},{"line_number":46,"context_line":"When launching using uwsgi, this will entirely vary upon your configuration,"},{"line_number":47,"context_line":"but balancing workers/threads based upon your load and needs is highly"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bf51134e_edd4c649","line":44,"range":{"start_line":43,"start_character":0,"end_line":44,"end_character":18},"in_reply_to":"bf51134e_dc01b606","updated":"2020-07-17 23:38:13.000000000","message":"I\u0027m not sure it can be configured. I think it is like 4 processes and unlimited thread.","commit_id":"769ad4b3305039ae6a936a2b6f689a6ef9782ad7"},{"author":{"_account_id":11292,"name":"Arne Wiebalck","email":"Arne.Wiebalck@cern.ch","username":"wiebalck"},"change_message_id":"2f6a171426c1cba052a4b23e847e0357c3bf1b73","unresolved":false,"context_lines":[{"line_number":43,"context_line":"When the webserver is launched by the API process directly, the default"},{"line_number":44,"context_line":"is four processes."},{"line_number":45,"context_line":""},{"line_number":46,"context_line":"When launching using uwsgi, this will entirely vary upon your configuration,"},{"line_number":47,"context_line":"but balancing workers/threads based upon your load and needs is highly"},{"line_number":48,"context_line":"advisable."},{"line_number":49,"context_line":""},{"line_number":50,"context_line":"One important item to note: Each ironic API *does* keep a copy of the"},{"line_number":51,"context_line":"hash ring as generated from the database *in-memory*. This is done to"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bf51134e_46ab1788","line":48,"range":{"start_line":46,"start_character":0,"end_line":48,"end_character":10},"updated":"2020-07-07 16:40:37.000000000","message":"Should we include some math here how to do this? Like: with N n-compute, there should be P workers with Q threads each and P * Q \u003d N or something like this? Not saying this math is correct, just thinking it\u0027d be good if we had some guidelines how to avoid API exhaustion.","commit_id":"769ad4b3305039ae6a936a2b6f689a6ef9782ad7"},{"author":{"_account_id":11655,"name":"Julia Kreger","email":"juliaashleykreger@gmail.com","username":"jkreger","status":"Flying to the moon with a Jetpack!"},"change_message_id":"15bfcc0c5ec8f004f24203018e820f2536c00b8b","unresolved":false,"context_lines":[{"line_number":43,"context_line":"When the webserver is launched by the API process directly, the default"},{"line_number":44,"context_line":"is four processes."},{"line_number":45,"context_line":""},{"line_number":46,"context_line":"When launching using uwsgi, this will entirely vary upon your configuration,"},{"line_number":47,"context_line":"but balancing workers/threads based upon your load and needs is highly"},{"line_number":48,"context_line":"advisable."},{"line_number":49,"context_line":""},{"line_number":50,"context_line":"One important item to note: Each ironic API *does* keep a copy of the"},{"line_number":51,"context_line":"hash ring as generated from the database *in-memory*. This is done to"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bf51134e_cdbf42f9","line":48,"range":{"start_line":46,"start_character":0,"end_line":48,"end_character":10},"in_reply_to":"bf51134e_46ab1788","updated":"2020-07-17 23:38:13.000000000","message":"Hmm, We should!\n\nAnd on second thought, I have it a try. It doesn\u0027t quite work out cleanly, to convey. At least my perspective. A larger operator supplying perspective/context would be good though!","commit_id":"769ad4b3305039ae6a936a2b6f689a6ef9782ad7"},{"author":{"_account_id":11292,"name":"Arne Wiebalck","email":"Arne.Wiebalck@cern.ch","username":"wiebalck"},"change_message_id":"2f6a171426c1cba052a4b23e847e0357c3bf1b73","unresolved":false,"context_lines":[{"line_number":52,"context_line":"help allocate load across a cluster in-line with how individual nodes"},{"line_number":53,"context_line":"and their responsible conductors are allocated across the cluster."},{"line_number":54,"context_line":"In other words, your amount of memory WILL increase corresponding to"},{"line_number":55,"context_line":"the number of nodes in your ironic deployment."},{"line_number":56,"context_line":""},{"line_number":57,"context_line":"Conductor"},{"line_number":58,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bf51134e_3c09d22b","line":55,"range":{"start_line":55,"start_character":28,"end_line":55,"end_character":34},"updated":"2020-07-07 16:40:37.000000000","message":"Ironic","commit_id":"769ad4b3305039ae6a936a2b6f689a6ef9782ad7"},{"author":{"_account_id":11292,"name":"Arne Wiebalck","email":"Arne.Wiebalck@cern.ch","username":"wiebalck"},"change_message_id":"2f6a171426c1cba052a4b23e847e0357c3bf1b73","unresolved":false,"context_lines":[{"line_number":57,"context_line":"Conductor"},{"line_number":58,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":59,"context_line":""},{"line_number":60,"context_line":"A conductor process will launch a number of other processes, as required,"},{"line_number":61,"context_line":"in order to complete the requested work. Ultimately this means it can quickly"},{"line_number":62,"context_line":"consume large amounts of memory because it was asked to complete a substantial"},{"line_number":63,"context_line":"amount of work all at once."},{"line_number":64,"context_line":""},{"line_number":65,"context_line":"The ``ironic-conductor`` process, by default consumes about 340MB of RAM in"},{"line_number":66,"context_line":"an idle configuration. This process, by default, operates as a single process."}],"source_content_type":"text/x-rst","patch_set":1,"id":"bf51134e_d9c0a8e3","line":63,"range":{"start_line":60,"start_character":0,"end_line":63,"end_character":27},"updated":"2020-07-07 16:40:37.000000000","message":"I feel the second paragraph repeats some of the information from the first. How about merging the first paragraph into the second (which is more specific)?","commit_id":"769ad4b3305039ae6a936a2b6f689a6ef9782ad7"},{"author":{"_account_id":11292,"name":"Arne Wiebalck","email":"Arne.Wiebalck@cern.ch","username":"wiebalck"},"change_message_id":"2f6a171426c1cba052a4b23e847e0357c3bf1b73","unresolved":false,"context_lines":[{"line_number":73,"context_line":"image conversion for deployment, which is limited to 1GB of RAM per conversion"},{"line_number":74,"context_line":"process."},{"line_number":75,"context_line":""},{"line_number":76,"context_line":"Most deployments, my default, do have a concurrency limit depending on thier"},{"line_number":77,"context_line":"Compute (See `nova.conf \u003chttps://docs.openstack.org/nova/latest/configuration/sample-config.html\u003e`_"},{"line_number":78,"context_line":"setting ``max_concurrent_builds``) configuration. However, this is only per"},{"line_number":79,"context_line":"``nova-compute`` worker, so naturally this concurrency will scale with"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bf51134e_79b19c8f","line":76,"range":{"start_line":76,"start_character":18,"end_line":76,"end_character":20},"updated":"2020-07-07 16:40:37.000000000","message":"by","commit_id":"769ad4b3305039ae6a936a2b6f689a6ef9782ad7"},{"author":{"_account_id":11292,"name":"Arne Wiebalck","email":"Arne.Wiebalck@cern.ch","username":"wiebalck"},"change_message_id":"2f6a171426c1cba052a4b23e847e0357c3bf1b73","unresolved":false,"context_lines":[{"line_number":73,"context_line":"image conversion for deployment, which is limited to 1GB of RAM per conversion"},{"line_number":74,"context_line":"process."},{"line_number":75,"context_line":""},{"line_number":76,"context_line":"Most deployments, my default, do have a concurrency limit depending on thier"},{"line_number":77,"context_line":"Compute (See `nova.conf \u003chttps://docs.openstack.org/nova/latest/configuration/sample-config.html\u003e`_"},{"line_number":78,"context_line":"setting ``max_concurrent_builds``) configuration. However, this is only per"},{"line_number":79,"context_line":"``nova-compute`` worker, so naturally this concurrency will scale with"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bf51134e_b9ab341b","line":76,"range":{"start_line":76,"start_character":71,"end_line":76,"end_character":76},"updated":"2020-07-07 16:40:37.000000000","message":"their","commit_id":"769ad4b3305039ae6a936a2b6f689a6ef9782ad7"},{"author":{"_account_id":11292,"name":"Arne Wiebalck","email":"Arne.Wiebalck@cern.ch","username":"wiebalck"},"change_message_id":"2f6a171426c1cba052a4b23e847e0357c3bf1b73","unresolved":false,"context_lines":[{"line_number":85,"context_line":""},{"line_number":86,"context_line":"With a single nova-compute process talking to a single conductor, asked to"},{"line_number":87,"context_line":"perform ten concurrent deployments of images requiring conversion, the memory"},{"line_number":88,"context_line":"needed may exceed 10GB. This does, however entirely depend up on image,"},{"line_number":89,"context_line":"and also driver used for the deployment."},{"line_number":90,"context_line":""},{"line_number":91,"context_line":"What can I do?"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bf51134e_f962cc15","line":88,"range":{"start_line":88,"start_character":65,"end_line":88,"end_character":70},"updated":"2020-07-07 16:40:37.000000000","message":"the image?","commit_id":"769ad4b3305039ae6a936a2b6f689a6ef9782ad7"},{"author":{"_account_id":11292,"name":"Arne Wiebalck","email":"Arne.Wiebalck@cern.ch","username":"wiebalck"},"change_message_id":"2f6a171426c1cba052a4b23e847e0357c3bf1b73","unresolved":false,"context_lines":[{"line_number":86,"context_line":"With a single nova-compute process talking to a single conductor, asked to"},{"line_number":87,"context_line":"perform ten concurrent deployments of images requiring conversion, the memory"},{"line_number":88,"context_line":"needed may exceed 10GB. This does, however entirely depend up on image,"},{"line_number":89,"context_line":"and also driver used for the deployment."},{"line_number":90,"context_line":""},{"line_number":91,"context_line":"What can I do?"},{"line_number":92,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bf51134e_396d6404","line":89,"range":{"start_line":89,"start_character":9,"end_line":89,"end_character":15},"updated":"2020-07-07 16:40:37.000000000","message":"the driver?","commit_id":"769ad4b3305039ae6a936a2b6f689a6ef9782ad7"},{"author":{"_account_id":11292,"name":"Arne Wiebalck","email":"Arne.Wiebalck@cern.ch","username":"wiebalck"},"change_message_id":"2f6a171426c1cba052a4b23e847e0357c3bf1b73","unresolved":false,"context_lines":[{"line_number":99,"context_line":"* Use the ``direct`` deploy interface. This offloads any final image"},{"line_number":100,"context_line":"  conversion to the host running the ``ironic-python-agent``. Additionally,"},{"line_number":101,"context_line":"  if Swift or other object storage such as RadosGW is used, downloads can"},{"line_number":102,"context_line":"  be completely separated from the host running the ``ironic-condutor``."},{"line_number":103,"context_line":"* Use small/compact \"raw\" images. Qcow2 files are generally compressed"},{"line_number":104,"context_line":"  and require substantial amounts of memory to decompress and stream."},{"line_number":105,"context_line":"* Tune the internal memory limit for the conductor using the"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bf51134e_197060dc","line":102,"range":{"start_line":102,"start_character":61,"end_line":102,"end_character":69},"updated":"2020-07-07 16:40:37.000000000","message":"conductor","commit_id":"769ad4b3305039ae6a936a2b6f689a6ef9782ad7"},{"author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"tag":"autogenerated:zuul:check","change_message_id":"f3948ead801db823e52d37236ff80068c91a69c5","unresolved":false,"context_lines":[{"line_number":109,"context_line":"  Before changing this setting, it is highly advised to consult with your"},{"line_number":110,"context_line":"  resident \"Unix wizzard\" or even the Ironic development team in upstream"},{"line_number":111,"context_line":"  IRC. This feature was added in the Victoria development cycle."},{"line_number":112,"context_line":"* If network bandwidth is the problem your seeking to solve for, you may wish"},{"line_number":113,"context_line":"  to explore a mix of the ``direct`` deploy interface and caching proxies."},{"line_number":114,"context_line":"  Such a configuration can be highly beneficial in wide area deployments. See"},{"line_number":115,"context_line":"  :ref:`Using proxies for image download`."}],"source_content_type":"text/x-rst","patch_set":1,"id":"bf51134e_f9c64c16","line":112,"updated":"2020-07-07 16:24:08.000000000","message":"docs: undefined label: using proxies for image download (if the link has no caption the label must precede a section header)","commit_id":"769ad4b3305039ae6a936a2b6f689a6ef9782ad7"},{"author":{"_account_id":11292,"name":"Arne Wiebalck","email":"Arne.Wiebalck@cern.ch","username":"wiebalck"},"change_message_id":"2f6a171426c1cba052a4b23e847e0357c3bf1b73","unresolved":false,"context_lines":[{"line_number":109,"context_line":"  Before changing this setting, it is highly advised to consult with your"},{"line_number":110,"context_line":"  resident \"Unix wizzard\" or even the Ironic development team in upstream"},{"line_number":111,"context_line":"  IRC. This feature was added in the Victoria development cycle."},{"line_number":112,"context_line":"* If network bandwidth is the problem your seeking to solve for, you may wish"},{"line_number":113,"context_line":"  to explore a mix of the ``direct`` deploy interface and caching proxies."},{"line_number":114,"context_line":"  Such a configuration can be highly beneficial in wide area deployments. See"},{"line_number":115,"context_line":"  :ref:`Using proxies for image download`."}],"source_content_type":"text/x-rst","patch_set":1,"id":"bf51134e_3946047e","line":112,"range":{"start_line":112,"start_character":38,"end_line":112,"end_character":42},"updated":"2020-07-07 16:40:37.000000000","message":"you\u0027re?","commit_id":"769ad4b3305039ae6a936a2b6f689a6ef9782ad7"},{"author":{"_account_id":11292,"name":"Arne Wiebalck","email":"Arne.Wiebalck@cern.ch","username":"wiebalck"},"change_message_id":"0483ba08d044326df2027dc30405938c17be793a","unresolved":false,"context_lines":[{"line_number":18,"context_line":"stresses the kernel buffers and cache."},{"line_number":19,"context_line":""},{"line_number":20,"context_line":"This ultimately comes down to a trade-off of Memory versus Performance,"},{"line_number":21,"context_line":"similar to the trade off of Performance versus Cost."},{"line_number":22,"context_line":""},{"line_number":23,"context_line":"On a plus side, an idle Ironic deployment does not need much in the way"},{"line_number":24,"context_line":"of memory. On the down side, a highly bursty environment where a large"}],"source_content_type":"text/x-rst","patch_set":2,"id":"bf51134e_0e280632","line":21,"range":{"start_line":21,"start_character":15,"end_line":21,"end_character":24},"updated":"2020-07-24 14:24:01.000000000","message":"Nit: trade-off (like in the line above)","commit_id":"d08a75cb19fe977f79de92a63ca8e3d2629878d5"},{"author":{"_account_id":23851,"name":"Riccardo Pittau","email":"elfosardo@gmail.com","username":"elfosardo"},"change_message_id":"ef58c89fc2e244a9b0d7ccf1ccdc4a4ccff4892c","unresolved":false,"context_lines":[{"line_number":25,"context_line":"number of concurrent deployments may be requested should consider two"},{"line_number":26,"context_line":"aspects:"},{"line_number":27,"context_line":""},{"line_number":28,"context_line":"* How the ironic-api is set up? Will more processes be launched automatically?"},{"line_number":29,"context_line":"* Number of concurrent possible image conversions."},{"line_number":30,"context_line":""},{"line_number":31,"context_line":"API"}],"source_content_type":"text/x-rst","patch_set":2,"id":"bf51134e_959cbac0","line":28,"range":{"start_line":28,"start_character":10,"end_line":28,"end_character":20},"updated":"2020-07-15 14:27:40.000000000","message":"ironic-api service/process ?","commit_id":"d08a75cb19fe977f79de92a63ca8e3d2629878d5"},{"author":{"_account_id":11655,"name":"Julia Kreger","email":"juliaashleykreger@gmail.com","username":"jkreger","status":"Flying to the moon with a Jetpack!"},"change_message_id":"15bfcc0c5ec8f004f24203018e820f2536c00b8b","unresolved":false,"context_lines":[{"line_number":25,"context_line":"number of concurrent deployments may be requested should consider two"},{"line_number":26,"context_line":"aspects:"},{"line_number":27,"context_line":""},{"line_number":28,"context_line":"* How the ironic-api is set up? Will more processes be launched automatically?"},{"line_number":29,"context_line":"* Number of concurrent possible image conversions."},{"line_number":30,"context_line":""},{"line_number":31,"context_line":"API"}],"source_content_type":"text/x-rst","patch_set":2,"id":"bf51134e_3e4d9fc4","line":28,"range":{"start_line":28,"start_character":10,"end_line":28,"end_character":20},"in_reply_to":"bf51134e_959cbac0","updated":"2020-07-17 23:38:13.000000000","message":"Done","commit_id":"d08a75cb19fe977f79de92a63ca8e3d2629878d5"},{"author":{"_account_id":11292,"name":"Arne Wiebalck","email":"Arne.Wiebalck@cern.ch","username":"wiebalck"},"change_message_id":"0483ba08d044326df2027dc30405938c17be793a","unresolved":false,"context_lines":[{"line_number":26,"context_line":"aspects:"},{"line_number":27,"context_line":""},{"line_number":28,"context_line":"* How the ironic-api is set up? Will more processes be launched automatically?"},{"line_number":29,"context_line":"* Number of concurrent possible image conversions."},{"line_number":30,"context_line":""},{"line_number":31,"context_line":"API"},{"line_number":32,"context_line":"\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":2,"id":"bf51134e_2e1cea11","line":29,"range":{"start_line":29,"start_character":2,"end_line":29,"end_character":50},"updated":"2020-07-24 14:24:01.000000000","message":"Nit: Would sound nicer if this was a question as well.","commit_id":"d08a75cb19fe977f79de92a63ca8e3d2629878d5"},{"author":{"_account_id":23851,"name":"Riccardo Pittau","email":"elfosardo@gmail.com","username":"elfosardo"},"change_message_id":"ef58c89fc2e244a9b0d7ccf1ccdc4a4ccff4892c","unresolved":false,"context_lines":[{"line_number":35,"context_line":"however depending on how the webserver is running the API, additional"},{"line_number":36,"context_line":"processes can be launched."},{"line_number":37,"context_line":""},{"line_number":38,"context_line":"Under normal conditions, as of July 2020, the ``ironic-api`` consumes"},{"line_number":39,"context_line":"approximately 270MB of memory per worker. Workers being separate processes."},{"line_number":40,"context_line":"Threads may be more useful, but again there are trade-offs with hosting"},{"line_number":41,"context_line":"HTTP based resources."}],"source_content_type":"text/x-rst","patch_set":2,"id":"bf51134e_b57e1e67","line":38,"range":{"start_line":38,"start_character":46,"end_line":38,"end_character":60},"updated":"2020-07-15 14:27:40.000000000","message":"ditto","commit_id":"d08a75cb19fe977f79de92a63ca8e3d2629878d5"},{"author":{"_account_id":11655,"name":"Julia Kreger","email":"juliaashleykreger@gmail.com","username":"jkreger","status":"Flying to the moon with a Jetpack!"},"change_message_id":"15bfcc0c5ec8f004f24203018e820f2536c00b8b","unresolved":false,"context_lines":[{"line_number":35,"context_line":"however depending on how the webserver is running the API, additional"},{"line_number":36,"context_line":"processes can be launched."},{"line_number":37,"context_line":""},{"line_number":38,"context_line":"Under normal conditions, as of July 2020, the ``ironic-api`` consumes"},{"line_number":39,"context_line":"approximately 270MB of memory per worker. Workers being separate processes."},{"line_number":40,"context_line":"Threads may be more useful, but again there are trade-offs with hosting"},{"line_number":41,"context_line":"HTTP based resources."}],"source_content_type":"text/x-rst","patch_set":2,"id":"bf51134e_bececf2a","line":38,"range":{"start_line":38,"start_character":46,"end_line":38,"end_character":60},"in_reply_to":"bf51134e_b57e1e67","updated":"2020-07-17 23:38:13.000000000","message":"Done","commit_id":"d08a75cb19fe977f79de92a63ca8e3d2629878d5"},{"author":{"_account_id":23851,"name":"Riccardo Pittau","email":"elfosardo@gmail.com","username":"elfosardo"},"change_message_id":"ef58c89fc2e244a9b0d7ccf1ccdc4a4ccff4892c","unresolved":false,"context_lines":[{"line_number":36,"context_line":"processes can be launched."},{"line_number":37,"context_line":""},{"line_number":38,"context_line":"Under normal conditions, as of July 2020, the ``ironic-api`` consumes"},{"line_number":39,"context_line":"approximately 270MB of memory per worker. Workers being separate processes."},{"line_number":40,"context_line":"Threads may be more useful, but again there are trade-offs with hosting"},{"line_number":41,"context_line":"HTTP based resources."},{"line_number":42,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"bf51134e_358a0e63","line":39,"range":{"start_line":39,"start_character":40,"end_line":39,"end_character":42},"updated":"2020-07-15 14:27:40.000000000","message":"nit: use comma instead","commit_id":"d08a75cb19fe977f79de92a63ca8e3d2629878d5"},{"author":{"_account_id":11655,"name":"Julia Kreger","email":"juliaashleykreger@gmail.com","username":"jkreger","status":"Flying to the moon with a Jetpack!"},"change_message_id":"15bfcc0c5ec8f004f24203018e820f2536c00b8b","unresolved":false,"context_lines":[{"line_number":36,"context_line":"processes can be launched."},{"line_number":37,"context_line":""},{"line_number":38,"context_line":"Under normal conditions, as of July 2020, the ``ironic-api`` consumes"},{"line_number":39,"context_line":"approximately 270MB of memory per worker. Workers being separate processes."},{"line_number":40,"context_line":"Threads may be more useful, but again there are trade-offs with hosting"},{"line_number":41,"context_line":"HTTP based resources."},{"line_number":42,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"bf51134e_9ed30bd6","line":39,"range":{"start_line":39,"start_character":40,"end_line":39,"end_character":42},"in_reply_to":"bf51134e_358a0e63","updated":"2020-07-17 23:38:13.000000000","message":"Done","commit_id":"d08a75cb19fe977f79de92a63ca8e3d2629878d5"},{"author":{"_account_id":11292,"name":"Arne Wiebalck","email":"Arne.Wiebalck@cern.ch","username":"wiebalck"},"change_message_id":"0483ba08d044326df2027dc30405938c17be793a","unresolved":false,"context_lines":[{"line_number":46,"context_line":"When launching using uwsgi, this will entirely vary upon your configuration,"},{"line_number":47,"context_line":"but balancing workers/threads based upon your load and needs is highly"},{"line_number":48,"context_line":"advisable. Each worker process is unique and consumes far more memory than"},{"line_number":49,"context_line":"a comperable number of worker threads. At the same time, the scheduler will"},{"line_number":50,"context_line":"focus on worker processes as the threads are greenthreads."},{"line_number":51,"context_line":""},{"line_number":52,"context_line":".. note::"}],"source_content_type":"text/x-rst","patch_set":2,"id":"bf51134e_aebe3ad1","line":49,"range":{"start_line":49,"start_character":2,"end_line":49,"end_character":12},"updated":"2020-07-24 14:24:01.000000000","message":"Nit: comparable","commit_id":"d08a75cb19fe977f79de92a63ca8e3d2629878d5"},{"author":{"_account_id":23851,"name":"Riccardo Pittau","email":"elfosardo@gmail.com","username":"elfosardo"},"change_message_id":"ef58c89fc2e244a9b0d7ccf1ccdc4a4ccff4892c","unresolved":false,"context_lines":[{"line_number":55,"context_line":"   but this is not something the development team has measured and will vary"},{"line_number":56,"context_line":"   based upon multiple factors."},{"line_number":57,"context_line":""},{"line_number":58,"context_line":"One important item to note: Each ironic API *does* keep a copy of the"},{"line_number":59,"context_line":"hash ring as generated from the database *in-memory*. This is done to"},{"line_number":60,"context_line":"help allocate load across a cluster in-line with how individual nodes"},{"line_number":61,"context_line":"and their responsible conductors are allocated across the cluster."}],"source_content_type":"text/x-rst","patch_set":2,"id":"bf51134e_d53e3289","line":58,"range":{"start_line":58,"start_character":33,"end_line":58,"end_character":43},"updated":"2020-07-15 14:27:40.000000000","message":"ironic-api service/process ?","commit_id":"d08a75cb19fe977f79de92a63ca8e3d2629878d5"},{"author":{"_account_id":23851,"name":"Riccardo Pittau","email":"elfosardo@gmail.com","username":"elfosardo"},"change_message_id":"ef58c89fc2e244a9b0d7ccf1ccdc4a4ccff4892c","unresolved":false,"context_lines":[{"line_number":55,"context_line":"   but this is not something the development team has measured and will vary"},{"line_number":56,"context_line":"   based upon multiple factors."},{"line_number":57,"context_line":""},{"line_number":58,"context_line":"One important item to note: Each ironic API *does* keep a copy of the"},{"line_number":59,"context_line":"hash ring as generated from the database *in-memory*. This is done to"},{"line_number":60,"context_line":"help allocate load across a cluster in-line with how individual nodes"},{"line_number":61,"context_line":"and their responsible conductors are allocated across the cluster."}],"source_content_type":"text/x-rst","patch_set":2,"id":"bf51134e_15deea6b","line":58,"range":{"start_line":58,"start_character":28,"end_line":58,"end_character":32},"updated":"2020-07-15 14:27:40.000000000","message":"nit: each","commit_id":"d08a75cb19fe977f79de92a63ca8e3d2629878d5"},{"author":{"_account_id":24828,"name":"Kaifeng Wang","email":"kaifeng.w@gmail.com","username":"wangkf"},"change_message_id":"5e116503589ed0bd20b853eb55946dc9de4713c4","unresolved":false,"context_lines":[{"line_number":73,"context_line":"The ``ironic-conductor`` process, by default consumes about 340MB of RAM in"},{"line_number":74,"context_line":"an idle configuration. This process, by default, operates as a single process."},{"line_number":75,"context_line":"Additional processes can be launched, but they must have unique resolvable"},{"line_number":76,"context_line":"hostnames and addresses for JSON-RPC or use use a central oslo.messaging"},{"line_number":77,"context_line":"supported message bus in order for Webserver to API communication to"},{"line_number":78,"context_line":"be functional."},{"line_number":79,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"bf51134e_fbad2731","line":76,"range":{"start_line":76,"start_character":44,"end_line":76,"end_character":47},"updated":"2020-07-15 16:16:26.000000000","message":"nit: dup word","commit_id":"d08a75cb19fe977f79de92a63ca8e3d2629878d5"},{"author":{"_account_id":11655,"name":"Julia Kreger","email":"juliaashleykreger@gmail.com","username":"jkreger","status":"Flying to the moon with a Jetpack!"},"change_message_id":"15bfcc0c5ec8f004f24203018e820f2536c00b8b","unresolved":false,"context_lines":[{"line_number":73,"context_line":"The ``ironic-conductor`` process, by default consumes about 340MB of RAM in"},{"line_number":74,"context_line":"an idle configuration. This process, by default, operates as a single process."},{"line_number":75,"context_line":"Additional processes can be launched, but they must have unique resolvable"},{"line_number":76,"context_line":"hostnames and addresses for JSON-RPC or use use a central oslo.messaging"},{"line_number":77,"context_line":"supported message bus in order for Webserver to API communication to"},{"line_number":78,"context_line":"be functional."},{"line_number":79,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"bf51134e_3ee2bfa1","line":76,"range":{"start_line":76,"start_character":44,"end_line":76,"end_character":47},"in_reply_to":"bf51134e_fbad2731","updated":"2020-07-17 23:38:13.000000000","message":"Done","commit_id":"d08a75cb19fe977f79de92a63ca8e3d2629878d5"},{"author":{"_account_id":24828,"name":"Kaifeng Wang","email":"kaifeng.w@gmail.com","username":"wangkf"},"change_message_id":"5e116503589ed0bd20b853eb55946dc9de4713c4","unresolved":false,"context_lines":[{"line_number":75,"context_line":"Additional processes can be launched, but they must have unique resolvable"},{"line_number":76,"context_line":"hostnames and addresses for JSON-RPC or use use a central oslo.messaging"},{"line_number":77,"context_line":"supported message bus in order for Webserver to API communication to"},{"line_number":78,"context_line":"be functional."},{"line_number":79,"context_line":""},{"line_number":80,"context_line":"Typically, the most memory intensive operation that can be triggered is a"},{"line_number":81,"context_line":"image conversion for deployment, which is limited to 1GB of RAM per conversion"}],"source_content_type":"text/x-rst","patch_set":2,"id":"bf51134e_86881271","line":78,"updated":"2020-07-15 16:16:26.000000000","message":"I thought we were having issues with multiple conductors on the same host (as they have the same hostname), but after diving with code it seems current lock implementation works.\nCurrently we don\u0027t support tuning conductor process number, I wonder if we have other considerations for not doing so?","commit_id":"d08a75cb19fe977f79de92a63ca8e3d2629878d5"},{"author":{"_account_id":24828,"name":"Kaifeng Wang","email":"kaifeng.w@gmail.com","username":"wangkf"},"change_message_id":"5903a0192362e64b022733b381f9c7a5629c9831","unresolved":false,"context_lines":[{"line_number":75,"context_line":"Additional processes can be launched, but they must have unique resolvable"},{"line_number":76,"context_line":"hostnames and addresses for JSON-RPC or use use a central oslo.messaging"},{"line_number":77,"context_line":"supported message bus in order for Webserver to API communication to"},{"line_number":78,"context_line":"be functional."},{"line_number":79,"context_line":""},{"line_number":80,"context_line":"Typically, the most memory intensive operation that can be triggered is a"},{"line_number":81,"context_line":"image conversion for deployment, which is limited to 1GB of RAM per conversion"}],"source_content_type":"text/x-rst","patch_set":2,"id":"bf51134e_448fe474","line":78,"in_reply_to":"bf51134e_5e97b3ec","updated":"2020-07-18 02:33:22.000000000","message":"In theory it should be fine for the pxe and image cache management, a request will only be delivered to one conductor process (with the same topic) in the case of rabbitmq, so for a time window there is only one process servicing the node. I am just going ahead to do some test on this, as one concern is one conductor service can\u0027t fully utilize the resources of the host. We\u0027re using agent and turned off image conversion, so the overload for a conductor should be light.","commit_id":"d08a75cb19fe977f79de92a63ca8e3d2629878d5"},{"author":{"_account_id":11655,"name":"Julia Kreger","email":"juliaashleykreger@gmail.com","username":"jkreger","status":"Flying to the moon with a Jetpack!"},"change_message_id":"15bfcc0c5ec8f004f24203018e820f2536c00b8b","unresolved":false,"context_lines":[{"line_number":75,"context_line":"Additional processes can be launched, but they must have unique resolvable"},{"line_number":76,"context_line":"hostnames and addresses for JSON-RPC or use use a central oslo.messaging"},{"line_number":77,"context_line":"supported message bus in order for Webserver to API communication to"},{"line_number":78,"context_line":"be functional."},{"line_number":79,"context_line":""},{"line_number":80,"context_line":"Typically, the most memory intensive operation that can be triggered is a"},{"line_number":81,"context_line":"image conversion for deployment, which is limited to 1GB of RAM per conversion"}],"source_content_type":"text/x-rst","patch_set":2,"id":"bf51134e_5e97b3ec","line":78,"in_reply_to":"bf51134e_86881271","updated":"2020-07-17 23:38:13.000000000","message":"So, in theory it _should_ be fine. I\u0027ve gone deep in to the code before, and this is actually why I wanted to be able to plug in a real DLM so we could be a lot safer about running concurrent processes. The thing that may go sideways with concurrent processes is tftpboot/httpboot folder management in terms of putting specific files in place and cleaning those up. Then again, if someone is needing that sort of scale, they may not necessarily need, or may not even worry about these sorts of things.\n\nThe other conundrum is who manages the launch. Operators that _DO_ this have written scripts to handle the launch and orchestration of processes launches. [u]wsgi will handle this for us in the process launch. *shrugs*","commit_id":"d08a75cb19fe977f79de92a63ca8e3d2629878d5"},{"author":{"_account_id":11292,"name":"Arne Wiebalck","email":"Arne.Wiebalck@cern.ch","username":"wiebalck"},"change_message_id":"0483ba08d044326df2027dc30405938c17be793a","unresolved":false,"context_lines":[{"line_number":93,"context_line":""},{"line_number":94,"context_line":"With a single nova-compute process talking to a single conductor, asked to"},{"line_number":95,"context_line":"perform ten concurrent deployments of images requiring conversion, the memory"},{"line_number":96,"context_line":"needed may exceed 10GB. This does, however entirely depend up on image block"},{"line_number":97,"context_line":"structure and layout, and what deploy interface is being used."},{"line_number":98,"context_line":""},{"line_number":99,"context_line":"What can I do?"}],"source_content_type":"text/x-rst","patch_set":2,"id":"bf51134e_aea3dab0","line":96,"range":{"start_line":96,"start_character":35,"end_line":96,"end_character":43},"updated":"2020-07-24 14:24:01.000000000","message":"Nit: however,","commit_id":"d08a75cb19fe977f79de92a63ca8e3d2629878d5"},{"author":{"_account_id":23851,"name":"Riccardo Pittau","email":"elfosardo@gmail.com","username":"elfosardo"},"change_message_id":"ef58c89fc2e244a9b0d7ccf1ccdc4a4ccff4892c","unresolved":false,"context_lines":[{"line_number":115,"context_line":"  back memory intensive operations. The default should prevent Out-of-Memory"},{"line_number":116,"context_line":"  operations, but under extreme memory pressure this may still be sub-optimal."},{"line_number":117,"context_line":"  Before changing this setting, it is highly advised to consult with your"},{"line_number":118,"context_line":"  resident \"Unix wizzard\" or even the Ironic development team in upstream"},{"line_number":119,"context_line":"  IRC. This feature was added in the Victoria development cycle."},{"line_number":120,"context_line":"* If network bandwidth is the problem you are seeking to solve for, you may"},{"line_number":121,"context_line":"  wish to explore a mix of the ``direct`` deploy interface and caching"}],"source_content_type":"text/x-rst","patch_set":2,"id":"bf51134e_7599261d","line":118,"range":{"start_line":118,"start_character":17,"end_line":118,"end_character":24},"updated":"2020-07-15 14:27:40.000000000","message":"wizard :)","commit_id":"d08a75cb19fe977f79de92a63ca8e3d2629878d5"},{"author":{"_account_id":11292,"name":"Arne Wiebalck","email":"Arne.Wiebalck@cern.ch","username":"wiebalck"},"change_message_id":"0483ba08d044326df2027dc30405938c17be793a","unresolved":false,"context_lines":[{"line_number":25,"context_line":"number of concurrent deployments may be requested should consider two"},{"line_number":26,"context_line":"aspects:"},{"line_number":27,"context_line":""},{"line_number":28,"context_line":"* How the ironic-api service/process set up? Will more"},{"line_number":29,"context_line":"  processes be launched automatically?"},{"line_number":30,"context_line":"* Number of concurrent possible image conversions."},{"line_number":31,"context_line":""}],"source_content_type":"text/x-rst","patch_set":9,"id":"bf51134e_08f597cb","line":28,"range":{"start_line":28,"start_character":2,"end_line":28,"end_character":6},"updated":"2020-07-24 14:24:01.000000000","message":"s/How/How is/ ?","commit_id":"3498a74e277f6d4124f536e34d07275fb8aef0ea"},{"author":{"_account_id":11655,"name":"Julia Kreger","email":"juliaashleykreger@gmail.com","username":"jkreger","status":"Flying to the moon with a Jetpack!"},"change_message_id":"d8f324bd8dfa906c38086a6d7e5a609d7f184b9c","unresolved":false,"context_lines":[{"line_number":25,"context_line":"number of concurrent deployments may be requested should consider two"},{"line_number":26,"context_line":"aspects:"},{"line_number":27,"context_line":""},{"line_number":28,"context_line":"* How the ironic-api service/process set up? Will more"},{"line_number":29,"context_line":"  processes be launched automatically?"},{"line_number":30,"context_line":"* Number of concurrent possible image conversions."},{"line_number":31,"context_line":""}],"source_content_type":"text/x-rst","patch_set":9,"id":"9f560f44_b1757036","line":28,"range":{"start_line":28,"start_character":2,"end_line":28,"end_character":6},"in_reply_to":"bf51134e_08f597cb","updated":"2020-11-23 14:18:18.000000000","message":"Done","commit_id":"3498a74e277f6d4124f536e34d07275fb8aef0ea"},{"author":{"_account_id":11292,"name":"Arne Wiebalck","email":"Arne.Wiebalck@cern.ch","username":"wiebalck"},"change_message_id":"0483ba08d044326df2027dc30405938c17be793a","unresolved":false,"context_lines":[{"line_number":27,"context_line":""},{"line_number":28,"context_line":"* How the ironic-api service/process set up? Will more"},{"line_number":29,"context_line":"  processes be launched automatically?"},{"line_number":30,"context_line":"* Number of concurrent possible image conversions."},{"line_number":31,"context_line":""},{"line_number":32,"context_line":"API"},{"line_number":33,"context_line":"\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":9,"id":"bf51134e_e8230357","line":30,"range":{"start_line":30,"start_character":2,"end_line":30,"end_character":50},"updated":"2020-07-24 14:24:01.000000000","message":"Formulate as a question as well maybe?","commit_id":"3498a74e277f6d4124f536e34d07275fb8aef0ea"},{"author":{"_account_id":11655,"name":"Julia Kreger","email":"juliaashleykreger@gmail.com","username":"jkreger","status":"Flying to the moon with a Jetpack!"},"change_message_id":"d8f324bd8dfa906c38086a6d7e5a609d7f184b9c","unresolved":false,"context_lines":[{"line_number":27,"context_line":""},{"line_number":28,"context_line":"* How the ironic-api service/process set up? Will more"},{"line_number":29,"context_line":"  processes be launched automatically?"},{"line_number":30,"context_line":"* Number of concurrent possible image conversions."},{"line_number":31,"context_line":""},{"line_number":32,"context_line":"API"},{"line_number":33,"context_line":"\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":9,"id":"9f560f44_d178e42d","line":30,"range":{"start_line":30,"start_character":2,"end_line":30,"end_character":50},"in_reply_to":"bf51134e_e8230357","updated":"2020-11-23 14:18:18.000000000","message":"Done","commit_id":"3498a74e277f6d4124f536e34d07275fb8aef0ea"},{"author":{"_account_id":11292,"name":"Arne Wiebalck","email":"Arne.Wiebalck@cern.ch","username":"wiebalck"},"change_message_id":"0483ba08d044326df2027dc30405938c17be793a","unresolved":false,"context_lines":[{"line_number":38,"context_line":""},{"line_number":39,"context_line":"Under normal conditions, as of July 2020, the ``ironic-api`` service/process"},{"line_number":40,"context_line":"consumes approximately 270MB of memory per worker. Workers being separate"},{"line_number":41,"context_line":"processes, where as threads may be more useful, there are trade-offs with"},{"line_number":42,"context_line":"hosting HTTP based resources."},{"line_number":43,"context_line":""},{"line_number":44,"context_line":"When the webserver is launched by the API process directly, the default is"}],"source_content_type":"text/x-rst","patch_set":9,"id":"bf51134e_a8d2ab32","line":41,"range":{"start_line":41,"start_character":11,"end_line":41,"end_character":19},"updated":"2020-07-24 14:24:01.000000000","message":"whereas ?","commit_id":"3498a74e277f6d4124f536e34d07275fb8aef0ea"},{"author":{"_account_id":11655,"name":"Julia Kreger","email":"juliaashleykreger@gmail.com","username":"jkreger","status":"Flying to the moon with a Jetpack!"},"change_message_id":"d8f324bd8dfa906c38086a6d7e5a609d7f184b9c","unresolved":false,"context_lines":[{"line_number":38,"context_line":""},{"line_number":39,"context_line":"Under normal conditions, as of July 2020, the ``ironic-api`` service/process"},{"line_number":40,"context_line":"consumes approximately 270MB of memory per worker. Workers being separate"},{"line_number":41,"context_line":"processes, where as threads may be more useful, there are trade-offs with"},{"line_number":42,"context_line":"hosting HTTP based resources."},{"line_number":43,"context_line":""},{"line_number":44,"context_line":"When the webserver is launched by the API process directly, the default is"}],"source_content_type":"text/x-rst","patch_set":9,"id":"9f560f44_518a1413","line":41,"range":{"start_line":41,"start_character":11,"end_line":41,"end_character":19},"in_reply_to":"bf51134e_a8d2ab32","updated":"2020-11-23 14:18:18.000000000","message":"Done","commit_id":"3498a74e277f6d4124f536e34d07275fb8aef0ea"},{"author":{"_account_id":11292,"name":"Arne Wiebalck","email":"Arne.Wiebalck@cern.ch","username":"wiebalck"},"change_message_id":"0483ba08d044326df2027dc30405938c17be793a","unresolved":false,"context_lines":[{"line_number":37,"context_line":"processes can be launched."},{"line_number":38,"context_line":""},{"line_number":39,"context_line":"Under normal conditions, as of July 2020, the ``ironic-api`` service/process"},{"line_number":40,"context_line":"consumes approximately 270MB of memory per worker. Workers being separate"},{"line_number":41,"context_line":"processes, where as threads may be more useful, there are trade-offs with"},{"line_number":42,"context_line":"hosting HTTP based resources."},{"line_number":43,"context_line":""},{"line_number":44,"context_line":"When the webserver is launched by the API process directly, the default is"},{"line_number":45,"context_line":"based upon the number of CPU sockets in your machine."}],"source_content_type":"text/x-rst","patch_set":9,"id":"bf51134e_e8d82353","line":42,"range":{"start_line":40,"start_character":51,"end_line":42,"end_character":29},"updated":"2020-07-24 14:24:01.000000000","message":"Could you please review this sentence? I find it hard to diges :)","commit_id":"3498a74e277f6d4124f536e34d07275fb8aef0ea"},{"author":{"_account_id":11292,"name":"Arne Wiebalck","email":"Arne.Wiebalck@cern.ch","username":"wiebalck"},"change_message_id":"0483ba08d044326df2027dc30405938c17be793a","unresolved":false,"context_lines":[{"line_number":41,"context_line":"processes, where as threads may be more useful, there are trade-offs with"},{"line_number":42,"context_line":"hosting HTTP based resources."},{"line_number":43,"context_line":""},{"line_number":44,"context_line":"When the webserver is launched by the API process directly, the default is"},{"line_number":45,"context_line":"based upon the number of CPU sockets in your machine."},{"line_number":46,"context_line":""},{"line_number":47,"context_line":"When launching using uwsgi, this will entirely vary upon your configuration,"}],"source_content_type":"text/x-rst","patch_set":9,"id":"bf51134e_48e94fe1","line":44,"range":{"start_line":44,"start_character":64,"end_line":44,"end_character":71},"updated":"2020-07-24 14:24:01.000000000","message":"s/default/default number of workers/","commit_id":"3498a74e277f6d4124f536e34d07275fb8aef0ea"},{"author":{"_account_id":11292,"name":"Arne Wiebalck","email":"Arne.Wiebalck@cern.ch","username":"wiebalck"},"change_message_id":"0483ba08d044326df2027dc30405938c17be793a","unresolved":false,"context_lines":[{"line_number":47,"context_line":"When launching using uwsgi, this will entirely vary upon your configuration,"},{"line_number":48,"context_line":"but balancing workers/threads based upon your load and needs is highly"},{"line_number":49,"context_line":"advisable. Each worker process is unique and consumes far more memory than"},{"line_number":50,"context_line":"a comperable number of worker threads. At the same time, the scheduler will"},{"line_number":51,"context_line":"focus on worker processes as the threads are greenthreads."},{"line_number":52,"context_line":""},{"line_number":53,"context_line":".. note::"}],"source_content_type":"text/x-rst","patch_set":9,"id":"bf51134e_c8b6fff9","line":50,"range":{"start_line":50,"start_character":2,"end_line":50,"end_character":12},"updated":"2020-07-24 14:24:01.000000000","message":"comparable","commit_id":"3498a74e277f6d4124f536e34d07275fb8aef0ea"},{"author":{"_account_id":11655,"name":"Julia Kreger","email":"juliaashleykreger@gmail.com","username":"jkreger","status":"Flying to the moon with a Jetpack!"},"change_message_id":"d8f324bd8dfa906c38086a6d7e5a609d7f184b9c","unresolved":false,"context_lines":[{"line_number":47,"context_line":"When launching using uwsgi, this will entirely vary upon your configuration,"},{"line_number":48,"context_line":"but balancing workers/threads based upon your load and needs is highly"},{"line_number":49,"context_line":"advisable. Each worker process is unique and consumes far more memory than"},{"line_number":50,"context_line":"a comperable number of worker threads. At the same time, the scheduler will"},{"line_number":51,"context_line":"focus on worker processes as the threads are greenthreads."},{"line_number":52,"context_line":""},{"line_number":53,"context_line":".. note::"}],"source_content_type":"text/x-rst","patch_set":9,"id":"9f560f44_710e388d","line":50,"range":{"start_line":50,"start_character":2,"end_line":50,"end_character":12},"in_reply_to":"bf51134e_c8b6fff9","updated":"2020-11-23 14:18:18.000000000","message":"Done","commit_id":"3498a74e277f6d4124f536e34d07275fb8aef0ea"},{"author":{"_account_id":11292,"name":"Arne Wiebalck","email":"Arne.Wiebalck@cern.ch","username":"wiebalck"},"change_message_id":"0483ba08d044326df2027dc30405938c17be793a","unresolved":false,"context_lines":[{"line_number":56,"context_line":"   but this is not something the development team has measured and will vary"},{"line_number":57,"context_line":"   based upon multiple factors."},{"line_number":58,"context_line":""},{"line_number":59,"context_line":"One important item to note: each ironic API service/process *does* keep a"},{"line_number":60,"context_line":"copy of the hash ring as generated from the database *in-memory*. This is"},{"line_number":61,"context_line":"done to help allocate load across a cluster in-line with how individual nodes"},{"line_number":62,"context_line":"and their responsible conductors are allocated across the cluster."}],"source_content_type":"text/x-rst","patch_set":9,"id":"bf51134e_c8ab9f85","line":59,"range":{"start_line":59,"start_character":33,"end_line":59,"end_character":39},"updated":"2020-07-24 14:24:01.000000000","message":"Ironic","commit_id":"3498a74e277f6d4124f536e34d07275fb8aef0ea"},{"author":{"_account_id":11655,"name":"Julia Kreger","email":"juliaashleykreger@gmail.com","username":"jkreger","status":"Flying to the moon with a Jetpack!"},"change_message_id":"d8f324bd8dfa906c38086a6d7e5a609d7f184b9c","unresolved":false,"context_lines":[{"line_number":56,"context_line":"   but this is not something the development team has measured and will vary"},{"line_number":57,"context_line":"   based upon multiple factors."},{"line_number":58,"context_line":""},{"line_number":59,"context_line":"One important item to note: each ironic API service/process *does* keep a"},{"line_number":60,"context_line":"copy of the hash ring as generated from the database *in-memory*. This is"},{"line_number":61,"context_line":"done to help allocate load across a cluster in-line with how individual nodes"},{"line_number":62,"context_line":"and their responsible conductors are allocated across the cluster."}],"source_content_type":"text/x-rst","patch_set":9,"id":"9f560f44_917e6ce1","line":59,"range":{"start_line":59,"start_character":33,"end_line":59,"end_character":39},"in_reply_to":"bf51134e_c8ab9f85","updated":"2020-11-23 14:18:18.000000000","message":"Done","commit_id":"3498a74e277f6d4124f536e34d07275fb8aef0ea"},{"author":{"_account_id":11292,"name":"Arne Wiebalck","email":"Arne.Wiebalck@cern.ch","username":"wiebalck"},"change_message_id":"0483ba08d044326df2027dc30405938c17be793a","unresolved":false,"context_lines":[{"line_number":71,"context_line":"consume large amounts of memory because it was asked to complete a substantial"},{"line_number":72,"context_line":"amount of work all at once."},{"line_number":73,"context_line":""},{"line_number":74,"context_line":"The ``ironic-conductor`` process, by default consumes about 340MB of RAM in"},{"line_number":75,"context_line":"an idle configuration. This process, by default, operates as a single process."},{"line_number":76,"context_line":"Additional processes can be launched, but they must have unique resolvable"},{"line_number":77,"context_line":"hostnames and addresses for JSON-RPC or use a central oslo.messaging"}],"source_content_type":"text/x-rst","patch_set":9,"id":"bf51134e_c8843f0e","line":74,"range":{"start_line":74,"start_character":32,"end_line":74,"end_character":33},"updated":"2020-07-24 14:24:01.000000000","message":"s/, by default/, by default,/","commit_id":"3498a74e277f6d4124f536e34d07275fb8aef0ea"},{"author":{"_account_id":11292,"name":"Arne Wiebalck","email":"Arne.Wiebalck@cern.ch","username":"wiebalck"},"change_message_id":"0483ba08d044326df2027dc30405938c17be793a","unresolved":false,"context_lines":[{"line_number":94,"context_line":""},{"line_number":95,"context_line":"With a single nova-compute process talking to a single conductor, asked to"},{"line_number":96,"context_line":"perform ten concurrent deployments of images requiring conversion, the memory"},{"line_number":97,"context_line":"needed may exceed 10GB. This does, however entirely depend up on image block"},{"line_number":98,"context_line":"structure and layout, and what deploy interface is being used."},{"line_number":99,"context_line":""},{"line_number":100,"context_line":"What can I do?"}],"source_content_type":"text/x-rst","patch_set":9,"id":"bf51134e_88bc07bc","line":97,"range":{"start_line":97,"start_character":59,"end_line":97,"end_character":64},"updated":"2020-07-24 14:24:01.000000000","message":"s/up on/upon/","commit_id":"3498a74e277f6d4124f536e34d07275fb8aef0ea"},{"author":{"_account_id":10239,"name":"Dmitry Tantsur","email":"dtantsur@protonmail.com","username":"dtantsur"},"change_message_id":"bbbb1281a9d46f00379b5a7ad44527e11235e006","unresolved":false,"context_lines":[{"line_number":1,"context_line":".. _tuning:"},{"line_number":2,"context_line":""},{"line_number":3,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":4,"context_line":"Tuning Ironic"}],"source_content_type":"text/x-rst","patch_set":12,"id":"1f621f24_3183fcce","line":1,"updated":"2020-10-30 09:55:13.000000000","message":"anchors are never needed in the top of a document, you can always use :doc:`/admin/tuning`","commit_id":"c43f3b127ba6a3b9013f3618e685eb417e05b0f0"},{"author":{"_account_id":4571,"name":"Steve Baker","email":"sbaker@redhat.com","username":"steve-stevebaker"},"change_message_id":"4e0ab4e9c57944b23ed8adbf4f0d9fad3738a579","unresolved":false,"context_lines":[{"line_number":37,"context_line":"however depending on how the webserver is running the API, additional"},{"line_number":38,"context_line":"processes can be launched."},{"line_number":39,"context_line":""},{"line_number":40,"context_line":"Under normal conditions, as of July 2020, the ``ironic-api`` service/process"},{"line_number":41,"context_line":"consumes approximately 270MB of memory per worker. Depending on how the"},{"line_number":42,"context_line":"process is being launched, the number of workers and maximum request threads"},{"line_number":43,"context_line":"per worker may differ. Naturally there are configuration and performance"}],"source_content_type":"text/x-rst","patch_set":12,"id":"9f560f44_c8b12d47","line":40,"range":{"start_line":40,"start_character":31,"end_line":40,"end_character":40},"updated":"2020-10-01 01:42:40.000000000","message":"I\u0027m not sure a date would be meaningful for the reader, how about a release version, \"as of Ironic version 15.1.0)","commit_id":"c43f3b127ba6a3b9013f3618e685eb417e05b0f0"},{"author":{"_account_id":10342,"name":"Jay Faulkner","display_name":"JayF","email":"jay@jvf.cc","username":"JayF","status":"youtube.com/@oss-gr / podcast.gr-oss.io"},"change_message_id":"27459652ccc043dd297e1d742a9819a0b83cfbae","unresolved":false,"context_lines":[{"line_number":37,"context_line":"however depending on how the webserver is running the API, additional"},{"line_number":38,"context_line":"processes can be launched."},{"line_number":39,"context_line":""},{"line_number":40,"context_line":"Under normal conditions, as of July 2020, the ``ironic-api`` service/process"},{"line_number":41,"context_line":"consumes approximately 270MB of memory per worker. Depending on how the"},{"line_number":42,"context_line":"process is being launched, the number of workers and maximum request threads"},{"line_number":43,"context_line":"per worker may differ. Naturally there are configuration and performance"}],"source_content_type":"text/x-rst","patch_set":12,"id":"3f65232a_44734f81","line":40,"range":{"start_line":40,"start_character":31,"end_line":40,"end_character":40},"in_reply_to":"9f560f44_c8b12d47","updated":"2020-10-23 16:27:54.000000000","message":"++","commit_id":"c43f3b127ba6a3b9013f3618e685eb417e05b0f0"},{"author":{"_account_id":11655,"name":"Julia Kreger","email":"juliaashleykreger@gmail.com","username":"jkreger","status":"Flying to the moon with a Jetpack!"},"change_message_id":"d8f324bd8dfa906c38086a6d7e5a609d7f184b9c","unresolved":false,"context_lines":[{"line_number":46,"context_line":"* Directly as a native python process, i.e. execute ``ironic-api``"},{"line_number":47,"context_line":"  processes. Each single worker allows for multiple requests to be handled"},{"line_number":48,"context_line":"  and threaded at the same time which can allow high levels of request"},{"line_number":49,"context_line":"  concurrency."},{"line_number":50,"context_line":"* Launched via a wrapper such as Apache+uWSGI may allow for multiple distinct"},{"line_number":51,"context_line":"  worker processes, but these workers typically limit the number of request"},{"line_number":52,"context_line":"  processing threads that are permitted to execute. This means requests can"}],"source_content_type":"text/x-rst","patch_set":12,"id":"1f621f24_ce706275","line":49,"updated":"2020-11-23 14:18:18.000000000","message":"Dmitry has changed this as of victoria to max out at 4.","commit_id":"c43f3b127ba6a3b9013f3618e685eb417e05b0f0"},{"author":{"_account_id":24828,"name":"Kaifeng Wang","email":"kaifeng.w@gmail.com","username":"wangkf"},"change_message_id":"8cf617502a16363eef61b57ccfb7444ec6fdde65","unresolved":false,"context_lines":[{"line_number":75,"context_line":"done to help allocate load across a cluster in-line with how individual nodes"},{"line_number":76,"context_line":"and their responsible conductors are allocated across the cluster."},{"line_number":77,"context_line":"In other words, your amount of memory WILL increase corresponding to"},{"line_number":78,"context_line":"the number of nodes in your Ironic deployment."},{"line_number":79,"context_line":""},{"line_number":80,"context_line":"Conductor"},{"line_number":81,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":12,"id":"3f65232a_9112bd57","line":78,"range":{"start_line":78,"start_character":4,"end_line":78,"end_character":19},"updated":"2020-10-25 04:35:16.000000000","message":"to be precise it actually means conductor nodes and drivers enabled.","commit_id":"c43f3b127ba6a3b9013f3618e685eb417e05b0f0"},{"author":{"_account_id":10239,"name":"Dmitry Tantsur","email":"dtantsur@protonmail.com","username":"dtantsur"},"change_message_id":"bbbb1281a9d46f00379b5a7ad44527e11235e006","unresolved":false,"context_lines":[{"line_number":75,"context_line":"done to help allocate load across a cluster in-line with how individual nodes"},{"line_number":76,"context_line":"and their responsible conductors are allocated across the cluster."},{"line_number":77,"context_line":"In other words, your amount of memory WILL increase corresponding to"},{"line_number":78,"context_line":"the number of nodes in your Ironic deployment."},{"line_number":79,"context_line":""},{"line_number":80,"context_line":"Conductor"},{"line_number":81,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":12,"id":"1f621f24_118880e5","line":78,"updated":"2020-10-30 09:55:13.000000000","message":"yep, each Ironic node is not stored in the ring","commit_id":"c43f3b127ba6a3b9013f3618e685eb417e05b0f0"},{"author":{"_account_id":11655,"name":"Julia Kreger","email":"juliaashleykreger@gmail.com","username":"jkreger","status":"Flying to the moon with a Jetpack!"},"change_message_id":"79ff764565daa8cddee9c15dfd3617697dcf1109","unresolved":false,"context_lines":[{"line_number":75,"context_line":"done to help allocate load across a cluster in-line with how individual nodes"},{"line_number":76,"context_line":"and their responsible conductors are allocated across the cluster."},{"line_number":77,"context_line":"In other words, your amount of memory WILL increase corresponding to"},{"line_number":78,"context_line":"the number of nodes in your Ironic deployment."},{"line_number":79,"context_line":""},{"line_number":80,"context_line":"Conductor"},{"line_number":81,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":12,"id":"ef1b7674_3f8aad70","line":78,"in_reply_to":"1f621f24_118880e5","updated":"2021-02-13 14:27:58.000000000","message":"Added more clarity, I don\u0027t want to explain the inner workings of the hash rings, but I pointed to conductor groups indicating that it can be filtered down with that.","commit_id":"c43f3b127ba6a3b9013f3618e685eb417e05b0f0"},{"author":{"_account_id":10342,"name":"Jay Faulkner","display_name":"JayF","email":"jay@jvf.cc","username":"JayF","status":"youtube.com/@oss-gr / podcast.gr-oss.io"},"change_message_id":"27459652ccc043dd297e1d742a9819a0b83cfbae","unresolved":false,"context_lines":[{"line_number":85,"context_line":"consume large amounts of memory because it was asked to complete a substantial"},{"line_number":86,"context_line":"amount of work all at once."},{"line_number":87,"context_line":""},{"line_number":88,"context_line":"The ``ironic-conductor`` process, by default, consumes about 340MB of RAM in"},{"line_number":89,"context_line":"an idle configuration. This process, by default, operates as a single process."},{"line_number":90,"context_line":"Additional processes can be launched, but they must have unique resolvable"},{"line_number":91,"context_line":"hostnames and addresses for JSON-RPC or use a central oslo.messaging"}],"source_content_type":"text/x-rst","patch_set":12,"id":"3f65232a_e42c1b49","line":88,"updated":"2020-10-23 16:27:54.000000000","message":"Cite ironic version here, too?","commit_id":"c43f3b127ba6a3b9013f3618e685eb417e05b0f0"},{"author":{"_account_id":24828,"name":"Kaifeng Wang","email":"kaifeng.w@gmail.com","username":"wangkf"},"change_message_id":"8cf617502a16363eef61b57ccfb7444ec6fdde65","unresolved":false,"context_lines":[{"line_number":126,"context_line":"* Use small/compact \"raw\" images. Qcow2 files are generally compressed"},{"line_number":127,"context_line":"  and require substantial amounts of memory to decompress and stream."},{"line_number":128,"context_line":"* Tune the internal memory limit for the conductor using the"},{"line_number":129,"context_line":"  ``[DEFAULT]memory_minimum`` setting. This will help the conductor throttle"},{"line_number":130,"context_line":"  back memory intensive operations. The default should prevent Out-of-Memory"},{"line_number":131,"context_line":"  operations, but under extreme memory pressure this may still be sub-optimal."},{"line_number":132,"context_line":"  Before changing this setting, it is highly advised to consult with your"}],"source_content_type":"text/x-rst","patch_set":12,"id":"3f65232a_710f212e","line":129,"range":{"start_line":129,"start_character":13,"end_line":129,"end_character":27},"updated":"2020-10-25 04:35:16.000000000","message":"I guess this option has been renamed.","commit_id":"c43f3b127ba6a3b9013f3618e685eb417e05b0f0"},{"author":{"_account_id":10239,"name":"Dmitry Tantsur","email":"dtantsur@protonmail.com","username":"dtantsur"},"change_message_id":"bbbb1281a9d46f00379b5a7ad44527e11235e006","unresolved":false,"context_lines":[{"line_number":126,"context_line":"* Use small/compact \"raw\" images. Qcow2 files are generally compressed"},{"line_number":127,"context_line":"  and require substantial amounts of memory to decompress and stream."},{"line_number":128,"context_line":"* Tune the internal memory limit for the conductor using the"},{"line_number":129,"context_line":"  ``[DEFAULT]memory_minimum`` setting. This will help the conductor throttle"},{"line_number":130,"context_line":"  back memory intensive operations. The default should prevent Out-of-Memory"},{"line_number":131,"context_line":"  operations, but under extreme memory pressure this may still be sub-optimal."},{"line_number":132,"context_line":"  Before changing this setting, it is highly advised to consult with your"}],"source_content_type":"text/x-rst","patch_set":12,"id":"1f621f24_716c5436","line":129,"updated":"2020-10-30 09:55:13.000000000","message":"Seems to","commit_id":"c43f3b127ba6a3b9013f3618e685eb417e05b0f0"}]}