)]}'
{"/PATCHSET_LEVEL":[{"author":{"_account_id":1131,"name":"Brian Haley","email":"haleyb.dev@gmail.com","username":"brian-haley"},"change_message_id":"5395beaae04d953ba89060bc60ec4483cf148215","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":1,"id":"bf2582cc_f87d8641","updated":"2022-09-09 21:01:57.000000000","message":"Thanks for the spec, some comments.","commit_id":"89528429f91ae1a9667a9a53d7c64bc4b58f9608"},{"author":{"_account_id":9642,"name":"Guillaume Espanel","email":"guillaume.espanel@gmail.com","username":"quatre"},"change_message_id":"40a4bb63379aa580200a90451802fc534c38b5fb","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":3,"id":"c53b4e2b_71cbed0c","updated":"2022-09-12 08:48:48.000000000","message":"Thanks for the reviews!\nSorry I didn\u0027t detail this more in the first patch, I was a bit hesitant to go lower-level in this spec.\nI added some details about the solution I have in mind. Let me know if we need to dive deeper now or if we should keep that for later.\nCheers!","commit_id":"964104720cc202a8348757624409bae1c9d98035"}],"specs/2023.1/metadata-rate-limit.rst":[{"author":{"_account_id":1131,"name":"Brian Haley","email":"haleyb.dev@gmail.com","username":"brian-haley"},"change_message_id":"5395beaae04d953ba89060bc60ec4483cf148215","unresolved":true,"context_lines":[{"line_number":12,"context_line":"https://bugs.launchpad.net/neutron/+bug/1989199"},{"line_number":13,"context_line":""},{"line_number":14,"context_line":"The metadata agents currently do not limit the number of requests they try to"},{"line_number":15,"context_line":"process. Mis-behaved instances hammering the metadata endpoint can incur a"},{"line_number":16,"context_line":"high load on services above the metadata-agents (Nova and Neutron)."},{"line_number":17,"context_line":""},{"line_number":18,"context_line":"Problem Description"}],"source_content_type":"text/x-rst","patch_set":1,"id":"1577d820_740237b9","line":15,"range":{"start_line":15,"start_character":31,"end_line":15,"end_character":40},"updated":"2022-09-09 21:01:57.000000000","message":"Although I understand what you mean, should probably use a better word or words, like \"overwhelming\" ? or some DoS word ?","commit_id":"89528429f91ae1a9667a9a53d7c64bc4b58f9608"},{"author":{"_account_id":9642,"name":"Guillaume Espanel","email":"guillaume.espanel@gmail.com","username":"quatre"},"change_message_id":"40a4bb63379aa580200a90451802fc534c38b5fb","unresolved":false,"context_lines":[{"line_number":12,"context_line":"https://bugs.launchpad.net/neutron/+bug/1989199"},{"line_number":13,"context_line":""},{"line_number":14,"context_line":"The metadata agents currently do not limit the number of requests they try to"},{"line_number":15,"context_line":"process. Mis-behaved instances hammering the metadata endpoint can incur a"},{"line_number":16,"context_line":"high load on services above the metadata-agents (Nova and Neutron)."},{"line_number":17,"context_line":""},{"line_number":18,"context_line":"Problem Description"}],"source_content_type":"text/x-rst","patch_set":1,"id":"7542c164_76931590","line":15,"range":{"start_line":15,"start_character":31,"end_line":15,"end_character":40},"in_reply_to":"1577d820_740237b9","updated":"2022-09-12 08:48:48.000000000","message":"Done","commit_id":"89528429f91ae1a9667a9a53d7c64bc4b58f9608"},{"author":{"_account_id":1131,"name":"Brian Haley","email":"haleyb.dev@gmail.com","username":"brian-haley"},"change_message_id":"5395beaae04d953ba89060bc60ec4483cf148215","unresolved":true,"context_lines":[{"line_number":18,"context_line":"Problem Description"},{"line_number":19,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":20,"context_line":""},{"line_number":21,"context_line":"Platforms administrators would benefit from being able to rate-limit requests"},{"line_number":22,"context_line":"handled by metadata in order to protect other OpenStack components from DoS."},{"line_number":23,"context_line":""},{"line_number":24,"context_line":"* Rate-limiting should be configurable via the config files"}],"source_content_type":"text/x-rst","patch_set":1,"id":"cb615a6b_af43deeb","line":21,"range":{"start_line":21,"start_character":0,"end_line":21,"end_character":9},"updated":"2022-09-09 21:01:57.000000000","message":"s/Platform","commit_id":"89528429f91ae1a9667a9a53d7c64bc4b58f9608"},{"author":{"_account_id":9642,"name":"Guillaume Espanel","email":"guillaume.espanel@gmail.com","username":"quatre"},"change_message_id":"40a4bb63379aa580200a90451802fc534c38b5fb","unresolved":false,"context_lines":[{"line_number":18,"context_line":"Problem Description"},{"line_number":19,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":20,"context_line":""},{"line_number":21,"context_line":"Platforms administrators would benefit from being able to rate-limit requests"},{"line_number":22,"context_line":"handled by metadata in order to protect other OpenStack components from DoS."},{"line_number":23,"context_line":""},{"line_number":24,"context_line":"* Rate-limiting should be configurable via the config files"}],"source_content_type":"text/x-rst","patch_set":1,"id":"872a52de_b6959a6a","line":21,"range":{"start_line":21,"start_character":0,"end_line":21,"end_character":9},"in_reply_to":"cb615a6b_af43deeb","updated":"2022-09-12 08:48:48.000000000","message":"Done","commit_id":"89528429f91ae1a9667a9a53d7c64bc4b58f9608"},{"author":{"_account_id":1131,"name":"Brian Haley","email":"haleyb.dev@gmail.com","username":"brian-haley"},"change_message_id":"5395beaae04d953ba89060bc60ec4483cf148215","unresolved":true,"context_lines":[{"line_number":44,"context_line":"In order to accommodate some events that normally occur during the life of an"},{"line_number":45,"context_line":"instance (for example: cloud-init, periodic refresh of the network-metadata),"},{"line_number":46,"context_line":"we could make the rate-limit burstable, for example: limit the request rate"},{"line_number":47,"context_line":"to 30 over 60s and to 10 over 5s."},{"line_number":48,"context_line":""},{"line_number":49,"context_line":"References"},{"line_number":50,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":1,"id":"c21d141e_b4932ac0","line":47,"updated":"2022-09-09 21:01:57.000000000","message":"Can you take a look at one of the zed specs and add some sections here that seem to be missing? Like Impact sections and Implementation/assignee, Testing, etc?","commit_id":"89528429f91ae1a9667a9a53d7c64bc4b58f9608"},{"author":{"_account_id":9642,"name":"Guillaume Espanel","email":"guillaume.espanel@gmail.com","username":"quatre"},"change_message_id":"40a4bb63379aa580200a90451802fc534c38b5fb","unresolved":false,"context_lines":[{"line_number":44,"context_line":"In order to accommodate some events that normally occur during the life of an"},{"line_number":45,"context_line":"instance (for example: cloud-init, periodic refresh of the network-metadata),"},{"line_number":46,"context_line":"we could make the rate-limit burstable, for example: limit the request rate"},{"line_number":47,"context_line":"to 30 over 60s and to 10 over 5s."},{"line_number":48,"context_line":""},{"line_number":49,"context_line":"References"},{"line_number":50,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":1,"id":"7530dd98_4c7b7433","line":47,"in_reply_to":"69b88330_426fd63e","updated":"2022-09-12 08:48:48.000000000","message":"Done","commit_id":"89528429f91ae1a9667a9a53d7c64bc4b58f9608"},{"author":{"_account_id":5948,"name":"Oleg Bondarev","email":"obondarev@mirantis.com","username":"obondarev"},"change_message_id":"d076422886ff0a59310da38f59c232dda364ed17","unresolved":true,"context_lines":[{"line_number":44,"context_line":"In order to accommodate some events that normally occur during the life of an"},{"line_number":45,"context_line":"instance (for example: cloud-init, periodic refresh of the network-metadata),"},{"line_number":46,"context_line":"we could make the rate-limit burstable, for example: limit the request rate"},{"line_number":47,"context_line":"to 30 over 60s and to 10 over 5s."},{"line_number":48,"context_line":""},{"line_number":49,"context_line":"References"},{"line_number":50,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"}],"source_content_type":"text/x-rst","patch_set":1,"id":"69b88330_426fd63e","line":47,"in_reply_to":"c21d141e_b4932ac0","updated":"2022-09-12 08:14:13.000000000","message":"+1 please add more details - what would be new config options, what are proposed haproxy config changes, etc","commit_id":"89528429f91ae1a9667a9a53d7c64bc4b58f9608"},{"author":{"_account_id":5948,"name":"Oleg Bondarev","email":"obondarev@mirantis.com","username":"obondarev"},"change_message_id":"a336a278e06145fdf2c57a6ccfa36b5936c7c003","unresolved":true,"context_lines":[{"line_number":22,"context_line":"handled by metadata in order to protect other OpenStack components from DoS."},{"line_number":23,"context_line":""},{"line_number":24,"context_line":"* Rate-limiting should be configurable via the config files"},{"line_number":25,"context_line":"* Requests should be rate-limited by source IP"},{"line_number":26,"context_line":"* The default settings for the rate-limiting should not impact the normal"},{"line_number":27,"context_line":"  provisioning of an instance (eg. through cloud-init)"},{"line_number":28,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"4323913b_e9fb01f7","line":25,"range":{"start_line":25,"start_character":2,"end_line":25,"end_character":46},"updated":"2022-09-15 06:58:14.000000000","message":"So will metadata service stay vulnerable to \"DDoS\" from multiple sources?","commit_id":"964104720cc202a8348757624409bae1c9d98035"},{"author":{"_account_id":9642,"name":"Guillaume Espanel","email":"guillaume.espanel@gmail.com","username":"quatre"},"change_message_id":"2db52a782b5ca2ef1196976f2ea86783a8c26cf8","unresolved":false,"context_lines":[{"line_number":22,"context_line":"handled by metadata in order to protect other OpenStack components from DoS."},{"line_number":23,"context_line":""},{"line_number":24,"context_line":"* Rate-limiting should be configurable via the config files"},{"line_number":25,"context_line":"* Requests should be rate-limited by source IP"},{"line_number":26,"context_line":"* The default settings for the rate-limiting should not impact the normal"},{"line_number":27,"context_line":"  provisioning of an instance (eg. through cloud-init)"},{"line_number":28,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"8b5aa33c_64dcd816","line":25,"range":{"start_line":25,"start_character":2,"end_line":25,"end_character":46},"in_reply_to":"4323913b_e9fb01f7","updated":"2022-09-15 08:58:55.000000000","message":"I think so, at least in theory. In practice, if end-users cannot forge the source-IP of their instance in a way that the metadata can answer to, they\u0027ll need the assistance of the control plane to get more source IPs in order to effectively DDoS metadata.\nFrom what we have seen, end-users are generally not trying to cause harm when they query metadata 10 times a second; they usually don\u0027t even realize they do it.\n\nThat said, the load caused by all the metadata queries over the whole region are also a concern to us, even after accounting for the rate-limiting we are discussing. It would be nice to discuss mitigation of issues like the thundering herd problem, but I feel this should be done through another RFE.","commit_id":"964104720cc202a8348757624409bae1c9d98035"},{"author":{"_account_id":11975,"name":"Slawek Kaplonski","email":"skaplons@redhat.com","username":"slaweq"},"change_message_id":"4366c2b1540b4e9d6e6357d27a3794016c5942da","unresolved":true,"context_lines":[{"line_number":58,"context_line":"Add new configuration directives to enable and control the rate-limiting:"},{"line_number":59,"context_line":""},{"line_number":60,"context_line":"* rate_limit_enabled (bool): Whether or not to enable request rate-limiting."},{"line_number":61,"context_line":"* base_window_duration (seconds): Duration of the base base window"},{"line_number":62,"context_line":"* burst_window_duration (seconds): Duration of the burst window"},{"line_number":63,"context_line":"* base_query_rate_limit (short): Limit of the query rate expressed over the"},{"line_number":64,"context_line":"  duration of the base window."}],"source_content_type":"text/x-rst","patch_set":3,"id":"85b165fb_6a99e7e0","line":61,"range":{"start_line":61,"start_character":50,"end_line":61,"end_character":59},"updated":"2022-09-14 20:22:38.000000000","message":"nitty nit: double \"base\"","commit_id":"964104720cc202a8348757624409bae1c9d98035"},{"author":{"_account_id":9642,"name":"Guillaume Espanel","email":"guillaume.espanel@gmail.com","username":"quatre"},"change_message_id":"2db52a782b5ca2ef1196976f2ea86783a8c26cf8","unresolved":false,"context_lines":[{"line_number":58,"context_line":"Add new configuration directives to enable and control the rate-limiting:"},{"line_number":59,"context_line":""},{"line_number":60,"context_line":"* rate_limit_enabled (bool): Whether or not to enable request rate-limiting."},{"line_number":61,"context_line":"* base_window_duration (seconds): Duration of the base base window"},{"line_number":62,"context_line":"* burst_window_duration (seconds): Duration of the burst window"},{"line_number":63,"context_line":"* base_query_rate_limit (short): Limit of the query rate expressed over the"},{"line_number":64,"context_line":"  duration of the base window."}],"source_content_type":"text/x-rst","patch_set":3,"id":"d47fdd6a_ea4c5921","line":61,"range":{"start_line":61,"start_character":50,"end_line":61,"end_character":59},"in_reply_to":"85b165fb_6a99e7e0","updated":"2022-09-15 08:58:55.000000000","message":"Done","commit_id":"964104720cc202a8348757624409bae1c9d98035"},{"author":{"_account_id":1131,"name":"Brian Haley","email":"haleyb.dev@gmail.com","username":"brian-haley"},"change_message_id":"971ab8b1655ee61476a1b63400045f5dc889b3c2","unresolved":true,"context_lines":[{"line_number":60,"context_line":"* rate_limit_enabled (bool): Whether or not to enable request rate-limiting."},{"line_number":61,"context_line":"* base_window_duration (seconds): Duration of the base window"},{"line_number":62,"context_line":"* burst_window_duration (seconds): Duration of the burst window"},{"line_number":63,"context_line":"* base_query_rate_limit (short): Limit of the query rate expressed over the"},{"line_number":64,"context_line":"  duration of the base window."},{"line_number":65,"context_line":"* burst_query_rate_limit (short): Limit of the query rate expressed over the"},{"line_number":66,"context_line":"  duration of the burst window."}],"source_content_type":"text/x-rst","patch_set":4,"id":"31bb3142_ec4aa244","line":63,"range":{"start_line":63,"start_character":25,"end_line":63,"end_character":30},"updated":"2022-09-16 01:00:54.000000000","message":"nit: Technically this is an integer, as oslo.config doesn\u0027t have a short type, but I understand what you mean with my networking brain.","commit_id":"9462d10a7b1c65c877bec5002315fafcc49ac28e"}]}