)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":9816,"name":"Takashi Kajinami","email":"kajinamit@oss.nttdata.com","username":"kajinamit"},"change_message_id":"aa7ff07a3fa14537f57f592daabcd7b4948000ac","unresolved":true,"context_lines":[{"line_number":6,"context_line":""},{"line_number":7,"context_line":"WIP: Avoid storage partition to become full"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"When a storage partition becomes full, it\u0027s an operator nightmare, where"},{"line_number":10,"context_line":"swift-object and swift-object-replicator process can become completely"},{"line_number":11,"context_line":"stuck, leading to a rebalance (that maybe has been triggered to add some"},{"line_number":12,"context_line":"more space) will never finish."}],"source_content_type":"text/x-gerrit-commit-message","patch_set":3,"id":"cd89ad33_83266680","line":9,"range":{"start_line":9,"start_character":5,"end_line":9,"end_character":37},"updated":"2024-01-31 14:06:29.000000000","message":"wondering if fallocate_reserve is set and tuned in this env ? I don\u0027t know the full context behind this problem but that is definitely the one potential guard you should try first.","commit_id":"f034d67fec9b1f6b1cd1596e7014b8a14743fea3"},{"author":{"_account_id":9816,"name":"Takashi Kajinami","email":"kajinamit@oss.nttdata.com","username":"kajinamit"},"change_message_id":"aa7ff07a3fa14537f57f592daabcd7b4948000ac","unresolved":true,"context_lines":[{"line_number":8,"context_line":""},{"line_number":9,"context_line":"When a storage partition becomes full, it\u0027s an operator nightmare, where"},{"line_number":10,"context_line":"swift-object and swift-object-replicator process can become completely"},{"line_number":11,"context_line":"stuck, leading to a rebalance (that maybe has been triggered to add some"},{"line_number":12,"context_line":"more space) will never finish."},{"line_number":13,"context_line":""},{"line_number":14,"context_line":"The solution is to set the max_connections to -1 in rsync.conf if the"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":3,"id":"ca0df123_dd437fe6","line":11,"range":{"start_line":11,"start_character":7,"end_line":11,"end_character":29},"updated":"2024-01-31 14:06:29.000000000","message":"I don\u0027t think this is correct. AFAIK swift does not start rebalance unless object-server returns a specific response indicating the target space is unmounted (when mount check is True). If a object server is not responding then proxy server may put a object replica to a hand off but it does not attempt rebalance to replicate the data on the full disk to the others in the cluster.","commit_id":"f034d67fec9b1f6b1cd1596e7014b8a14743fea3"},{"author":{"_account_id":9816,"name":"Takashi Kajinami","email":"kajinamit@oss.nttdata.com","username":"kajinamit"},"change_message_id":"aa7ff07a3fa14537f57f592daabcd7b4948000ac","unresolved":true,"context_lines":[{"line_number":16,"context_line":"$facts[\u0027mountpoints\u0027][\"${devices}/${block_dev_name}\"][\u0027available_bytes\u0027]"},{"line_number":17,"context_line":"to see if that\u0027s the case."},{"line_number":18,"context_line":""},{"line_number":19,"context_line":"Note: as each puppet run is done every 30 minutes by default, this patch"},{"line_number":20,"context_line":"isn\u0027t enough, it has to be enriched with a cron job every 5 minutes to"},{"line_number":21,"context_line":"also do the job, otherwise, in 30 minutes, rsync can fill the device"},{"line_number":22,"context_line":"partition entirely, defeating the purpose of this patch. We\u0027re currently"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":3,"id":"84bf403b_55cbb0c1","line":19,"range":{"start_line":19,"start_character":6,"end_line":19,"end_character":60},"updated":"2024-01-31 14:06:29.000000000","message":"this is not true for all cases.","commit_id":"f034d67fec9b1f6b1cd1596e7014b8a14743fea3"}],"/PATCHSET_LEVEL":[{"author":{"_account_id":9816,"name":"Takashi Kajinami","email":"kajinamit@oss.nttdata.com","username":"kajinamit"},"change_message_id":"c441bebd5196217102255cfd50a3f699ef28d8bf","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":3,"id":"3025db7b_33b34e51","updated":"2024-01-31 14:01:32.000000000","message":"I don\u0027t think we should add this in the deployment module. We can\u0027t guarantee users may run puppet such often to change the settings according to the \"current\" available space.\n\nAs you said this should be done as a cron job or even something built-in within swift, IMO.","commit_id":"f034d67fec9b1f6b1cd1596e7014b8a14743fea3"},{"author":{"_account_id":6476,"name":"Thomas Goirand","email":"thomas@goirand.fr","username":"thomas-goirand"},"change_message_id":"ceb951100f898be85731bfc50b123354dd42ca25","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":3,"id":"80a79788_51f7580b","in_reply_to":"3025db7b_33b34e51","updated":"2024-02-01 09:01:21.000000000","message":"Hi Takashi,\nWell, our approach is from the Swift doc:\nhttps://docs.openstack.org/swift/latest/admin_guide.html#preventing-disk-full-scenarios\n\nWhat we intend to do here, is have:\n1/ the cron job to do the necessary change on the fly in near real time\n2/ the puppet job to *NOT* change what the cron job may change in rsyncd.conf\n\nwhich is why I believe it is necessary to have both the puppet module and the cron job deployed.","commit_id":"f034d67fec9b1f6b1cd1596e7014b8a14743fea3"}]}