)]}'
{"doc/manpages/object-server.conf.5":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"070947f3f284af2aa5f403abf062b93d905d1342","unresolved":false,"context_lines":[{"line_number":500,"context_line":"increment a counter for every object whose size is \u003c\u003d to the given break"},{"line_number":501,"context_line":"points and report the result after a full scan."},{"line_number":502,"context_line":".IP \\fBrsync_timeout\\fR"},{"line_number":503,"context_line":"Max duration of a partition rsync. The default is 900 seconds."},{"line_number":504,"context_line":".RE"},{"line_number":505,"context_line":""},{"line_number":506,"context_line":""}],"source_content_type":"text/troff","patch_set":4,"id":"fa0719c6_b26f46c4","line":503,"updated":"2016-03-22 20:19:39.000000000","message":"crap, need to update this words to match the deployment guide","commit_id":"b06ccd46a3f759777118298d6db5587668ccd622"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"4156f3d8f5957cfea4105aaa50b5d01f871d0587","unresolved":false,"context_lines":[{"line_number":502,"context_line":".IP \\fBrsync_tempfile_timeout\\fR"},{"line_number":503,"context_line":"Max duration of a partition rsync. Rsync tempfiles older than this will be unlinked.  It"},{"line_number":504,"context_line":"should not be shorter than the replicator\u0027s rsync_timeout.  The default value is \"auto\""},{"line_number":505,"context_line":"which will use the replicator\u0027s rsync_timeout + 900 seconds."},{"line_number":506,"context_line":".RE"},{"line_number":507,"context_line":""},{"line_number":508,"context_line":""}],"source_content_type":"text/troff","patch_set":7,"id":"fa0719c6_24baa50a","line":505,"updated":"2016-03-23 16:24:52.000000000","message":"see comment as in deployment_guide.rst\n\nIt is a pain that we are maintaining these config docs in *3* places :/","commit_id":"cd4dc9bfc907ba218dc4476694dc16b7b256498c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"5ae9843cfda3c242f3c73c0afb0b4baf13028adf","unresolved":false,"context_lines":[{"line_number":502,"context_line":".IP \\fBrsync_tempfile_timeout\\fR"},{"line_number":503,"context_line":"Max duration of a partition rsync. Rsync tempfiles older than this will be unlinked.  It"},{"line_number":504,"context_line":"should not be shorter than the replicator\u0027s rsync_timeout.  The default value is \"auto\""},{"line_number":505,"context_line":"which will use the replicator\u0027s rsync_timeout + 900 seconds."},{"line_number":506,"context_line":".RE"},{"line_number":507,"context_line":""},{"line_number":508,"context_line":""}],"source_content_type":"text/troff","patch_set":7,"id":"fa0719c6_90d9255a","line":505,"in_reply_to":"fa0719c6_24baa50a","updated":"2016-03-23 17:30:03.000000000","message":"Done","commit_id":"cd4dc9bfc907ba218dc4476694dc16b7b256498c"}],"doc/source/deployment_guide.rst":[{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"4156f3d8f5957cfea4105aaa50b5d01f871d0587","unresolved":false,"context_lines":[{"line_number":738,"context_line":"zero_byte_files_per_second  50"},{"line_number":739,"context_line":"object_size_stats"},{"line_number":740,"context_line":"recon_cache_path            /var/cache/swift    Path to recon cache"},{"line_number":741,"context_line":"rsync_tempfile_timeout      auto                Max duration of a partition rsync, any"},{"line_number":742,"context_line":"                                                rsync tempfiles older than this will be"},{"line_number":743,"context_line":"                                                unlinked.  Config value of \"auto\" will use"},{"line_number":744,"context_line":"                                                object-replicator\u0027s rsync_timeout + 900."}],"source_content_type":"text/x-rst","patch_set":7,"id":"fa0719c6_eddd3be1","line":741,"updated":"2016-03-23 16:24:52.000000000","message":"This needs updating since it is now \u003e max duration of an rsync.\n\nMaybe:\n\n\"Time elapsed in seconds before rsync tempfiles will be unlinked. Config value of \"auto\" will use (in order of preference) object-replicator\u0027s rsync_timeout + 900, or reclaim_age, or 604800 (one week).\"","commit_id":"cd4dc9bfc907ba218dc4476694dc16b7b256498c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"5ae9843cfda3c242f3c73c0afb0b4baf13028adf","unresolved":false,"context_lines":[{"line_number":738,"context_line":"zero_byte_files_per_second  50"},{"line_number":739,"context_line":"object_size_stats"},{"line_number":740,"context_line":"recon_cache_path            /var/cache/swift    Path to recon cache"},{"line_number":741,"context_line":"rsync_tempfile_timeout      auto                Max duration of a partition rsync, any"},{"line_number":742,"context_line":"                                                rsync tempfiles older than this will be"},{"line_number":743,"context_line":"                                                unlinked.  Config value of \"auto\" will use"},{"line_number":744,"context_line":"                                                object-replicator\u0027s rsync_timeout + 900."}],"source_content_type":"text/x-rst","patch_set":7,"id":"fa0719c6_7018d11e","line":741,"in_reply_to":"fa0719c6_eddd3be1","updated":"2016-03-23 17:30:03.000000000","message":"Done","commit_id":"cd4dc9bfc907ba218dc4476694dc16b7b256498c"}],"etc/object-server.conf-sample":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"8298d5da967ead9f180509a24023e45dadc6248d","unresolved":false,"context_lines":[{"line_number":70,"context_line":"# disk_chunk_size \u003d 65536"},{"line_number":71,"context_line":"#"},{"line_number":72,"context_line":"# max duration of a partition rsync, rsync tempfiles older than this will be unlinked"},{"line_number":73,"context_line":"# rsync_timeout \u003d 900"},{"line_number":74,"context_line":""},{"line_number":75,"context_line":"[pipeline:main]"},{"line_number":76,"context_line":"pipeline \u003d healthcheck recon object-server"}],"source_content_type":"application/octet-stream","patch_set":2,"id":"1af94dfe_2c78f129","line":73,"updated":"2016-03-17 04:39:27.000000000","message":"I think we should probably bike-shed about this a *little* bit, there\u0027s technically a bit of a race when the auditor decides to clean one of these up because of the low resolution on getmtime and clock drift between nodes (the file may be 15m old, but if the sending node is busy or it\u0027s clock running slow it might not have killed it\u0027s rsync just yet)\n\nSmart thing to do is probably to run your replicator rsync_timeout at something less aggressive like 3600 [1] and then have your auditor just wait till 4500 or something.\n\nBut i don\u0027t want to write down all those stupid words.\n\nWe could make the auditor use rsync_timeout * 1.1 (or something similar) - but then I have to come up with words to explain *that* - so it\u0027s probably slightly *harder* to type ;)\n\n1. We\u0027ve found that 900 doesn\u0027t seem to be detecting rsync deadlocks as much as slow networks, and the *other* rsync_io_timeout option is already super aggressive and plumbed directly into the rsync args - so it\u0027s probably doing a fairly decent job of not tarpittings things too bad when disks are hozed.  If the entire proc is going completely off the rails it\u0027s probably nice to kill it *eventually* but 15 mins for a whole partition seems aggressive for sop.  Obviously it\u0027s a balance, and I\u0027m not sure the 15m default should change - but you might grep for that \"Killing long-running rsync\" next time you do a big rebalance and also consider the cost/benefit of wasting a bunch time and disk over-replicating and restarting vs slightly more patient deadlock recovery.","commit_id":"d8e82f72cb4eb1c70aa4dce509d6b66753d086d1"},{"author":{"_account_id":6968,"name":"Christian Schwede","email":"cschwede@nvidia.com","username":"cschwede"},"change_message_id":"fa512c71fe684118663b5ecbb2eb685655f989b0","unresolved":false,"context_lines":[{"line_number":70,"context_line":"# disk_chunk_size \u003d 65536"},{"line_number":71,"context_line":"#"},{"line_number":72,"context_line":"# max duration of a partition rsync, rsync tempfiles older than this will be unlinked"},{"line_number":73,"context_line":"# rsync_timeout \u003d 900"},{"line_number":74,"context_line":""},{"line_number":75,"context_line":"[pipeline:main]"},{"line_number":76,"context_line":"pipeline \u003d healthcheck recon object-server"}],"source_content_type":"application/octet-stream","patch_set":2,"id":"fa0719c6_5f837c04","line":73,"in_reply_to":"1af94dfe_2c78f129","updated":"2016-03-22 09:22:49.000000000","message":"How about making the auditor setting fixed to rsync_timeout + 900 (which is basically what you proposed as the smart thing)?\nAnd keep the rsync_timeout setting in the replicator section? I like that. But then I also prefer to have a larger default value, because that\u0027s needed anyways and most of us already do this (or not?)\n\nThis has the added benefit that this doesn not require an \"UpgradeImpact\" flag in the commit msg (which is missing, btw).\n\nPeople might be more confused when there is a rsync setting that is used somewhere else than the replicator; why does the object server even has a rsync setting, and why is it 42. Or something like that.","commit_id":"d8e82f72cb4eb1c70aa4dce509d6b66753d086d1"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"9193ffa0ba6369189a591606ce11b48d5718f896","unresolved":false,"context_lines":[{"line_number":70,"context_line":"# disk_chunk_size \u003d 65536"},{"line_number":71,"context_line":"#"},{"line_number":72,"context_line":"# max duration of a partition rsync, rsync tempfiles older than this will be unlinked"},{"line_number":73,"context_line":"# rsync_timeout \u003d 900"},{"line_number":74,"context_line":""},{"line_number":75,"context_line":"[pipeline:main]"},{"line_number":76,"context_line":"pipeline \u003d healthcheck recon object-server"}],"source_content_type":"application/octet-stream","patch_set":2,"id":"fa0719c6_af13620c","line":73,"in_reply_to":"fa0719c6_5f837c04","updated":"2016-03-22 20:06:08.000000000","message":"it needs to move to [DEFAULT] in order to be seen by the auditor unless I re-parse the config - which might be worth it?\n\nI\u0027ll add the tag for now.","commit_id":"d8e82f72cb4eb1c70aa4dce509d6b66753d086d1"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"070947f3f284af2aa5f403abf062b93d905d1342","unresolved":false,"context_lines":[{"line_number":70,"context_line":"# disk_chunk_size \u003d 65536"},{"line_number":71,"context_line":"#"},{"line_number":72,"context_line":"# max duration of a partition rsync, rsync tempfiles older than this will be unlinked"},{"line_number":73,"context_line":"# rsync_timeout \u003d 900"},{"line_number":74,"context_line":""},{"line_number":75,"context_line":"[pipeline:main]"},{"line_number":76,"context_line":"pipeline \u003d healthcheck recon object-server"}],"source_content_type":"application/octet-stream","patch_set":4,"id":"fa0719c6_7275ae96","line":73,"updated":"2016-03-22 20:19:39.000000000","message":"oops this was supposed to get moved back - no UpgradeImpact","commit_id":"b06ccd46a3f759777118298d6db5587668ccd622"},{"author":{"_account_id":6968,"name":"Christian Schwede","email":"cschwede@nvidia.com","username":"cschwede"},"change_message_id":"b4ee2c6e88d1aea89d556168519d758afac6d20d","unresolved":false,"context_lines":[{"line_number":306,"context_line":"# points and report the result after a full scan."},{"line_number":307,"context_line":"# object_size_stats \u003d"},{"line_number":308,"context_line":""},{"line_number":309,"context_line":"# The auditor will cleaup old rsync tempfiles after they are \"old"},{"line_number":310,"context_line":"# enough\" to delete, the default value of auto will use the value of the"},{"line_number":311,"context_line":"# \"object-replicator\u0027s config section rsync_timeout option\" + 900"},{"line_number":312,"context_line":"# rsync_tempfile_timeout \u003d auto"}],"source_content_type":"application/octet-stream","patch_set":7,"id":"fa0719c6_af05ed11","line":309,"range":{"start_line":309,"start_character":19,"end_line":309,"end_character":25},"updated":"2016-03-23 15:49:21.000000000","message":"nit: typo, but can be fixed later","commit_id":"cd4dc9bfc907ba218dc4476694dc16b7b256498c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"5ae9843cfda3c242f3c73c0afb0b4baf13028adf","unresolved":false,"context_lines":[{"line_number":306,"context_line":"# points and report the result after a full scan."},{"line_number":307,"context_line":"# object_size_stats \u003d"},{"line_number":308,"context_line":""},{"line_number":309,"context_line":"# The auditor will cleaup old rsync tempfiles after they are \"old"},{"line_number":310,"context_line":"# enough\" to delete, the default value of auto will use the value of the"},{"line_number":311,"context_line":"# \"object-replicator\u0027s config section rsync_timeout option\" + 900"},{"line_number":312,"context_line":"# rsync_tempfile_timeout \u003d auto"}],"source_content_type":"application/octet-stream","patch_set":7,"id":"fa0719c6_30e939e7","line":309,"in_reply_to":"fa0719c6_af05ed11","updated":"2016-03-23 17:30:03.000000000","message":"Done","commit_id":"cd4dc9bfc907ba218dc4476694dc16b7b256498c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"4156f3d8f5957cfea4105aaa50b5d01f871d0587","unresolved":false,"context_lines":[{"line_number":309,"context_line":"# The auditor will cleaup old rsync tempfiles after they are \"old"},{"line_number":310,"context_line":"# enough\" to delete, the default value of auto will use the value of the"},{"line_number":311,"context_line":"# \"object-replicator\u0027s config section rsync_timeout option\" + 900"},{"line_number":312,"context_line":"# rsync_tempfile_timeout \u003d auto"},{"line_number":313,"context_line":""},{"line_number":314,"context_line":"# Note: Put it at the beginning of the pipleline to profile all middleware. But"},{"line_number":315,"context_line":"# it is safer to put this after healthcheck."}],"source_content_type":"application/octet-stream","patch_set":7,"id":"fa0719c6_c40da9e9","line":312,"updated":"2016-03-23 16:24:52.000000000","message":"see comment in deployment_guide.rst - this description is better but should probably mention that a value in seconds can be used.","commit_id":"cd4dc9bfc907ba218dc4476694dc16b7b256498c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"5ae9843cfda3c242f3c73c0afb0b4baf13028adf","unresolved":false,"context_lines":[{"line_number":309,"context_line":"# The auditor will cleaup old rsync tempfiles after they are \"old"},{"line_number":310,"context_line":"# enough\" to delete, the default value of auto will use the value of the"},{"line_number":311,"context_line":"# \"object-replicator\u0027s config section rsync_timeout option\" + 900"},{"line_number":312,"context_line":"# rsync_tempfile_timeout \u003d auto"},{"line_number":313,"context_line":""},{"line_number":314,"context_line":"# Note: Put it at the beginning of the pipleline to profile all middleware. But"},{"line_number":315,"context_line":"# it is safer to put this after healthcheck."}],"source_content_type":"application/octet-stream","patch_set":7,"id":"fa0719c6_50ecadf2","line":312,"in_reply_to":"fa0719c6_c40da9e9","updated":"2016-03-23 17:30:03.000000000","message":"Done","commit_id":"cd4dc9bfc907ba218dc4476694dc16b7b256498c"}],"swift/common/utils.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"8298d5da967ead9f180509a24023e45dadc6248d","unresolved":false,"context_lines":[{"line_number":2124,"context_line":"    :param path: path to remove file from"},{"line_number":2125,"context_line":"    :param mtime: timestamp of oldest file to keep"},{"line_number":2126,"context_line":"    \"\"\""},{"line_number":2127,"context_line":"    filepaths \u003d map(functools.partial(os.path.join, path), listdir(path))"},{"line_number":2128,"context_line":"    return unlink_paths_older_than(filepaths, mtime)"},{"line_number":2129,"context_line":""},{"line_number":2130,"context_line":""}],"source_content_type":"text/x-python","patch_set":2,"id":"1af94dfe_cccd0d5f","line":2127,"updated":"2016-03-17 04:39:27.000000000","message":"boom - map that partial - do i get my functional hipster badge?","commit_id":"d8e82f72cb4eb1c70aa4dce509d6b66753d086d1"},{"author":{"_account_id":6968,"name":"Christian Schwede","email":"cschwede@nvidia.com","username":"cschwede"},"change_message_id":"fa512c71fe684118663b5ecbb2eb685655f989b0","unresolved":false,"context_lines":[{"line_number":2124,"context_line":"    :param path: path to remove file from"},{"line_number":2125,"context_line":"    :param mtime: timestamp of oldest file to keep"},{"line_number":2126,"context_line":"    \"\"\""},{"line_number":2127,"context_line":"    filepaths \u003d map(functools.partial(os.path.join, path), listdir(path))"},{"line_number":2128,"context_line":"    return unlink_paths_older_than(filepaths, mtime)"},{"line_number":2129,"context_line":""},{"line_number":2130,"context_line":""}],"source_content_type":"text/x-python","patch_set":2,"id":"fa0719c6_5f5b7c87","line":2127,"in_reply_to":"1af94dfe_cccd0d5f","updated":"2016-03-22 09:22:49.000000000","message":"Fancy stuff! And here is your badge:\n\n (ʘ෴̴͜ʘ)","commit_id":"d8e82f72cb4eb1c70aa4dce509d6b66753d086d1"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"9193ffa0ba6369189a591606ce11b48d5718f896","unresolved":false,"context_lines":[{"line_number":2124,"context_line":"    :param path: path to remove file from"},{"line_number":2125,"context_line":"    :param mtime: timestamp of oldest file to keep"},{"line_number":2126,"context_line":"    \"\"\""},{"line_number":2127,"context_line":"    filepaths \u003d map(functools.partial(os.path.join, path), listdir(path))"},{"line_number":2128,"context_line":"    return unlink_paths_older_than(filepaths, mtime)"},{"line_number":2129,"context_line":""},{"line_number":2130,"context_line":""}],"source_content_type":"text/x-python","patch_set":2,"id":"fa0719c6_6f302a9f","line":2127,"in_reply_to":"fa0719c6_5f5b7c87","updated":"2016-03-22 20:06:08.000000000","message":"YES!","commit_id":"d8e82f72cb4eb1c70aa4dce509d6b66753d086d1"},{"author":{"_account_id":6968,"name":"Christian Schwede","email":"cschwede@nvidia.com","username":"cschwede"},"change_message_id":"085283967253a86a75d47452b76a02d4e72c59c4","unresolved":false,"context_lines":[{"line_number":2133,"context_line":"    Remove any paths from the given list that that were"},{"line_number":2134,"context_line":"    last modified before mtime."},{"line_number":2135,"context_line":""},{"line_number":2136,"context_line":"    :param paths: a list of strings, the full paths of files to check"},{"line_number":2137,"context_line":"    :param mtime: timestamp of oldest file to keep"},{"line_number":2138,"context_line":"    \"\"\""},{"line_number":2139,"context_line":"    for fpath in paths:"}],"source_content_type":"text/x-python","patch_set":3,"id":"fa0719c6_8c3690e5","line":2136,"updated":"2016-03-22 09:27:15.000000000","message":"nit: so these are \"filenames\" at the end. \"paths\" sounds to me like a list of, well, \"paths\", i mean directories. but then it might be related to being a non-native speaker... anyways, it\u0027s just a nit.","commit_id":"45e1effc02c0355f5cc3d028f7f718270e48d63f"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a05dfea7a024f31b5b6d6e7046477ea2b5ac238e","unresolved":false,"context_lines":[{"line_number":2133,"context_line":"    Remove any paths from the given list that that were"},{"line_number":2134,"context_line":"    last modified before mtime."},{"line_number":2135,"context_line":""},{"line_number":2136,"context_line":"    :param paths: a list of strings, the full paths of files to check"},{"line_number":2137,"context_line":"    :param mtime: timestamp of oldest file to keep"},{"line_number":2138,"context_line":"    \"\"\""},{"line_number":2139,"context_line":"    for fpath in paths:"}],"source_content_type":"text/x-python","patch_set":3,"id":"fa0719c6_7e646221","line":2136,"in_reply_to":"fa0719c6_8c3690e5","updated":"2016-03-22 20:06:44.000000000","message":"I think of file *names* as \"swift.conf\" I think of file *paths* as like \"/etc/swift/swift.conf\" (i.e. the full path to the file).  I was trying to suggest that there is no relative dir - you need to pass in complete file paths.\n\nI\u0027ll try \"file_paths\" - will make me happy - and might be less confusing than thinking like for example - you can not send in a list of hashdirs.","commit_id":"45e1effc02c0355f5cc3d028f7f718270e48d63f"}],"swift/obj/auditor.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"8298d5da967ead9f180509a24023e45dadc6248d","unresolved":false,"context_lines":[{"line_number":31,"context_line":"from swift.common.daemon import Daemon"},{"line_number":32,"context_line":"from swift.common.storage_policy import POLICIES"},{"line_number":33,"context_line":""},{"line_number":34,"context_line":"re_rsync_tempfile \u003d re.compile(r\u0027^\\..*\\.([a-zA-Z0-9_]){6}$\u0027)"},{"line_number":35,"context_line":""},{"line_number":36,"context_line":""},{"line_number":37,"context_line":"class AuditorWorker(object):"}],"source_content_type":"text/x-python","patch_set":2,"id":"1af94dfe_8ceb05b0","line":34,"updated":"2016-03-17 04:39:27.000000000","message":"does anyone care if this is upper/lower case?","commit_id":"d8e82f72cb4eb1c70aa4dce509d6b66753d086d1"},{"author":{"_account_id":6968,"name":"Christian Schwede","email":"cschwede@nvidia.com","username":"cschwede"},"change_message_id":"fa512c71fe684118663b5ecbb2eb685655f989b0","unresolved":false,"context_lines":[{"line_number":31,"context_line":"from swift.common.daemon import Daemon"},{"line_number":32,"context_line":"from swift.common.storage_policy import POLICIES"},{"line_number":33,"context_line":""},{"line_number":34,"context_line":"re_rsync_tempfile \u003d re.compile(r\u0027^\\..*\\.([a-zA-Z0-9_]){6}$\u0027)"},{"line_number":35,"context_line":""},{"line_number":36,"context_line":""},{"line_number":37,"context_line":"class AuditorWorker(object):"}],"source_content_type":"text/x-python","patch_set":2,"id":"fa0719c6_826497e5","line":34,"in_reply_to":"1af94dfe_8ceb05b0","updated":"2016-03-22 09:22:49.000000000","message":"Well, seems it might be uppercase, so the regex looks ok to me:\n\nhttps://git.samba.org/?p\u003drsync.git;a\u003dblob;f\u003dreceiver.c;hb\u003dHEAD#l76","commit_id":"d8e82f72cb4eb1c70aa4dce509d6b66753d086d1"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"9193ffa0ba6369189a591606ce11b48d5718f896","unresolved":false,"context_lines":[{"line_number":31,"context_line":"from swift.common.daemon import Daemon"},{"line_number":32,"context_line":"from swift.common.storage_policy import POLICIES"},{"line_number":33,"context_line":""},{"line_number":34,"context_line":"re_rsync_tempfile \u003d re.compile(r\u0027^\\..*\\.([a-zA-Z0-9_]){6}$\u0027)"},{"line_number":35,"context_line":""},{"line_number":36,"context_line":""},{"line_number":37,"context_line":"class AuditorWorker(object):"}],"source_content_type":"text/x-python","patch_set":2,"id":"fa0719c6_2f73d24e","line":34,"in_reply_to":"fa0719c6_826497e5","updated":"2016-03-22 20:06:08.000000000","message":"thanks for looking that up, so helpful!\n\nAlthough I must admit I don\u0027t entirely follow that code, why don\u0027t the temp values all end up as decimal based off the getpid() counter call down in the \"if (make_unique)\" block?\n\noh well...","commit_id":"d8e82f72cb4eb1c70aa4dce509d6b66753d086d1"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"8298d5da967ead9f180509a24023e45dadc6248d","unresolved":false,"context_lines":[{"line_number":223,"context_line":"                        self.bytes_processed +\u003d chunk_len"},{"line_number":224,"context_line":"                        self.total_bytes_processed +\u003d chunk_len"},{"line_number":225,"context_line":"        except DiskFileNotExist:"},{"line_number":226,"context_line":"            pass"},{"line_number":227,"context_line":"        except DiskFileQuarantined as err:"},{"line_number":228,"context_line":"            self.quarantines +\u003d 1"},{"line_number":229,"context_line":"            self.logger.error(_(\u0027ERROR Object %(obj)s failed audit and was\u0027"}],"source_content_type":"text/x-python","patch_set":2,"id":"1af94dfe_2caf5182","line":226,"updated":"2016-03-17 04:39:27.000000000","message":"so tombstones used to not increment passes I think - but quarantined would - now they both do?  tests didn\u0027t care... I don\u0027t care... I think the existing behavior was a bug.","commit_id":"d8e82f72cb4eb1c70aa4dce509d6b66753d086d1"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"8298d5da967ead9f180509a24023e45dadc6248d","unresolved":false,"context_lines":[{"line_number":231,"context_line":"                              {\u0027obj\u0027: location, \u0027err\u0027: err})"},{"line_number":232,"context_line":"        self.passes +\u003d 1"},{"line_number":233,"context_line":"        # _ondisk_info attr is initialized to None and filled in by open"},{"line_number":234,"context_line":"        ondisk_info_dict \u003d df._ondisk_info or {}"},{"line_number":235,"context_line":"        if \u0027unexpected\u0027 in ondisk_info_dict:"},{"line_number":236,"context_line":"            is_rsync_tempfile \u003d lambda fpath: re_rsync_tempfile.match("},{"line_number":237,"context_line":"                os.path.basename(fpath))"}],"source_content_type":"text/x-python","patch_set":2,"id":"1af94dfe_ace881b5","line":234,"updated":"2016-03-17 04:39:27.000000000","message":"i know this is behind the interface a little bit, but ... well I don\u0027t think it\u0027s appropriate to expose this weird corner of this particular consistency engines\u0027 behavior in the public diskfile interface.  this Diskfile goes with this replicator and auditor - which use rsync - and everything here knows it.","commit_id":"d8e82f72cb4eb1c70aa4dce509d6b66753d086d1"},{"author":{"_account_id":6968,"name":"Christian Schwede","email":"cschwede@nvidia.com","username":"cschwede"},"change_message_id":"085283967253a86a75d47452b76a02d4e72c59c4","unresolved":false,"context_lines":[{"line_number":30,"context_line":"from swift.common.exceptions import DiskFileQuarantined, DiskFileNotExist"},{"line_number":31,"context_line":"from swift.common.daemon import Daemon"},{"line_number":32,"context_line":"from swift.common.storage_policy import POLICIES"},{"line_number":33,"context_line":""},{"line_number":34,"context_line":"re_rsync_tempfile \u003d re.compile(r\u0027^\\..*\\.([a-zA-Z0-9_]){6}$\u0027)"},{"line_number":35,"context_line":""},{"line_number":36,"context_line":""}],"source_content_type":"text/x-python","patch_set":3,"id":"fa0719c6_ff3a5097","line":33,"updated":"2016-03-22 09:27:15.000000000","message":"A comment might be helpful (or not?), for example:\n# This matches rsync tempfiles, like \".something.ab12CD\"","commit_id":"45e1effc02c0355f5cc3d028f7f718270e48d63f"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a05dfea7a024f31b5b6d6e7046477ea2b5ac238e","unresolved":false,"context_lines":[{"line_number":30,"context_line":"from swift.common.exceptions import DiskFileQuarantined, DiskFileNotExist"},{"line_number":31,"context_line":"from swift.common.daemon import Daemon"},{"line_number":32,"context_line":"from swift.common.storage_policy import POLICIES"},{"line_number":33,"context_line":""},{"line_number":34,"context_line":"re_rsync_tempfile \u003d re.compile(r\u0027^\\..*\\.([a-zA-Z0-9_]){6}$\u0027)"},{"line_number":35,"context_line":""},{"line_number":36,"context_line":""}],"source_content_type":"text/x-python","patch_set":3,"id":"fa0719c6_7e190288","line":33,"in_reply_to":"fa0719c6_ff3a5097","updated":"2016-03-22 20:06:44.000000000","message":"Done","commit_id":"45e1effc02c0355f5cc3d028f7f718270e48d63f"},{"author":{"_account_id":6968,"name":"Christian Schwede","email":"cschwede@nvidia.com","username":"cschwede"},"change_message_id":"b4ee2c6e88d1aea89d556168519d758afac6d20d","unresolved":false,"context_lines":[{"line_number":57,"context_line":"            # if we can\u0027t parse the real config (generally a KeyError on"},{"line_number":58,"context_line":"            # __file__, or SystemExit on no object-replicator section) we use"},{"line_number":59,"context_line":"            # a very conservative default of reclaim_age"},{"line_number":60,"context_line":"            default \u003d int(self.conf.get(\u0027reclaim_age\u0027, 86400 * 7))"},{"line_number":61,"context_line":"        else:"},{"line_number":62,"context_line":"            replicator_rsync_timeout \u003d int(replicator_config.get("},{"line_number":63,"context_line":"                \u0027rsync_timeout\u0027, replicator.DEFAULT_RSYNC_TIMEOUT))"}],"source_content_type":"text/x-python","patch_set":7,"id":"fa0719c6_cf61413b","line":60,"updated":"2016-03-23 15:49:21.000000000","message":"Hmm, now you lost me - why reclaim_age? I\u0027m probably missing something here...","commit_id":"cd4dc9bfc907ba218dc4476694dc16b7b256498c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"4156f3d8f5957cfea4105aaa50b5d01f871d0587","unresolved":false,"context_lines":[{"line_number":57,"context_line":"            # if we can\u0027t parse the real config (generally a KeyError on"},{"line_number":58,"context_line":"            # __file__, or SystemExit on no object-replicator section) we use"},{"line_number":59,"context_line":"            # a very conservative default of reclaim_age"},{"line_number":60,"context_line":"            default \u003d int(self.conf.get(\u0027reclaim_age\u0027, 86400 * 7))"},{"line_number":61,"context_line":"        else:"},{"line_number":62,"context_line":"            replicator_rsync_timeout \u003d int(replicator_config.get("},{"line_number":63,"context_line":"                \u0027rsync_timeout\u0027, replicator.DEFAULT_RSYNC_TIMEOUT))"}],"source_content_type":"text/x-python","patch_set":7,"id":"fa0719c6_bf0c0a3c","line":60,"updated":"2016-03-23 16:24:52.000000000","message":"in diskfile.py we have defined ONE_WEEK \u003d 604800 - could we move that to utils and then re-use here?","commit_id":"cd4dc9bfc907ba218dc4476694dc16b7b256498c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"5ae9843cfda3c242f3c73c0afb0b4baf13028adf","unresolved":false,"context_lines":[{"line_number":57,"context_line":"            # if we can\u0027t parse the real config (generally a KeyError on"},{"line_number":58,"context_line":"            # __file__, or SystemExit on no object-replicator section) we use"},{"line_number":59,"context_line":"            # a very conservative default of reclaim_age"},{"line_number":60,"context_line":"            default \u003d int(self.conf.get(\u0027reclaim_age\u0027, 86400 * 7))"},{"line_number":61,"context_line":"        else:"},{"line_number":62,"context_line":"            replicator_rsync_timeout \u003d int(replicator_config.get("},{"line_number":63,"context_line":"                \u0027rsync_timeout\u0027, replicator.DEFAULT_RSYNC_TIMEOUT))"}],"source_content_type":"text/x-python","patch_set":7,"id":"fa0719c6_10c99559","line":60,"in_reply_to":"fa0719c6_bf0c0a3c","updated":"2016-03-23 17:30:03.000000000","message":"didn\u0027t see anything for ONE_DAY","commit_id":"cd4dc9bfc907ba218dc4476694dc16b7b256498c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"4156f3d8f5957cfea4105aaa50b5d01f871d0587","unresolved":false,"context_lines":[{"line_number":58,"context_line":"            # __file__, or SystemExit on no object-replicator section) we use"},{"line_number":59,"context_line":"            # a very conservative default of reclaim_age"},{"line_number":60,"context_line":"            default \u003d int(self.conf.get(\u0027reclaim_age\u0027, 86400 * 7))"},{"line_number":61,"context_line":"        else:"},{"line_number":62,"context_line":"            replicator_rsync_timeout \u003d int(replicator_config.get("},{"line_number":63,"context_line":"                \u0027rsync_timeout\u0027, replicator.DEFAULT_RSYNC_TIMEOUT))"},{"line_number":64,"context_line":"            # Here we can do some light math for ops and use the *replicator\u0027s*"}],"source_content_type":"text/x-python","patch_set":7,"id":"fa0719c6_1f70960a","line":61,"updated":"2016-03-23 16:24:52.000000000","message":"I\u0027m curious - why did you choose to use an else clause here rather than continuing in the try clause? I can\u0027t see how these lines would raise either of the exceptions being caught.\n\n#stillLearningPython ;)","commit_id":"cd4dc9bfc907ba218dc4476694dc16b7b256498c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"5ae9843cfda3c242f3c73c0afb0b4baf13028adf","unresolved":false,"context_lines":[{"line_number":58,"context_line":"            # __file__, or SystemExit on no object-replicator section) we use"},{"line_number":59,"context_line":"            # a very conservative default of reclaim_age"},{"line_number":60,"context_line":"            default \u003d int(self.conf.get(\u0027reclaim_age\u0027, 86400 * 7))"},{"line_number":61,"context_line":"        else:"},{"line_number":62,"context_line":"            replicator_rsync_timeout \u003d int(replicator_config.get("},{"line_number":63,"context_line":"                \u0027rsync_timeout\u0027, replicator.DEFAULT_RSYNC_TIMEOUT))"},{"line_number":64,"context_line":"            # Here we can do some light math for ops and use the *replicator\u0027s*"}],"source_content_type":"text/x-python","patch_set":7,"id":"fa0719c6_d0371d4b","line":61,"in_reply_to":"fa0719c6_1f70960a","updated":"2016-03-23 17:30:03.000000000","message":"search pep8 for \"Additionally, for all try/except clauses, limit the try clause to the absolute minimum amount of code necessary\"\n\nin all honesty I\u0027d be worried about catching a KeyError in readconf it wasn\u0027t so old and tested elsewhere.","commit_id":"cd4dc9bfc907ba218dc4476694dc16b7b256498c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"4156f3d8f5957cfea4105aaa50b5d01f871d0587","unresolved":false,"context_lines":[{"line_number":64,"context_line":"            # Here we can do some light math for ops and use the *replicator\u0027s*"},{"line_number":65,"context_line":"            # rsync_timeout (plus 15 mins to avoid deleting local tempfiles"},{"line_number":66,"context_line":"            # before the remote replicator kills it\u0027s rsync)"},{"line_number":67,"context_line":"            default \u003d replicator_rsync_timeout + 900"},{"line_number":68,"context_line":"        self.rsync_tempfile_timeout \u003d config_auto_int_value("},{"line_number":69,"context_line":"            self.conf.get(\u0027rsync_tempfile_timeout\u0027), default)"},{"line_number":70,"context_line":""}],"source_content_type":"text/x-python","patch_set":7,"id":"fa0719c6_bfa24a49","line":67,"updated":"2016-03-23 16:24:52.000000000","message":"this is the rsync timeout for this node\u0027s replicator daemon, but the replicator that creates the temp file could be governed by a different timeout (I think? do we anywhere recommend *in docs* using consistent config files throughout a cluster?)","commit_id":"cd4dc9bfc907ba218dc4476694dc16b7b256498c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"5ae9843cfda3c242f3c73c0afb0b4baf13028adf","unresolved":false,"context_lines":[{"line_number":64,"context_line":"            # Here we can do some light math for ops and use the *replicator\u0027s*"},{"line_number":65,"context_line":"            # rsync_timeout (plus 15 mins to avoid deleting local tempfiles"},{"line_number":66,"context_line":"            # before the remote replicator kills it\u0027s rsync)"},{"line_number":67,"context_line":"            default \u003d replicator_rsync_timeout + 900"},{"line_number":68,"context_line":"        self.rsync_tempfile_timeout \u003d config_auto_int_value("},{"line_number":69,"context_line":"            self.conf.get(\u0027rsync_tempfile_timeout\u0027), default)"},{"line_number":70,"context_line":""}],"source_content_type":"text/x-python","patch_set":7,"id":"fa0719c6_d03d5dde","line":67,"in_reply_to":"fa0719c6_bfa24a49","updated":"2016-03-23 17:30:03.000000000","message":"if you start the auditor and replicator using swift-init then they are both in the same object-server config \"path\" (either a file a dir).  that \"path\" gets dropping in the __file__ key when parsed by readconf - so I should be getting out the same thing.  But if you\u0027re starting the daemons by `swift-object-\u003cservice\u003e /path/to/random/config.conf` then yes it *could* be different files - but I don\u0027t know anyone who dose that.  However if that is the case - then god i hope - they don\u0027t have an *unused* [object-replicator] section in their object-auditor.conf - so the auto should catch the SystemExit from readconf and fallback to the larger default value of 1 day.","commit_id":"cd4dc9bfc907ba218dc4476694dc16b7b256498c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"4156f3d8f5957cfea4105aaa50b5d01f871d0587","unresolved":false,"context_lines":[{"line_number":256,"context_line":"            self.logger.error(_(\u0027ERROR Object %(obj)s failed audit and was\u0027"},{"line_number":257,"context_line":"                                \u0027 quarantined: %(err)s\u0027),"},{"line_number":258,"context_line":"                              {\u0027obj\u0027: location, \u0027err\u0027: err})"},{"line_number":259,"context_line":"        self.passes +\u003d 1"},{"line_number":260,"context_line":"        # _ondisk_info attr is initialized to None and filled in by open"},{"line_number":261,"context_line":"        ondisk_info_dict \u003d df._ondisk_info or {}"},{"line_number":262,"context_line":"        if \u0027unexpected\u0027 in ondisk_info_dict:"}],"source_content_type":"text/x-python","patch_set":7,"id":"fa0719c6_6932b860","line":259,"updated":"2016-03-23 16:24:52.000000000","message":"if the diskfile got quarantined then the datadir is gone to quarantine (including any unexpected files) and the remaining lines are redundant - the unlink will swallow the OSError for dir not existing - but could return early in the except clause above (not forgetting to increment passes first).","commit_id":"cd4dc9bfc907ba218dc4476694dc16b7b256498c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"5ae9843cfda3c242f3c73c0afb0b4baf13028adf","unresolved":false,"context_lines":[{"line_number":256,"context_line":"            self.logger.error(_(\u0027ERROR Object %(obj)s failed audit and was\u0027"},{"line_number":257,"context_line":"                                \u0027 quarantined: %(err)s\u0027),"},{"line_number":258,"context_line":"                              {\u0027obj\u0027: location, \u0027err\u0027: err})"},{"line_number":259,"context_line":"        self.passes +\u003d 1"},{"line_number":260,"context_line":"        # _ondisk_info attr is initialized to None and filled in by open"},{"line_number":261,"context_line":"        ondisk_info_dict \u003d df._ondisk_info or {}"},{"line_number":262,"context_line":"        if \u0027unexpected\u0027 in ondisk_info_dict:"}],"source_content_type":"text/x-python","patch_set":7,"id":"fa0719c6_6356492b","line":259,"in_reply_to":"fa0719c6_6932b860","updated":"2016-03-23 17:30:03.000000000","message":"i actually like this method a lot more with single return - so I feel like a guard return would be a step back - but I see what you\u0027re saying about needlessly trying to quarantine something that was eaten - i honestly didn\u0027t know/think about that behavior.\n\nProbably should have quarantine clear out _ondisk_info - or just do it ourselves in the except block?\n\nI\u0027m going to say it probably happens infrequently enough it doesn\u0027t matter - but I\u0027m really glad to know it doesn\u0027t blow up (hopefully the tests would have caught that - but who knows)?","commit_id":"cd4dc9bfc907ba218dc4476694dc16b7b256498c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"4156f3d8f5957cfea4105aaa50b5d01f871d0587","unresolved":false,"context_lines":[{"line_number":261,"context_line":"        ondisk_info_dict \u003d df._ondisk_info or {}"},{"line_number":262,"context_line":"        if \u0027unexpected\u0027 in ondisk_info_dict:"},{"line_number":263,"context_line":"            is_rsync_tempfile \u003d lambda fpath: RE_RSYNC_TEMPFILE.match("},{"line_number":264,"context_line":"                os.path.basename(fpath))"},{"line_number":265,"context_line":"            rsync_tempfile_paths \u003d filter(is_rsync_tempfile,"},{"line_number":266,"context_line":"                                          ondisk_info_dict[\u0027unexpected\u0027])"},{"line_number":267,"context_line":"            mtime \u003d time.time() - self.rsync_tempfile_timeout"}],"source_content_type":"text/x-python","patch_set":7,"id":"fa0719c6_8c476576","line":264,"updated":"2016-03-23 16:24:52.000000000","message":"NIT: this step might be simpler if diskfile get_ondisk_files had just stashed the basenames in \u0027unexpected\u0027.\n\nIn fact you could just use the re_rsync_tempfile.match method as the first param for the filter call, then change unlink_paths_older_than to accept a dir and list of files, then move the map + partial pattern into unlink_paths_older_than:\n\nlike here https://gist.github.com/alistairncoles/8e7aeaf64780299a512e\n\nhmm, totally a nit. I\u0027m just excited by all this functional programming ;)","commit_id":"cd4dc9bfc907ba218dc4476694dc16b7b256498c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"5ae9843cfda3c242f3c73c0afb0b4baf13028adf","unresolved":false,"context_lines":[{"line_number":261,"context_line":"        ondisk_info_dict \u003d df._ondisk_info or {}"},{"line_number":262,"context_line":"        if \u0027unexpected\u0027 in ondisk_info_dict:"},{"line_number":263,"context_line":"            is_rsync_tempfile \u003d lambda fpath: RE_RSYNC_TEMPFILE.match("},{"line_number":264,"context_line":"                os.path.basename(fpath))"},{"line_number":265,"context_line":"            rsync_tempfile_paths \u003d filter(is_rsync_tempfile,"},{"line_number":266,"context_line":"                                          ondisk_info_dict[\u0027unexpected\u0027])"},{"line_number":267,"context_line":"            mtime \u003d time.time() - self.rsync_tempfile_timeout"}],"source_content_type":"text/x-python","patch_set":7,"id":"fa0719c6_236f312c","line":264,"in_reply_to":"fa0719c6_8c476576","updated":"2016-03-23 17:30:03.000000000","message":"I did note the lambda here was annoying when writing it - and considered making some changes - but opted not too because:\n\ni like the full path interface on the \u0027unexpected\u0027 - seems to follow the the pattern in that structure (e.g. data_file key is a full path).  I think it\u0027s just a better interface to not have to *also* know about the df._datadir to use the names in the \u0027unexpected\u0027 key.\n\nI like the interface of unlink_paths_older_than as i wrote it supporting potentially files in different root directories.\n\nIf i was worried about the lambda/basename overhead I could have also just rewritten the regex to allow for arbitrary values up to the last \"/\" - but lambda/basename/regex seemed easiest to debug.","commit_id":"cd4dc9bfc907ba218dc4476694dc16b7b256498c"}],"test/unit/common/test_utils.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"8298d5da967ead9f180509a24023e45dadc6248d","unresolved":false,"context_lines":[{"line_number":3464,"context_line":"        self.touch(os.path.join(self.tempdir, \u0027test\u0027))"},{"line_number":3465,"context_line":"        with self.high_resolution_getmtime():"},{"line_number":3466,"context_line":"            utils.unlink_older_than(self.tempdir, time.time())"},{"line_number":3467,"context_line":"        self.assertEqual([], os.listdir(self.tempdir))"},{"line_number":3468,"context_line":""},{"line_number":3469,"context_line":"    def test_unlink_not_old_enough(self):"},{"line_number":3470,"context_line":"        start \u003d time.time()"}],"source_content_type":"text/x-python","patch_set":2,"id":"1af94dfe_6c1319b9","line":3467,"updated":"2016-03-17 04:39:27.000000000","message":"i couldn\u0027t find any existing tests for this in this file with a couple searches in vim, but in retrospect I didn\u0027t try adding an exception and seeing if anyone was testing it incidentally...","commit_id":"d8e82f72cb4eb1c70aa4dce509d6b66753d086d1"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"4156f3d8f5957cfea4105aaa50b5d01f871d0587","unresolved":false,"context_lines":[{"line_number":3451,"context_line":""},{"line_number":3452,"context_line":"    def test_unlink_older_than_path_not_exists(self):"},{"line_number":3453,"context_line":"        path \u003d os.path.join(self.tempdir, \u0027does-not-exist\u0027)"},{"line_number":3454,"context_line":"        utils.unlink_older_than(path, time.time())"},{"line_number":3455,"context_line":""},{"line_number":3456,"context_line":"    def test_unlink_older_than_file(self):"},{"line_number":3457,"context_line":"        path \u003d os.path.join(self.tempdir, \u0027some-file\u0027)"}],"source_content_type":"text/x-python","patch_set":7,"id":"fa0719c6_e45d4d48","line":3454,"updated":"2016-03-23 16:24:52.000000000","message":"one day someone will want to delete this test for making no assertions ;) ... defensive comment?","commit_id":"cd4dc9bfc907ba218dc4476694dc16b7b256498c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"5ae9843cfda3c242f3c73c0afb0b4baf13028adf","unresolved":false,"context_lines":[{"line_number":3451,"context_line":""},{"line_number":3452,"context_line":"    def test_unlink_older_than_path_not_exists(self):"},{"line_number":3453,"context_line":"        path \u003d os.path.join(self.tempdir, \u0027does-not-exist\u0027)"},{"line_number":3454,"context_line":"        utils.unlink_older_than(path, time.time())"},{"line_number":3455,"context_line":""},{"line_number":3456,"context_line":"    def test_unlink_older_than_file(self):"},{"line_number":3457,"context_line":"        path \u003d os.path.join(self.tempdir, \u0027some-file\u0027)"}],"source_content_type":"text/x-python","patch_set":7,"id":"fa0719c6_c60be322","line":3454,"in_reply_to":"fa0719c6_e45d4d48","updated":"2016-03-23 17:30:03.000000000","message":"Done","commit_id":"cd4dc9bfc907ba218dc4476694dc16b7b256498c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"4156f3d8f5957cfea4105aaa50b5d01f871d0587","unresolved":false,"context_lines":[{"line_number":3497,"context_line":""},{"line_number":3498,"context_line":"    def test_unlink_not_exists_paths(self):"},{"line_number":3499,"context_line":"        path \u003d os.path.join(self.tempdir, \u0027does-not-exist\u0027)"},{"line_number":3500,"context_line":"        utils.unlink_paths_older_than([path], time.time())"},{"line_number":3501,"context_line":""},{"line_number":3502,"context_line":""},{"line_number":3503,"context_line":"class TestSwiftInfo(unittest.TestCase):"}],"source_content_type":"text/x-python","patch_set":7,"id":"fa0719c6_09333963","line":3500,"updated":"2016-03-23 16:24:52.000000000","message":"all looks good, broke utils, saw tests fail.","commit_id":"cd4dc9bfc907ba218dc4476694dc16b7b256498c"}],"test/unit/obj/test_auditor.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"8298d5da967ead9f180509a24023e45dadc6248d","unresolved":false,"context_lines":[{"line_number":273,"context_line":"        open(self.disk_file._datadir, \u0027w\u0027)"},{"line_number":274,"context_line":""},{"line_number":275,"context_line":"        # after we turn the crank ..."},{"line_number":276,"context_line":"        auditor_worker.object_audit(location)"},{"line_number":277,"context_line":""},{"line_number":278,"context_line":"        # ... it should get quarantined"},{"line_number":279,"context_line":"        self.assertFalse(os.path.exists(self.disk_file._datadir))"}],"source_content_type":"text/x-python","patch_set":2,"id":"1af94dfe_ac8401d4","line":276,"updated":"2016-03-17 04:39:27.000000000","message":"this actually hits the condition where the open fails to set _ondisk_info if you were wondering...","commit_id":"d8e82f72cb4eb1c70aa4dce509d6b66753d086d1"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"8298d5da967ead9f180509a24023e45dadc6248d","unresolved":false,"context_lines":[{"line_number":355,"context_line":""},{"line_number":356,"context_line":"        # that\u0027s someone elses problem"},{"line_number":357,"context_line":"        self.assertIn(os.path.basename(random_crazy_file_path),"},{"line_number":358,"context_line":"                      os.listdir(self.disk_file._datadir))"},{"line_number":359,"context_line":""},{"line_number":360,"context_line":"    def test_generic_exception_handling(self):"},{"line_number":361,"context_line":"        auditor_worker \u003d auditor.AuditorWorker(self.conf, self.logger,"}],"source_content_type":"text/x-python","patch_set":2,"id":"1af94dfe_ec272910","line":358,"updated":"2016-03-17 04:39:27.000000000","message":"I don\u0027t normally like these long story based unittests that hit a bunch of different behaviors, but for some reason I did that here...","commit_id":"d8e82f72cb4eb1c70aa4dce509d6b66753d086d1"},{"author":{"_account_id":6968,"name":"Christian Schwede","email":"cschwede@nvidia.com","username":"cschwede"},"change_message_id":"085283967253a86a75d47452b76a02d4e72c59c4","unresolved":false,"context_lines":[{"line_number":270,"context_line":""},{"line_number":271,"context_line":"        # instead of a datadir, we\u0027ll make a file!"},{"line_number":272,"context_line":"        mkdirs(os.path.dirname(self.disk_file._datadir))"},{"line_number":273,"context_line":"        open(self.disk_file._datadir, \u0027w\u0027)"},{"line_number":274,"context_line":""},{"line_number":275,"context_line":"        # after we turn the crank ..."},{"line_number":276,"context_line":"        auditor_worker.object_audit(location)"}],"source_content_type":"text/x-python","patch_set":3,"id":"fa0719c6_4cebd8b0","line":273,"updated":"2016-03-22 09:27:15.000000000","message":"So, mkdirs creates the parent directories of the arg and therefore self.disk_file._datadir is a file eventually - ok","commit_id":"45e1effc02c0355f5cc3d028f7f718270e48d63f"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"4156f3d8f5957cfea4105aaa50b5d01f871d0587","unresolved":false,"context_lines":[{"line_number":278,"context_line":"                              policy\u003dPOLICIES.legacy))"},{"line_number":279,"context_line":"        self.assertEqual(auditor_worker.errors, 1)"},{"line_number":280,"context_line":""},{"line_number":281,"context_line":"    def test_audit_location_gets_quantined(self):"},{"line_number":282,"context_line":"        auditor_worker \u003d auditor.AuditorWorker(self.conf, self.logger,"},{"line_number":283,"context_line":"                                               self.rcache, self.devices)"},{"line_number":284,"context_line":""}],"source_content_type":"text/x-python","patch_set":7,"id":"fa0719c6_322e72a5","line":281,"updated":"2016-03-23 16:24:52.000000000","message":"s/quantined/quarantined/","commit_id":"cd4dc9bfc907ba218dc4476694dc16b7b256498c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"4156f3d8f5957cfea4105aaa50b5d01f871d0587","unresolved":false,"context_lines":[{"line_number":319,"context_line":"        \"\"\""},{"line_number":320,"context_line":"        with open(config_path, \u0027w\u0027) as f:"},{"line_number":321,"context_line":"            f.write(textwrap.dedent(stub_config))"},{"line_number":322,"context_line":"        # the Daemon loader will hand the object-audtior config to the"},{"line_number":323,"context_line":"        # auditor who will build the workers from it"},{"line_number":324,"context_line":"        conf \u003d readconf(config_path, \u0027object-auditor\u0027)"},{"line_number":325,"context_line":"        auditor_worker \u003d auditor.AuditorWorker(conf, self.logger,"}],"source_content_type":"text/x-python","patch_set":7,"id":"fa0719c6_da5f41e3","line":322,"updated":"2016-03-23 16:24:52.000000000","message":"s/audtior/auditor/","commit_id":"cd4dc9bfc907ba218dc4476694dc16b7b256498c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"5ae9843cfda3c242f3c73c0afb0b4baf13028adf","unresolved":false,"context_lines":[{"line_number":319,"context_line":"        \"\"\""},{"line_number":320,"context_line":"        with open(config_path, \u0027w\u0027) as f:"},{"line_number":321,"context_line":"            f.write(textwrap.dedent(stub_config))"},{"line_number":322,"context_line":"        # the Daemon loader will hand the object-audtior config to the"},{"line_number":323,"context_line":"        # auditor who will build the workers from it"},{"line_number":324,"context_line":"        conf \u003d readconf(config_path, \u0027object-auditor\u0027)"},{"line_number":325,"context_line":"        auditor_worker \u003d auditor.AuditorWorker(conf, self.logger,"}],"source_content_type":"text/x-python","patch_set":7,"id":"fa0719c6_2643bf43","line":322,"in_reply_to":"fa0719c6_da5f41e3","updated":"2016-03-23 17:30:03.000000000","message":"Done","commit_id":"cd4dc9bfc907ba218dc4476694dc16b7b256498c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"4156f3d8f5957cfea4105aaa50b5d01f871d0587","unresolved":false,"context_lines":[{"line_number":330,"context_line":"                         604800)"},{"line_number":331,"context_line":"        stub_config \u003d \"\"\""},{"line_number":332,"context_line":"        [DEFAULT]"},{"line_number":333,"context_line":"        reclaim_age \u003d 1209600"},{"line_number":334,"context_line":"        [object-replicator]"},{"line_number":335,"context_line":"        [object-auditor]"},{"line_number":336,"context_line":"        rsync_tempfile_timeout \u003d auto"}],"source_content_type":"text/x-python","patch_set":7,"id":"fa0719c6_0c3987f2","line":333,"updated":"2016-03-23 16:24:52.000000000","message":"this test is missing the case when reclaim_age is not in config file, although if i change the default of one week that does get caught in the following test","commit_id":"cd4dc9bfc907ba218dc4476694dc16b7b256498c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"5ae9843cfda3c242f3c73c0afb0b4baf13028adf","unresolved":false,"context_lines":[{"line_number":330,"context_line":"                         604800)"},{"line_number":331,"context_line":"        stub_config \u003d \"\"\""},{"line_number":332,"context_line":"        [DEFAULT]"},{"line_number":333,"context_line":"        reclaim_age \u003d 1209600"},{"line_number":334,"context_line":"        [object-replicator]"},{"line_number":335,"context_line":"        [object-auditor]"},{"line_number":336,"context_line":"        rsync_tempfile_timeout \u003d auto"}],"source_content_type":"text/x-python","patch_set":7,"id":"fa0719c6_e63ee789","line":333,"in_reply_to":"fa0719c6_0c3987f2","updated":"2016-03-23 17:30:03.000000000","message":"Done","commit_id":"cd4dc9bfc907ba218dc4476694dc16b7b256498c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"4156f3d8f5957cfea4105aaa50b5d01f871d0587","unresolved":false,"context_lines":[{"line_number":341,"context_line":"        auditor_worker \u003d auditor.AuditorWorker(conf, self.logger,"},{"line_number":342,"context_line":"                                               self.rcache, self.devices)"},{"line_number":343,"context_line":"        # if the object-replicator section will parse but does not override"},{"line_number":344,"context_line":"        # the default rsync_timeout we assume the default rsync_tiemout value"},{"line_number":345,"context_line":"        # and add 15mins"},{"line_number":346,"context_line":"        self.assertEqual(auditor_worker.rsync_tempfile_timeout,"},{"line_number":347,"context_line":"                         replicator.DEFAULT_RSYNC_TIMEOUT + 900)"}],"source_content_type":"text/x-python","patch_set":7,"id":"fa0719c6_2c39ebd7","line":344,"updated":"2016-03-23 16:24:52.000000000","message":"s/tiemout/timeout/","commit_id":"cd4dc9bfc907ba218dc4476694dc16b7b256498c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"5ae9843cfda3c242f3c73c0afb0b4baf13028adf","unresolved":false,"context_lines":[{"line_number":341,"context_line":"        auditor_worker \u003d auditor.AuditorWorker(conf, self.logger,"},{"line_number":342,"context_line":"                                               self.rcache, self.devices)"},{"line_number":343,"context_line":"        # if the object-replicator section will parse but does not override"},{"line_number":344,"context_line":"        # the default rsync_timeout we assume the default rsync_tiemout value"},{"line_number":345,"context_line":"        # and add 15mins"},{"line_number":346,"context_line":"        self.assertEqual(auditor_worker.rsync_tempfile_timeout,"},{"line_number":347,"context_line":"                         replicator.DEFAULT_RSYNC_TIMEOUT + 900)"}],"source_content_type":"text/x-python","patch_set":7,"id":"fa0719c6_264d7f3e","line":344,"in_reply_to":"fa0719c6_2c39ebd7","updated":"2016-03-23 17:30:03.000000000","message":"Done","commit_id":"cd4dc9bfc907ba218dc4476694dc16b7b256498c"}]}