)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"8898ed7e81094158d38b5056b6697345f6712ea5","unresolved":false,"context_lines":[{"line_number":23,"context_line":"- Add tests"},{"line_number":24,"context_line":"- Resolve XXX"},{"line_number":25,"context_line":"- PUT returns 201, POST returns 202 - may get visible by client"},{"line_number":26,"context_line":"- Ensure a cleanup of tmp*/ that does not sweep fresh temporary files"},{"line_number":27,"context_line":""},{"line_number":28,"context_line":"Change-Id: I64b0d8fdb2ffce786f56665a74ed7eb2603abfda"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":8,"id":"5ff73747_75db034f","line":26,"updated":"2017-05-02 17:16:37.000000000","message":"What do we do about this currently? If the connection dies or something before the commit, how does master deal with cleanup?","commit_id":"78f47b42db89987c611761d87ef3ff2e2c28b491"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"c0b267d3eed42ceed4607a7a41993714a7a727ba","unresolved":false,"context_lines":[{"line_number":23,"context_line":"- Add tests"},{"line_number":24,"context_line":"- Resolve XXX"},{"line_number":25,"context_line":"- PUT returns 201, POST returns 202 - may get visible by client"},{"line_number":26,"context_line":"- Ensure a cleanup of tmp*/ that does not sweep fresh temporary files"},{"line_number":27,"context_line":""},{"line_number":28,"context_line":"Change-Id: I64b0d8fdb2ffce786f56665a74ed7eb2603abfda"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":8,"id":"3f044301_22878ada","line":26,"in_reply_to":"5ff73747_75db034f","updated":"2017-05-03 21:12:18.000000000","message":"No idea yet, sorry. Presumably it\u0027s indistinguishable from what we used to do for many years, before O_TMPFLE. But yes, I must investigate if the vestiges of that remain and are sufficient. That\u0027s why it\u0027s in TODO.","commit_id":"78f47b42db89987c611761d87ef3ff2e2c28b491"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"c842cb5ed37538269282bb08f3652c0bc749970a","unresolved":false,"context_lines":[{"line_number":7,"context_line":"Replace MIME with PUT+POST for EC and Encryption"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"Actually, we do not force a replacement outright, for compatibility"},{"line_number":10,"context_line":"and safety reasons. Instead, a configuration option is provided to"},{"line_number":11,"context_line":"enable the new protocol."},{"line_number":12,"context_line":""},{"line_number":13,"context_line":"This is the 3-phase version (PUT+POST+POST) with a persisting DiskFile."},{"line_number":14,"context_line":""}],"source_content_type":"text/x-gerrit-commit-message","patch_set":27,"id":"5f7c97a3_fb8127a2","line":11,"range":{"start_line":10,"start_character":20,"end_line":11,"end_character":24},"updated":"2018-06-14 17:43:39.000000000","message":"what is the thinking w.r.t. upgrade? we\u0027d want people to be using the new version at some point, or do we plan to maintain support for older object servers forever? I\u0027d like to think the new protocol was the default and we handled any compatibility issues in the simplest possible way but flagged up an \u0027upgrade your object server\u0027 warning message.","commit_id":"061690bbc467be6a017aaddd5c3cc79632ec55b8"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"c492dc9f4e7cc1ce5e7ba1e00fcdb6589b351be1","unresolved":false,"context_lines":[{"line_number":7,"context_line":"Replace MIME with PUT+POST for EC and Encryption"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"Actually, we do not force a replacement outright, for compatibility"},{"line_number":10,"context_line":"and safety reasons. Instead, a configuration option is provided to"},{"line_number":11,"context_line":"enable the new protocol."},{"line_number":12,"context_line":""},{"line_number":13,"context_line":"This is the 3-phase version (PUT+POST+POST) with a persisting DiskFile."},{"line_number":14,"context_line":""}],"source_content_type":"text/x-gerrit-commit-message","patch_set":27,"id":"5f7c97a3_e1d35ad2","line":11,"range":{"start_line":10,"start_character":20,"end_line":11,"end_character":24},"in_reply_to":"5f7c97a3_fb8127a2","updated":"2018-06-20 01:09:29.000000000","message":"My original plan was to make the new protocol default right away. The dot-in-path permits a proxy to detect old object servers. So, the patch some time around version #15 had a proxy controller try the new protocol first, then fall back (by substituting MIMEPutter) if 400 was returned. In order not to do it on every PUT, a timer was set up for 20 minutes. This allowed to upgrade storage and proxies in any order, with zero downtime.\n\nClay pointed out that this sort of behavior was difficult to test, so I threw all of it away. Once we have the code in and tested by setting the boolean explicitly, we can re-visit the automatic enablement.","commit_id":"061690bbc467be6a017aaddd5c3cc79632ec55b8"}],"bin/swift-object-server":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"cf5190bda49a26788552ee732aefcf3d11eb261a","unresolved":false,"context_lines":[{"line_number":26,"context_line":"    parser.add_option(\u0027-d\u0027, \u0027--swift-dir\u0027, default\u003d\u0027/etc/swift\u0027,"},{"line_number":27,"context_line":"                      dest\u003d\u0027swift_dir\u0027, help\u003d\u0027Path to swift directory\u0027)"},{"line_number":28,"context_line":"    parser.add_option(\u0027-n\u0027, \u0027--no-daemon\u0027, action\u003d\"store_false\", dest\u003d\"daemon\","},{"line_number":29,"context_line":"                      default\u003dTrue, help\u003d\"Do not drop privileges\")"},{"line_number":30,"context_line":"    conf_file, options \u003d utils.parse_options(parser\u003dparser)"},{"line_number":31,"context_line":"    if utils.set_swift_dir(options[\u0027swift_dir\u0027]):"},{"line_number":32,"context_line":"        storage_policy.reload_storage_policies()"}],"source_content_type":"application/octet-stream","patch_set":25,"id":"1f9dbf25_07179eef","line":29,"range":{"start_line":29,"start_character":49,"end_line":29,"end_character":64},"updated":"2018-03-03 01:37:58.000000000","message":"So how is this different from something like\n\n swift-init object start --no-daemon --config 1\n\n?","commit_id":"79fec861768bb219815e6cab80d3b1320e393b1e"}],"etc/proxy-server.conf-sample":[{"author":{"_account_id":9625,"name":"Thiago da Silva","email":"thiagodasilva@gmail.com","username":"thiago"},"change_message_id":"04a3c3dd769d7521372336fffcda15b841e5e8f5","unresolved":false,"context_lines":[{"line_number":289,"context_line":"# write_affinity_node_count \u003d"},{"line_number":290,"context_line":"# write_affinity_handoff_delete_count \u003d"},{"line_number":291,"context_line":""},{"line_number":292,"context_line":"# This setting allows for a controlled migration and/or experiments with"},{"line_number":293,"context_line":"# an updated protocol for EC PUT between proxy and object nodes."},{"line_number":294,"context_line":"# Upgrade all your storage first, then flip this on. But unless you run"},{"line_number":295,"context_line":"# a re-implementation of object server in Go, Python 3, or a Unicorn-based"},{"line_number":296,"context_line":"# accelerator, you probably don\u0027t care. May be required for an all-TLS"},{"line_number":297,"context_line":"# setup if Apache is used. We\u0027re shooting for making this a default"},{"line_number":298,"context_line":"# some time after Queens."},{"line_number":299,"context_line":"# XXX actually it\u0027s true for the time being"},{"line_number":300,"context_line":"# put_api_v1 \u003d true"},{"line_number":301,"context_line":""}],"source_content_type":"application/octet-stream","patch_set":32,"id":"5f7c97a3_5be6a415","line":298,"range":{"start_line":292,"start_character":2,"end_line":298,"end_character":24},"updated":"2018-08-02 20:50:49.000000000","message":"This is a but confusing, I\u0027d clean it up a bit. We should just say what the option will do, keep it simple and to the point.","commit_id":"83256db01ae98b902df7691cad155bbdc6024080"}],"swift/common/bufferedhttp.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"8898ed7e81094158d38b5056b6697345f6712ea5","unresolved":false,"context_lines":[{"line_number":215,"context_line":"            logging.exception(_(\u0027Error encoding to UTF-8: %s\u0027), str(e))"},{"line_number":216,"context_line":"    path \u003d quote(\u0027/\u0027 + device + \u0027/\u0027 + str(partition) + path)"},{"line_number":217,"context_line":"    if version:"},{"line_number":218,"context_line":"        path \u003d \u0027/./%s%s\u0027 % (version, path)"},{"line_number":219,"context_line":"    return http_connect_raw("},{"line_number":220,"context_line":"        ipaddr, port, method, path, headers, query_string, ssl)"},{"line_number":221,"context_line":""}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_fcd7004f","line":218,"updated":"2017-05-02 17:16:37.000000000","message":"Whoa! Interesting... I\u0027m not sure I get the advantage of it yet, though; isn\u0027t everything gated off the backend headers currently, anyway?","commit_id":"78f47b42db89987c611761d87ef3ff2e2c28b491"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"c0b267d3eed42ceed4607a7a41993714a7a727ba","unresolved":false,"context_lines":[{"line_number":215,"context_line":"            logging.exception(_(\u0027Error encoding to UTF-8: %s\u0027), str(e))"},{"line_number":216,"context_line":"    path \u003d quote(\u0027/\u0027 + device + \u0027/\u0027 + str(partition) + path)"},{"line_number":217,"context_line":"    if version:"},{"line_number":218,"context_line":"        path \u003d \u0027/./%s%s\u0027 % (version, path)"},{"line_number":219,"context_line":"    return http_connect_raw("},{"line_number":220,"context_line":"        ipaddr, port, method, path, headers, query_string, ssl)"},{"line_number":221,"context_line":""}],"source_content_type":"text/x-python","patch_set":8,"id":"3f044301_023ece28","line":218,"in_reply_to":"5ff73747_fcd7004f","updated":"2017-05-03 21:12:18.000000000","message":"The problem with headers is that the server is free to ignore additional headers. So, in order to know if the node is new, the proxy has to send a null request first (X-New-Protocol-Probe: v1), see if it got a suitable reply (X-New-Protocol-Supported: v1). MIMEPutter actually does exactly that, but it piggy-backs the reply on 100-Continue in order to hide the round-trip. But one of our objectives is to relieve 100-Continue from this function.\n\nWhat we do instead is ask for forgiveness, not permission. We send a PUT with path that is guaranteed to trip a 400 in the old code. If the requests succeeds, we\u0027re fine: there\u0027s no overhead incurred. If we get 400 back, we retry with the legacy MIMEPutter. In order not to do this on every request, we continue using the legacy putter straight for 20 minutes.\n\nAt Atlanta PTG, there were some wild ideas floated, because nobody knew for sure if a guaranteed way to trip old code existed. Someone (Sam?) proposed putting a non-UTF-8 character sequence into the path even. But it turned out that a simple period is sufficient.","commit_id":"78f47b42db89987c611761d87ef3ff2e2c28b491"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"192eab1ab5c1f33b952011933f016b45197656d9","unresolved":false,"context_lines":[{"line_number":208,"context_line":"        device \u003d device.encode(\"utf-8\")"},{"line_number":209,"context_line":"    path \u003d quote(\u0027/\u0027 + device + \u0027/\u0027 + str(partition) + path)"},{"line_number":210,"context_line":"    if version:"},{"line_number":211,"context_line":"        path \u003d \u0027/./%s%s\u0027 % (version, path)"},{"line_number":212,"context_line":"    return http_connect_raw("},{"line_number":213,"context_line":"        ipaddr, port, method, path, headers, query_string, ssl)"},{"line_number":214,"context_line":""}],"source_content_type":"text/x-python","patch_set":18,"id":"7f515b1d_cecaa24c","line":211,"range":{"start_line":211,"start_character":18,"end_line":211,"end_character":22},"updated":"2017-09-27 17:16:26.000000000","message":"make this  separate delimited path segment so that version string can be variable length","commit_id":"ad7a81a738ce86371535bbdf94b6c359ae0bb2cd"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"69e3c053e721241daf71d7118ec81ea161e5d7b8","unresolved":false,"context_lines":[{"line_number":208,"context_line":"        device \u003d device.encode(\"utf-8\")"},{"line_number":209,"context_line":"    path \u003d quote(\u0027/\u0027 + device + \u0027/\u0027 + str(partition) + path)"},{"line_number":210,"context_line":"    if version:"},{"line_number":211,"context_line":"        path \u003d \u0027/./%s%s\u0027 % (version, path)"},{"line_number":212,"context_line":"    return http_connect_raw("},{"line_number":213,"context_line":"        ipaddr, port, method, path, headers, query_string, ssl)"},{"line_number":214,"context_line":""}],"source_content_type":"text/x-python","patch_set":18,"id":"7f515b1d_e16b2fb4","line":211,"range":{"start_line":211,"start_character":18,"end_line":211,"end_character":22},"in_reply_to":"7f515b1d_cecaa24c","updated":"2017-09-27 17:22:07.000000000","message":"The variable \"path\" always starts with slash at this point, so the result is delimited. However, I made a note to verify that it is the case, so we never jam the v1 and path by accident.","commit_id":"ad7a81a738ce86371535bbdf94b6c359ae0bb2cd"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"c842cb5ed37538269282bb08f3652c0bc749970a","unresolved":false,"context_lines":[{"line_number":207,"context_line":"    if isinstance(device, six.text_type):"},{"line_number":208,"context_line":"        device \u003d device.encode(\"utf-8\")"},{"line_number":209,"context_line":"    path \u003d quote(\u0027/\u0027 + device + \u0027/\u0027 + str(partition) + path)"},{"line_number":210,"context_line":"    if version:"},{"line_number":211,"context_line":"        path \u003d \u0027/./\u0027 + version + path"},{"line_number":212,"context_line":"    return http_connect_raw("},{"line_number":213,"context_line":"        ipaddr, port, method, path, headers, query_string, ssl)"},{"line_number":214,"context_line":""}],"source_content_type":"text/x-python","patch_set":27,"id":"5f7c97a3_1bae6381","line":211,"range":{"start_line":210,"start_character":4,"end_line":211,"end_character":37},"updated":"2018-06-14 17:43:39.000000000","message":"I don\u0027t like how this object server specific feature spreads into a common class. I think we ought to able to used a 100 Continue header to figure out server versions, much like is already being done for the multiphase MIME approach.\n\n(forgive me if that idea has already been tried and discarded)","commit_id":"061690bbc467be6a017aaddd5c3cc79632ec55b8"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"c492dc9f4e7cc1ce5e7ba1e00fcdb6589b351be1","unresolved":false,"context_lines":[{"line_number":207,"context_line":"    if isinstance(device, six.text_type):"},{"line_number":208,"context_line":"        device \u003d device.encode(\"utf-8\")"},{"line_number":209,"context_line":"    path \u003d quote(\u0027/\u0027 + device + \u0027/\u0027 + str(partition) + path)"},{"line_number":210,"context_line":"    if version:"},{"line_number":211,"context_line":"        path \u003d \u0027/./\u0027 + version + path"},{"line_number":212,"context_line":"    return http_connect_raw("},{"line_number":213,"context_line":"        ipaddr, port, method, path, headers, query_string, ssl)"},{"line_number":214,"context_line":""}],"source_content_type":"text/x-python","patch_set":27,"id":"5f7c97a3_c1af3e29","line":211,"range":{"start_line":210,"start_character":4,"end_line":211,"end_character":37},"in_reply_to":"5f7c97a3_1bae6381","updated":"2018-06-20 01:09:29.000000000","message":"Yes, this sadly was required. We actually discussed it face to face at PTG, and looked for any way for proxy to force a fault (400, not 500) in object server. Sam even proposed using a non-UTF8 character that caused a validation fault. I think you were in attendance, but forgot. IIRC it was Bristol.\n\nPlacing this inside the class provides a degree of encapsulation - the caller does not know how the version is transmitted. But perhaps you are right and this is a pointless obfuscation. But then if this (dot in path) is the way we do it on the wire, there\u0027s no better place in the code, I think.","commit_id":"061690bbc467be6a017aaddd5c3cc79632ec55b8"},{"author":{"_account_id":9625,"name":"Thiago da Silva","email":"thiagodasilva@gmail.com","username":"thiago"},"change_message_id":"04a3c3dd769d7521372336fffcda15b841e5e8f5","unresolved":false,"context_lines":[{"line_number":208,"context_line":"        device \u003d device.encode(\"utf-8\")"},{"line_number":209,"context_line":"    path \u003d quote(\u0027/\u0027 + device + \u0027/\u0027 + str(partition) + path)"},{"line_number":210,"context_line":"    if version:"},{"line_number":211,"context_line":"        path \u003d \u0027/./\u0027 + version + path"},{"line_number":212,"context_line":"    return http_connect_raw("},{"line_number":213,"context_line":"        ipaddr, port, method, path, headers, query_string, ssl)"},{"line_number":214,"context_line":""}],"source_content_type":"text/x-python","patch_set":32,"id":"5f7c97a3_9b8afc1b","line":211,"range":{"start_line":211,"start_character":16,"end_line":211,"end_character":19},"updated":"2018-08-02 20:50:49.000000000","message":"I think this deserves a comment, or maybe I\u0027ll just understand later??\njust thinking of future maintainers looking at this and wondering what were we thinking....\n\n...\n\n\n...\n\nah, ok, so this is part of the whole discussion into how to support upgrades. Well, if this is what we endup with, then it does deserve at least a long comment explaining that here.","commit_id":"83256db01ae98b902df7691cad155bbdc6024080"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"76782e5bcd51d06b67169efe197ae16bbf2b57ff","unresolved":false,"context_lines":[{"line_number":208,"context_line":"        device \u003d device.encode(\"utf-8\")"},{"line_number":209,"context_line":"    path \u003d quote(\u0027/\u0027 + device + \u0027/\u0027 + str(partition) + path)"},{"line_number":210,"context_line":"    if version:"},{"line_number":211,"context_line":"        path \u003d \u0027/./\u0027 + version + path"},{"line_number":212,"context_line":"    return http_connect_raw("},{"line_number":213,"context_line":"        ipaddr, port, method, path, headers, query_string, ssl)"},{"line_number":214,"context_line":""}],"source_content_type":"text/x-python","patch_set":32,"id":"3f79a3b5_c622f252","line":211,"range":{"start_line":211,"start_character":16,"end_line":211,"end_character":19},"in_reply_to":"3f79a3b5_28451371","updated":"2018-08-09 07:24:06.000000000","message":"hmm, yeah, that\u0027s a very good point. In my playing, here is how the PUT + POST + POST looks in the object server logs: http://paste.openstack.org/show/727696/\n\nMaybe some identifier in the proxy -\u003e object server PATH or API, maybe it\u0027s time to increase the api version.. but only of the internal API. Maybe new verb, but I really don\u0027t want to be continually inventing new verbs.\n\nOh and now looking at the logs, there\u0027s a bug there. Initial PUT has a transaction id, but is lost in the 2 POSTs. The transaction id will need to be plumbed through to make the requests traceable. Maybe the transaction ID is enough to let us know this was a PPP request.","commit_id":"83256db01ae98b902df7691cad155bbdc6024080"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"1ed0dd252026e225621493b8cc2d4493c675ee73","unresolved":false,"context_lines":[{"line_number":208,"context_line":"        device \u003d device.encode(\"utf-8\")"},{"line_number":209,"context_line":"    path \u003d quote(\u0027/\u0027 + device + \u0027/\u0027 + str(partition) + path)"},{"line_number":210,"context_line":"    if version:"},{"line_number":211,"context_line":"        path \u003d \u0027/./\u0027 + version + path"},{"line_number":212,"context_line":"    return http_connect_raw("},{"line_number":213,"context_line":"        ipaddr, port, method, path, headers, query_string, ssl)"},{"line_number":214,"context_line":""}],"source_content_type":"text/x-python","patch_set":32,"id":"3f79a3b5_01c4940d","line":211,"range":{"start_line":211,"start_character":16,"end_line":211,"end_character":19},"in_reply_to":"3f79a3b5_466394de","updated":"2018-09-11 21:05:09.000000000","message":"I did add trans-id just where Matthew suggested.","commit_id":"83256db01ae98b902df7691cad155bbdc6024080"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a56346d93124dea05e7aded037ac4fce3fad9eb3","unresolved":false,"context_lines":[{"line_number":208,"context_line":"        device \u003d device.encode(\"utf-8\")"},{"line_number":209,"context_line":"    path \u003d quote(\u0027/\u0027 + device + \u0027/\u0027 + str(partition) + path)"},{"line_number":210,"context_line":"    if version:"},{"line_number":211,"context_line":"        path \u003d \u0027/./\u0027 + version + path"},{"line_number":212,"context_line":"    return http_connect_raw("},{"line_number":213,"context_line":"        ipaddr, port, method, path, headers, query_string, ssl)"},{"line_number":214,"context_line":""}],"source_content_type":"text/x-python","patch_set":32,"id":"3f79a3b5_466394de","line":211,"range":{"start_line":211,"start_character":16,"end_line":211,"end_character":19},"in_reply_to":"3f79a3b5_c622f252","updated":"2018-09-10 22:38:27.000000000","message":"XXX the trans-id thing needs to be fixed","commit_id":"83256db01ae98b902df7691cad155bbdc6024080"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"3a4e11c77137716276fc6c095aa1ad508299c7a0","unresolved":false,"context_lines":[{"line_number":208,"context_line":"        device \u003d device.encode(\"utf-8\")"},{"line_number":209,"context_line":"    path \u003d quote(\u0027/\u0027 + device + \u0027/\u0027 + str(partition) + path)"},{"line_number":210,"context_line":"    if version:"},{"line_number":211,"context_line":"        path \u003d \u0027/./\u0027 + version + path"},{"line_number":212,"context_line":"    return http_connect_raw("},{"line_number":213,"context_line":"        ipaddr, port, method, path, headers, query_string, ssl)"},{"line_number":214,"context_line":""}],"source_content_type":"text/x-python","patch_set":32,"id":"3f79a3b5_28451371","line":211,"range":{"start_line":211,"start_character":16,"end_line":211,"end_character":19},"in_reply_to":"5f7c97a3_37c6c243","updated":"2018-08-04 00:48:58.000000000","message":"So here\u0027s a question: do we want a new protocol version just for the PUT path, or for all possible methods?\n\nI\u0027m trying to think of what might cause us to need another new version after this... and whether we\u0027ll want to consider the resource versioned, or the method, or some other level of abstraction entirely...\n\nAnother thing to think about is how/whether to surface version usage in logs... that\u0027s certainly one advantage to embedding the version in the path or making up an alternate request method -- you can come up with a grep pattern for logs to know whether there are any servers still using an old version...","commit_id":"83256db01ae98b902df7691cad155bbdc6024080"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"a37021642dfe43c1b138023ff8b0bd2616bca57e","unresolved":false,"context_lines":[{"line_number":208,"context_line":"        device \u003d device.encode(\"utf-8\")"},{"line_number":209,"context_line":"    path \u003d quote(\u0027/\u0027 + device + \u0027/\u0027 + str(partition) + path)"},{"line_number":210,"context_line":"    if version:"},{"line_number":211,"context_line":"        path \u003d \u0027/./\u0027 + version + path"},{"line_number":212,"context_line":"    return http_connect_raw("},{"line_number":213,"context_line":"        ipaddr, port, method, path, headers, query_string, ssl)"},{"line_number":214,"context_line":""}],"source_content_type":"text/x-python","patch_set":32,"id":"5f7c97a3_37c6c243","line":211,"range":{"start_line":211,"start_character":16,"end_line":211,"end_character":19},"in_reply_to":"5f7c97a3_9b8afc1b","updated":"2018-08-03 06:27:10.000000000","message":"Al\u0027s code uses content-type so this mangling the path isn\u0027t required. Question is, should we use this or squash Al\u0027s patch in. In other words how much do we care about this change to the path.","commit_id":"83256db01ae98b902df7691cad155bbdc6024080"}],"swift/common/exceptions.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"3a4e11c77137716276fc6c095aa1ad508299c7a0","unresolved":false,"context_lines":[{"line_number":109,"context_line":"    pass"},{"line_number":110,"context_line":""},{"line_number":111,"context_line":""},{"line_number":112,"context_line":"class DiskFileUseError(DiskFileError):"},{"line_number":113,"context_line":"    pass"},{"line_number":114,"context_line":""},{"line_number":115,"context_line":""}],"source_content_type":"text/x-python","patch_set":32,"id":"3f79a3b5_881ea76e","line":112,"range":{"start_line":112,"start_character":14,"end_line":112,"end_character":22},"updated":"2018-08-04 00:48:58.000000000","message":"Reading the names of the other DiskFile* exceptions, I get some immediate sense of what may trigger the error -- but UseError seems rather vague...","commit_id":"83256db01ae98b902df7691cad155bbdc6024080"}],"swift/common/request_helpers.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"c4b0816434ae5f4b7237d7f53180dfcc86190c68","unresolved":false,"context_lines":[{"line_number":108,"context_line":"    \"\"\""},{"line_number":109,"context_line":"    segs \u003d request.path.lstrip(\u0027/\u0027).split(\u0027/\u0027, 2)"},{"line_number":110,"context_line":"    if len(segs) \u003d\u003d 3 and segs[0] \u003d\u003d \u0027.\u0027:"},{"line_number":111,"context_line":"        if segs[1] !\u003d \u0027v1\u0027:"},{"line_number":112,"context_line":"            raise HTTPBadRequest(body\u003d\"Only v1 for now\", request\u003drequest,"},{"line_number":113,"context_line":"                                 content_type\u003d\u0027text/plain\u0027)"},{"line_number":114,"context_line":"        path \u003d \u0027/\u0027 + segs[2]"}],"source_content_type":"text/x-python","patch_set":32,"id":"3f79a3b5_69b35f0b","line":111,"range":{"start_line":111,"start_character":23,"end_line":111,"end_character":25},"updated":"2018-09-10 15:51:15.000000000","message":"Assuming we keep the tactics here -- I feel like this should come in as a acceptable_versions arg. Do we *want* account/container servers responding to /./v1/... requests yet?","commit_id":"83256db01ae98b902df7691cad155bbdc6024080"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"75867a9c210c01e910797df485d336bb97fd2337","unresolved":false,"context_lines":[{"line_number":108,"context_line":"    \"\"\""},{"line_number":109,"context_line":"    segs \u003d request.path.lstrip(\u0027/\u0027).split(\u0027/\u0027, 2)"},{"line_number":110,"context_line":"    if len(segs) \u003d\u003d 3 and segs[0] \u003d\u003d \u0027.\u0027:"},{"line_number":111,"context_line":"        if segs[1] !\u003d \u0027v1\u0027:"},{"line_number":112,"context_line":"            raise HTTPBadRequest(body\u003d\"Only v1 for now\", request\u003drequest,"},{"line_number":113,"context_line":"                                 content_type\u003d\u0027text/plain\u0027)"},{"line_number":114,"context_line":"        path \u003d \u0027/\u0027 + segs[2]"}],"source_content_type":"text/x-python","patch_set":32,"id":"3f79a3b5_098ccfd5","line":111,"range":{"start_line":111,"start_character":23,"end_line":111,"end_character":25},"in_reply_to":"3f79a3b5_69b35f0b","updated":"2018-09-10 22:56:26.000000000","message":"Done, I think.","commit_id":"83256db01ae98b902df7691cad155bbdc6024080"},{"author":{"_account_id":9625,"name":"Thiago da Silva","email":"thiagodasilva@gmail.com","username":"thiago"},"change_message_id":"04a3c3dd769d7521372336fffcda15b841e5e8f5","unresolved":false,"context_lines":[{"line_number":106,"context_line":"              everything\u0027s okay"},{"line_number":107,"context_line":"    :raises HTTPBadRequest: if something\u0027s not okay"},{"line_number":108,"context_line":"    \"\"\""},{"line_number":109,"context_line":"    segs \u003d request.path.lstrip(\u0027/\u0027).split(\u0027/\u0027, 2)"},{"line_number":110,"context_line":"    if len(segs) \u003d\u003d 3 and segs[0] \u003d\u003d \u0027.\u0027:"},{"line_number":111,"context_line":"        if segs[1] !\u003d \u0027v1\u0027:"},{"line_number":112,"context_line":"            raise HTTPBadRequest(body\u003d\"Only v1 for now\", request\u003drequest,"},{"line_number":113,"context_line":"                                 content_type\u003d\u0027text/plain\u0027)"},{"line_number":114,"context_line":"        path \u003d \u0027/\u0027 + segs[2]"},{"line_number":115,"context_line":"    else:"},{"line_number":116,"context_line":"        path \u003d request.path"},{"line_number":117,"context_line":"    try:"},{"line_number":118,"context_line":"        segs \u003d split_path(unquote(path), minsegs, maxsegs, rest_with_last)"},{"line_number":119,"context_line":"        validate_device_partition(segs[0], segs[1])"}],"source_content_type":"text/x-python","patch_set":32,"id":"5f7c97a3_bbfa5874","line":116,"range":{"start_line":109,"start_character":4,"end_line":116,"end_character":27},"updated":"2018-08-02 20:50:49.000000000","message":"maybe move this into split_path? would also be nice if we added some tests to test_request_helpers.py","commit_id":"83256db01ae98b902df7691cad155bbdc6024080"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"a37021642dfe43c1b138023ff8b0bd2616bca57e","unresolved":false,"context_lines":[{"line_number":106,"context_line":"              everything\u0027s okay"},{"line_number":107,"context_line":"    :raises HTTPBadRequest: if something\u0027s not okay"},{"line_number":108,"context_line":"    \"\"\""},{"line_number":109,"context_line":"    segs \u003d request.path.lstrip(\u0027/\u0027).split(\u0027/\u0027, 2)"},{"line_number":110,"context_line":"    if len(segs) \u003d\u003d 3 and segs[0] \u003d\u003d \u0027.\u0027:"},{"line_number":111,"context_line":"        if segs[1] !\u003d \u0027v1\u0027:"},{"line_number":112,"context_line":"            raise HTTPBadRequest(body\u003d\"Only v1 for now\", request\u003drequest,"},{"line_number":113,"context_line":"                                 content_type\u003d\u0027text/plain\u0027)"},{"line_number":114,"context_line":"        path \u003d \u0027/\u0027 + segs[2]"},{"line_number":115,"context_line":"    else:"},{"line_number":116,"context_line":"        path \u003d request.path"},{"line_number":117,"context_line":"    try:"},{"line_number":118,"context_line":"        segs \u003d split_path(unquote(path), minsegs, maxsegs, rest_with_last)"},{"line_number":119,"context_line":"        validate_device_partition(segs[0], segs[1])"}],"source_content_type":"text/x-python","patch_set":32,"id":"5f7c97a3_f7e84aa8","line":116,"range":{"start_line":109,"start_character":4,"end_line":116,"end_character":27},"in_reply_to":"5f7c97a3_bbfa5874","updated":"2018-08-03 06:27:10.000000000","message":"If we did decide to use the content-type version rather then path \u0027/./\u0027 then this wouldn\u0027t be required so also cleaned up.\n\nI\u0027ve tested this patch a little I might go apply Al\u0027s and test it out too.","commit_id":"83256db01ae98b902df7691cad155bbdc6024080"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"3a4e11c77137716276fc6c095aa1ad508299c7a0","unresolved":false,"context_lines":[{"line_number":120,"context_line":"        return segs"},{"line_number":121,"context_line":"    except ValueError as err:"},{"line_number":122,"context_line":"        raise HTTPBadRequest(body\u003dstr(err), request\u003drequest,"},{"line_number":123,"context_line":"                             content_type\u003d\u0027text/plain\u0027)"},{"line_number":124,"context_line":""},{"line_number":125,"context_line":""},{"line_number":126,"context_line":"def is_user_meta(server_type, key):"}],"source_content_type":"text/x-python","patch_set":32,"id":"3f79a3b5_8351a811","line":123,"updated":"2018-08-04 00:48:58.000000000","message":"Right, so we do this in split_and_validate_path rather than get_name_and_placement to keep it general -- this way it affects account and container server, too...\n\nDo the DB servers have much need of this at the moment? Oh... I wonder it this might\u0027ve been a good thing to have when PUTing shard ranges with the sharder...","commit_id":"83256db01ae98b902df7691cad155bbdc6024080"}],"swift/common/storage_policy.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"8898ed7e81094158d38b5056b6697345f6712ea5","unresolved":false,"context_lines":[{"line_number":926,"context_line":"        return repr(_POLICIES)"},{"line_number":927,"context_line":""},{"line_number":928,"context_line":""},{"line_number":929,"context_line":"def reload_storage_policies(swift_conf\u003dNone):"},{"line_number":930,"context_line":"    \"\"\""},{"line_number":931,"context_line":"    Reload POLICIES from ``swift.conf``."},{"line_number":932,"context_line":"    \"\"\""}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_fc6ee094","line":929,"updated":"2017-05-02 17:16:37.000000000","message":"Yeah, that\u0027s surely a good idea.","commit_id":"78f47b42db89987c611761d87ef3ff2e2c28b491"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"c0b267d3eed42ceed4607a7a41993714a7a727ba","unresolved":false,"context_lines":[{"line_number":926,"context_line":"        return repr(_POLICIES)"},{"line_number":927,"context_line":""},{"line_number":928,"context_line":""},{"line_number":929,"context_line":"def reload_storage_policies(swift_conf\u003dNone):"},{"line_number":930,"context_line":"    \"\"\""},{"line_number":931,"context_line":"    Reload POLICIES from ``swift.conf``."},{"line_number":932,"context_line":"    \"\"\""}],"source_content_type":"text/x-python","patch_set":8,"id":"3f044301_62cb521e","line":929,"in_reply_to":"5ff73747_fc6ee094","updated":"2017-05-03 21:12:18.000000000","message":"You know how Churchill said, \"Americans can always be trusted to do the right thing, after they exhausted all the alternatives\"? This is obviously better but I only did it because I was unable to get module order and monkey patching work.","commit_id":"78f47b42db89987c611761d87ef3ff2e2c28b491"}],"swift/common/wsgi.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"8898ed7e81094158d38b5056b6697345f6712ea5","unresolved":false,"context_lines":[{"line_number":482,"context_line":"    def bind_ports(self, daemon\u003dTrue):"},{"line_number":483,"context_line":"        \"\"\""},{"line_number":484,"context_line":"        Bind the one listen socket for this strategy and drop privileges"},{"line_number":485,"context_line":"        (since the parent process will never need to bind again)."},{"line_number":486,"context_line":"        \"\"\""},{"line_number":487,"context_line":""},{"line_number":488,"context_line":"        try:"}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_dc00c4ae","line":485,"updated":"2017-05-02 17:16:37.000000000","message":"So is this part not true when running interactively? Why was this necessary?","commit_id":"78f47b42db89987c611761d87ef3ff2e2c28b491"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"c0b267d3eed42ceed4607a7a41993714a7a727ba","unresolved":false,"context_lines":[{"line_number":482,"context_line":"    def bind_ports(self, daemon\u003dTrue):"},{"line_number":483,"context_line":"        \"\"\""},{"line_number":484,"context_line":"        Bind the one listen socket for this strategy and drop privileges"},{"line_number":485,"context_line":"        (since the parent process will never need to bind again)."},{"line_number":486,"context_line":"        \"\"\""},{"line_number":487,"context_line":""},{"line_number":488,"context_line":"        try:"}],"source_content_type":"text/x-python","patch_set":8,"id":"3f044301_e53acc32","line":485,"in_reply_to":"5ff73747_dc00c4ae","updated":"2017-05-03 21:12:18.000000000","message":"Confusingly enough, this has nothing to do with binding ports. I added it because my test-ecput.py script blows up when I run it as an ordinary user. It happens because a non-root user cannot run setsid(). I think I\u0027ll remove this once test-ecput.py is not necessary anymore, when I add a proper functional test. I think we monkey-patch something in there when running in-process, so the problem does not arise.","commit_id":"78f47b42db89987c611761d87ef3ff2e2c28b491"}],"swift/obj/diskfile.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"cebc4f2009f5c91a67d19103054b32c6c1ae8589","unresolved":false,"context_lines":[{"line_number":2737,"context_line":"        # P3"},{"line_number":2738,"context_line":"        fp \u003d open(\"/tmp/dump\", \u0027a\u0027)"},{"line_number":2739,"context_line":"        fp.write(\"ECDiskFileWriter.put %s\\n\" % self._name)"},{"line_number":2740,"context_line":"        fp.close()"},{"line_number":2741,"context_line":"        fi \u003d None"},{"line_number":2742,"context_line":"        cleanup \u003d True"},{"line_number":2743,"context_line":"        if self._extension \u003d\u003d \u0027.data\u0027:"}],"source_content_type":"text/x-python","patch_set":1,"id":"3a461143_ef610013","line":2740,"updated":"2017-02-02 03:50:09.000000000","message":"I think you would like q\n\nhttps://pypi.python.org/pypi/q","commit_id":"442f483da525b9f7c9f6bdd35f412a1fb92c8974"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"8898ed7e81094158d38b5056b6697345f6712ea5","unresolved":false,"context_lines":[{"line_number":1562,"context_line":"        if ctype_timestamp:"},{"line_number":1563,"context_line":"            ctype_timestamp \u003d Timestamp(ctype_timestamp)"},{"line_number":1564,"context_line":"        # XXX what arguments and keyword arguments are passed through?!"},{"line_number":1565,"context_line":"        # XXX Is there a way for frag_index to leak down here but be discarded?"},{"line_number":1566,"context_line":"        filename \u003d self.manager.make_on_disk_filename("},{"line_number":1567,"context_line":"            timestamp, self._extension, ctype_timestamp\u003dctype_timestamp,"},{"line_number":1568,"context_line":"            *a, **kw)"}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_3cea08ac","line":1565,"updated":"2017-05-02 17:16:37.000000000","message":"FWIW, grepping for `._put(`, I just see metadata and cleanup (possibly hard-coded to be True) both passed positionally, and frag_index as a kwarg. I\u0027d be all for making frag_index an explicit kwarg.","commit_id":"78f47b42db89987c611761d87ef3ff2e2c28b491"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"8898ed7e81094158d38b5056b6697345f6712ea5","unresolved":false,"context_lines":[{"line_number":1580,"context_line":"        \"\"\""},{"line_number":1581,"context_line":"        raise NotImplementedError"},{"line_number":1582,"context_line":""},{"line_number":1583,"context_line":"    def hold(self, metadata):"},{"line_number":1584,"context_line":"        \"\"\""},{"line_number":1585,"context_line":"        Same as put(), but leave the file in the temp directory."},{"line_number":1586,"context_line":""}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_9c291cf4","line":1583,"updated":"2017-05-02 17:16:37.000000000","message":"(definite bike-shedding, so I apologize)\n\n\"stage\", maybe? Or maybe I\u0027ve gotten too used to git jargon...","commit_id":"78f47b42db89987c611761d87ef3ff2e2c28b491"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"8898ed7e81094158d38b5056b6697345f6712ea5","unresolved":false,"context_lines":[{"line_number":1582,"context_line":""},{"line_number":1583,"context_line":"    def hold(self, metadata):"},{"line_number":1584,"context_line":"        \"\"\""},{"line_number":1585,"context_line":"        Same as put(), but leave the file in the temp directory."},{"line_number":1586,"context_line":""},{"line_number":1587,"context_line":"        :returns: An ID that can be used later to find the file."},{"line_number":1588,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_fc670051","line":1585,"updated":"2017-05-02 17:16:37.000000000","message":"Interesting that we describe it as \"Same as put()\" while put just raises NotImplementedError -- a sign this should live elsewhere?\n\nDo we have any near-term plans to allow two-phase commits for replicated objects?","commit_id":"78f47b42db89987c611761d87ef3ff2e2c28b491"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"8898ed7e81094158d38b5056b6697345f6712ea5","unresolved":false,"context_lines":[{"line_number":2598,"context_line":"        if not self._datadir:"},{"line_number":2599,"context_line":"            raise DiskFileUseError()"},{"line_number":2600,"context_line":""},{"line_number":2601,"context_line":"        self._frag_index \u003d frag_index"},{"line_number":2602,"context_line":""},{"line_number":2603,"context_line":"        # XXX maybe factor this out into \"make_on_disk_filename_from_metadata\"?"},{"line_number":2604,"context_line":"        timestamp \u003d Timestamp(metadata[\u0027X-Timestamp\u0027])"}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_3c70c8dc","line":2601,"updated":"2017-05-02 17:16:37.000000000","message":"This isn\u0027t available in metadata somewhere? X-Object-Sysmeta-Ec-Frag-Index? I suppose that comes in the second phase... is there any reason *not* send it in the initial headers?","commit_id":"78f47b42db89987c611761d87ef3ff2e2c28b491"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"d530d7d387064ca77d05e0a9940295fa59989c53","unresolved":false,"context_lines":[{"line_number":2598,"context_line":"        if not self._datadir:"},{"line_number":2599,"context_line":"            raise DiskFileUseError()"},{"line_number":2600,"context_line":""},{"line_number":2601,"context_line":"        self._frag_index \u003d frag_index"},{"line_number":2602,"context_line":""},{"line_number":2603,"context_line":"        # XXX maybe factor this out into \"make_on_disk_filename_from_metadata\"?"},{"line_number":2604,"context_line":"        timestamp \u003d Timestamp(metadata[\u0027X-Timestamp\u0027])"}],"source_content_type":"text/x-python","patch_set":8,"id":"ff0f0b1f_fb0ceb94","line":2601,"in_reply_to":"3f044301_f99390a2","updated":"2017-05-24 01:52:04.000000000","message":"the frag index is *in* the pyeclib headers of every fragment - you can\u0027t *have* a datafile without at least *it* knowing it\u0027s frag_index\n\nBut Pete is definitely right about the plumbing being weird in the proxy putters - when it comes to handoffs you don\u0027t really decide which fragment archive you\u0027re going to send down until *after* the connection is established and PUT headers have been sent (we have to send PUT headers for the expect-100 behavior).\n\nWe *could* have the object server inspect the chunks as the come in from the wire tho if it matters... I like pete\u0027s point about doesn\u0027t matter tho.","commit_id":"78f47b42db89987c611761d87ef3ff2e2c28b491"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"9c98977a75d350825c281216b4e93da4986b1b00","unresolved":false,"context_lines":[{"line_number":2598,"context_line":"        if not self._datadir:"},{"line_number":2599,"context_line":"            raise DiskFileUseError()"},{"line_number":2600,"context_line":""},{"line_number":2601,"context_line":"        self._frag_index \u003d frag_index"},{"line_number":2602,"context_line":""},{"line_number":2603,"context_line":"        # XXX maybe factor this out into \"make_on_disk_filename_from_metadata\"?"},{"line_number":2604,"context_line":"        timestamp \u003d Timestamp(metadata[\u0027X-Timestamp\u0027])"}],"source_content_type":"text/x-python","patch_set":8,"id":"3f044301_f99390a2","line":2601,"in_reply_to":"5ff73747_3c70c8dc","updated":"2017-05-04 03:29:09.000000000","message":"Goodness gratious, not that nightmare again. As it happens, I did send them with the original headers - it\u0027s even in one of the previous versions of this patch. But since we create a putter after we established the connection, and sent the headers, proxy needs to know the index when opening connections. HOWEVER, the mapping is not straightforward in presence of connection failures. Current code only computes the map of putters to indexes when it already knows which connections succeeded. If we send the index alongside the initial connection, we need it before that point.\n\nSo, I went and implemented the algorithm from _determine_chunk_destinations() in iterative way. I made a map, in which indexes could be leased. Then, a provisional index is sent with a connection. If connection succeeded, it was upgraded to permanent, otherwise freed, so as to be included into the computation for next connection.\n\nThat was before Atlanta PTG. There, Kota modified _determine_chunk_destinations() to make it even more complicated and blew my iterative implementation of it out of the water. Rather than picking up the pieces, I postponed sending index until POST. It\u0027s not like anyone cares if the temp file has a correct index of not, anyway.","commit_id":"78f47b42db89987c611761d87ef3ff2e2c28b491"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"8898ed7e81094158d38b5056b6697345f6712ea5","unresolved":false,"context_lines":[{"line_number":2600,"context_line":""},{"line_number":2601,"context_line":"        self._frag_index \u003d frag_index"},{"line_number":2602,"context_line":""},{"line_number":2603,"context_line":"        # XXX maybe factor this out into \"make_on_disk_filename_from_metadata\"?"},{"line_number":2604,"context_line":"        timestamp \u003d Timestamp(metadata[\u0027X-Timestamp\u0027])"},{"line_number":2605,"context_line":"        ctype_timestamp \u003d metadata.get(\u0027Content-Type-Timestamp\u0027)"},{"line_number":2606,"context_line":"        if ctype_timestamp:"}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_df8f9673","line":2603,"updated":"2017-05-02 17:16:37.000000000","message":"Who else might use it? Auditor, I guess, to make sure we found this thing where we expected to find it?","commit_id":"78f47b42db89987c611761d87ef3ff2e2c28b491"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"9c98977a75d350825c281216b4e93da4986b1b00","unresolved":false,"context_lines":[{"line_number":2600,"context_line":""},{"line_number":2601,"context_line":"        self._frag_index \u003d frag_index"},{"line_number":2602,"context_line":""},{"line_number":2603,"context_line":"        # XXX maybe factor this out into \"make_on_disk_filename_from_metadata\"?"},{"line_number":2604,"context_line":"        timestamp \u003d Timestamp(metadata[\u0027X-Timestamp\u0027])"},{"line_number":2605,"context_line":"        ctype_timestamp \u003d metadata.get(\u0027Content-Type-Timestamp\u0027)"},{"line_number":2606,"context_line":"        if ctype_timestamp:"}],"source_content_type":"text/x-python","patch_set":8,"id":"3f044301_79ed4024","line":2603,"in_reply_to":"5ff73747_df8f9673","updated":"2017-05-04 03:29:09.000000000","message":"I stole it from somewhere almost verbatim, so I guess I should identify its origin and factor with it.","commit_id":"78f47b42db89987c611761d87ef3ff2e2c28b491"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"8898ed7e81094158d38b5056b6697345f6712ea5","unresolved":false,"context_lines":[{"line_number":2614,"context_line":""},{"line_number":2615,"context_line":"        # Not sure if it is completely correct to invalidate _before_"},{"line_number":2616,"context_line":"        # renaming, rather than after, but that\u0027s what put() does, so"},{"line_number":2617,"context_line":"        # replicate it for the time being."},{"line_number":2618,"context_line":"        self.manager.invalidate_hash(dirname(self._datadir))"},{"line_number":2619,"context_line":""},{"line_number":2620,"context_line":"        renamer(temp_id, target_path)"}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_bfebaada","line":2617,"updated":"2017-05-02 17:16:37.000000000","message":"Given the possibility of failure between the two, I guess it\u0027s better to risk doing unnecessary work (by invalidating early) than to risk having not-properly-invalidated hashes, yeah? I *think* I\u0027m on board with this as-is.","commit_id":"78f47b42db89987c611761d87ef3ff2e2c28b491"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"8898ed7e81094158d38b5056b6697345f6712ea5","unresolved":false,"context_lines":[{"line_number":2935,"context_line":"            remove_file(os.path.join(self._datadir, purge_file))"},{"line_number":2936,"context_line":"        self.manager.invalidate_hash(dirname(self._datadir))"},{"line_number":2937,"context_line":""},{"line_number":2938,"context_line":"    def _finalize_durable(self, data_file_path, durable_data_file_path):"},{"line_number":2939,"context_line":"        exc \u003d None"},{"line_number":2940,"context_line":"        try:"},{"line_number":2941,"context_line":"            try:"}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_1fa17e96","line":2938,"updated":"2017-05-02 17:16:37.000000000","message":"Right, so now these hang off ECDiskFile instead of ECDiskFileWriter... were many changes necessary? I haven\u0027t gotten to doing an aligned diff yet...","commit_id":"78f47b42db89987c611761d87ef3ff2e2c28b491"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"9c98977a75d350825c281216b4e93da4986b1b00","unresolved":false,"context_lines":[{"line_number":2935,"context_line":"            remove_file(os.path.join(self._datadir, purge_file))"},{"line_number":2936,"context_line":"        self.manager.invalidate_hash(dirname(self._datadir))"},{"line_number":2937,"context_line":""},{"line_number":2938,"context_line":"    def _finalize_durable(self, data_file_path, durable_data_file_path):"},{"line_number":2939,"context_line":"        exc \u003d None"},{"line_number":2940,"context_line":"        try:"},{"line_number":2941,"context_line":"            try:"}],"source_content_type":"text/x-python","patch_set":8,"id":"3f044301_59cba4a4","line":2938,"in_reply_to":"5ff73747_1fa17e96","updated":"2017-05-04 03:29:09.000000000","message":"I strived not to damage this unnecessarily, modulo getting rid of self._diskfile prefixes.","commit_id":"78f47b42db89987c611761d87ef3ff2e2c28b491"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"cf5190bda49a26788552ee732aefcf3d11eb261a","unresolved":false,"context_lines":[{"line_number":2661,"context_line":"                os.close(fd)"},{"line_number":2662,"context_line":"            except OSError:"},{"line_number":2663,"context_line":"                pass"},{"line_number":2664,"context_line":"            if (dfw is None) or (not dfw.put_succeeded):"},{"line_number":2665,"context_line":"                # Try removing the temp file only if put did NOT succeed."},{"line_number":2666,"context_line":"                #"},{"line_number":2667,"context_line":"                # dfw.put_succeeded is set to True after renamer() succeeds in"}],"source_content_type":"text/x-python","patch_set":25,"id":"1f9dbf25_df3176a4","side":"PARENT","line":2664,"range":{"start_line":2664,"start_character":37,"end_line":2664,"end_character":54},"updated":"2018-03-03 01:37:58.000000000","message":"Hmm... so before we were judging by the writer\u0027s put_completed state, but now we track that ourselves...","commit_id":"5cf96230c82d4fcbac297775997a7e0abe3e9ff9"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"cf5190bda49a26788552ee732aefcf3d11eb261a","unresolved":false,"context_lines":[{"line_number":624,"context_line":"        self.explist \u003d deque()"},{"line_number":625,"context_line":"        self.index \u003d dict()"},{"line_number":626,"context_line":""},{"line_number":627,"context_line":"    # XXX Examine limiting the expiration to no more than 2 objects at a time."},{"line_number":628,"context_line":"    def expire(self, now):"},{"line_number":629,"context_line":"        while len(self.explist) !\u003d 0:"},{"line_number":630,"context_line":"            if self.explist[0][1] + self.reclaim_time \u003e now:"}],"source_content_type":"text/x-python","patch_set":25,"id":"1f9dbf25_3f417a3a","line":627,"updated":"2018-03-03 01:37:58.000000000","message":"That seems like a maybe not-crazy thing to do -- then if some malfunctioning client sends a few thousand bad requests in space of 30s but the cluster\u0027s pretty quiet otherwise, there won\u0027t be a massive penalty on the next request when it comes along 5 minutes later.","commit_id":"79fec861768bb219815e6cab80d3b1320e393b1e"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"cf5190bda49a26788552ee732aefcf3d11eb261a","unresolved":false,"context_lines":[{"line_number":630,"context_line":"            if self.explist[0][1] + self.reclaim_time \u003e now:"},{"line_number":631,"context_line":"                return"},{"line_number":632,"context_line":"            p \u003d self.explist.popleft()"},{"line_number":633,"context_line":"            self.index.pop(p[0])"},{"line_number":634,"context_line":"            p[2].close(force2\u003dTrue)"},{"line_number":635,"context_line":""},{"line_number":636,"context_line":"    def save(self, df):"}],"source_content_type":"text/x-python","patch_set":25,"id":"1f9dbf25_1f46fe50","line":633,"updated":"2018-03-03 01:37:58.000000000","message":"If we\u0027re using pop() to try to tolerate missing keys, we need a default arg. If not (they\u0027re UUIDs, after all; it\u0027s not like we expect a collision), my impression was that it\u0027s generally (slightly) cheaper to use del.","commit_id":"79fec861768bb219815e6cab80d3b1320e393b1e"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"cf5190bda49a26788552ee732aefcf3d11eb261a","unresolved":false,"context_lines":[{"line_number":759,"context_line":""},{"line_number":760,"context_line":"        :param timestamp: the object timestamp, an instance of"},{"line_number":761,"context_line":"                          :class:`~swift.common.utils.Timestamp`"},{"line_number":762,"context_line":"        :param ext: an optional string representing a file extension to be"},{"line_number":763,"context_line":"                    appended to the returned file name"},{"line_number":764,"context_line":"        :param ctype_timestamp: an optional content-type timestamp, an instance"},{"line_number":765,"context_line":"                                of :class:`~swift.common.utils.Timestamp`"}],"source_content_type":"text/x-python","patch_set":25,"id":"1f9dbf25_ff6312db","line":762,"range":{"start_line":762,"start_character":23,"end_line":762,"end_character":31},"updated":"2018-03-03 01:37:58.000000000","message":"No longer optional. (Why?)","commit_id":"79fec861768bb219815e6cab80d3b1320e393b1e"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"cf5190bda49a26788552ee732aefcf3d11eb261a","unresolved":false,"context_lines":[{"line_number":1638,"context_line":""},{"line_number":1639,"context_line":"        return self._upload_size"},{"line_number":1640,"context_line":""},{"line_number":1641,"context_line":"    def close(self, force2\u003dNone):"},{"line_number":1642,"context_line":"        if self._suppress_file_closing and not force2:"},{"line_number":1643,"context_line":"            return"},{"line_number":1644,"context_line":"        try:"}],"source_content_type":"text/x-python","patch_set":25,"id":"1f9dbf25_ffb8f22a","line":1641,"range":{"start_line":1641,"start_character":20,"end_line":1641,"end_character":26},"updated":"2018-03-03 01:37:58.000000000","message":"Why 2?","commit_id":"79fec861768bb219815e6cab80d3b1320e393b1e"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"cf5190bda49a26788552ee732aefcf3d11eb261a","unresolved":false,"context_lines":[{"line_number":1651,"context_line":"            # self.put_succeeded becomes True after renamer() succeeds in"},{"line_number":1652,"context_line":"            # _finalize_put(), at which point there is no more temp file."},{"line_number":1653,"context_line":"            try:"},{"line_number":1654,"context_line":"                if self._tmppath:"},{"line_number":1655,"context_line":"                    # when mkstemp() was used"},{"line_number":1656,"context_line":"                    os.unlink(self._tmppath)"},{"line_number":1657,"context_line":"            except OSError:"}],"source_content_type":"text/x-python","patch_set":25,"id":"1f9dbf25_1fb4be34","line":1654,"updated":"2018-03-03 01:37:58.000000000","message":"Could we move this up so it\u0027s\n\n if not self.put_succeeded and self._tmppath:\n     try:\n         ...\n\nOr maybe swap the order... whichever reads better to you.","commit_id":"79fec861768bb219815e6cab80d3b1320e393b1e"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"cf5190bda49a26788552ee732aefcf3d11eb261a","unresolved":false,"context_lines":[{"line_number":1755,"context_line":"        # XXX This just saves the calculated Etag. Save in writer._etag?"},{"line_number":1756,"context_line":"        # XXX no, wait. There\u0027s also content-length in there."},{"line_number":1757,"context_line":"        write_metadata(self._fd, metadata)"},{"line_number":1758,"context_line":"        # fsync(self._fd)"},{"line_number":1759,"context_line":"        drop_buffer_cache(self._fd, 0, self._upload_size)"},{"line_number":1760,"context_line":"        self._staged_metadata \u003d metadata"},{"line_number":1761,"context_line":""}],"source_content_type":"text/x-python","patch_set":25,"id":"1f9dbf25_7f7e226c","line":1758,"updated":"2018-03-03 01:37:58.000000000","message":"Why commented out?","commit_id":"79fec861768bb219815e6cab80d3b1320e393b1e"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"cf5190bda49a26788552ee732aefcf3d11eb261a","unresolved":false,"context_lines":[{"line_number":1757,"context_line":"        write_metadata(self._fd, metadata)"},{"line_number":1758,"context_line":"        # fsync(self._fd)"},{"line_number":1759,"context_line":"        drop_buffer_cache(self._fd, 0, self._upload_size)"},{"line_number":1760,"context_line":"        self._staged_metadata \u003d metadata"},{"line_number":1761,"context_line":""},{"line_number":1762,"context_line":"        self._suppress_file_closing \u003d True"},{"line_number":1763,"context_line":""}],"source_content_type":"text/x-python","patch_set":25,"id":"1f9dbf25_9ff68ef9","line":1760,"updated":"2018-03-03 01:37:58.000000000","message":"So we both write the metadata to disk and hold it in our head -- as a developer, which should I consider authoritative? I guess, the copy still in my head? I don\u0027t actually *want* to read from disk again... and if I crash, and lose what\u0027s in my head, I\u0027ve also lost the mapping of UUIDs to tmp files, so I couldn\u0027t find it even if I wanted to.","commit_id":"79fec861768bb219815e6cab80d3b1320e393b1e"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"cf5190bda49a26788552ee732aefcf3d11eb261a","unresolved":false,"context_lines":[{"line_number":1777,"context_line":"            raise DiskFileUseError()"},{"line_number":1778,"context_line":""},{"line_number":1779,"context_line":"        timestamp \u003d Timestamp(metadata[\u0027X-Timestamp\u0027])"},{"line_number":1780,"context_line":"        ctype_timestamp \u003d metadata.get(\u0027Content-Type-Timestamp\u0027)"},{"line_number":1781,"context_line":"        if ctype_timestamp:"},{"line_number":1782,"context_line":"            ctype_timestamp \u003d Timestamp(ctype_timestamp)"},{"line_number":1783,"context_line":"        # Checks in POST() verify that we intend to use an EC storage, so"}],"source_content_type":"text/x-python","patch_set":25,"id":"1f9dbf25_bf602aa7","line":1780,"range":{"start_line":1780,"start_character":40,"end_line":1780,"end_character":62},"updated":"2018-03-03 01:37:58.000000000","message":"Huh. We send this as a distinct value on PUT? And judging by the below, I guess we only do so *sometimes*...","commit_id":"79fec861768bb219815e6cab80d3b1320e393b1e"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"cf5190bda49a26788552ee732aefcf3d11eb261a","unresolved":false,"context_lines":[{"line_number":1796,"context_line":"            renamer(self._tmppath, target_path)"},{"line_number":1797,"context_line":"        else:"},{"line_number":1798,"context_line":"            link_fd_to_path(self._fd, target_path,"},{"line_number":1799,"context_line":"                            self._diskfile._dirs_created)"},{"line_number":1800,"context_line":""},{"line_number":1801,"context_line":"        self._put_succeeded \u003d True"},{"line_number":1802,"context_line":""}],"source_content_type":"text/x-python","patch_set":25,"id":"1f9dbf25_3f741a68","line":1799,"range":{"start_line":1799,"start_character":28,"end_line":1799,"end_character":56},"updated":"2018-03-03 01:37:58.000000000","message":"?","commit_id":"79fec861768bb219815e6cab80d3b1320e393b1e"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"cf5190bda49a26788552ee732aefcf3d11eb261a","unresolved":false,"context_lines":[{"line_number":1806,"context_line":"        # someone only needs to call self.commit()."},{"line_number":1807,"context_line":"        return metadata"},{"line_number":1808,"context_line":""},{"line_number":1809,"context_line":"    def detach(self):"},{"line_number":1810,"context_line":"        self._diskfile \u003d None"},{"line_number":1811,"context_line":""},{"line_number":1812,"context_line":"    def reparent(self, diskfile):"}],"source_content_type":"text/x-python","patch_set":25,"id":"1f9dbf25_ff6d5298","line":1809,"updated":"2018-03-03 01:37:58.000000000","message":"I suppose this is something of a converse to reparent, but is it actually used anywhere?","commit_id":"79fec861768bb219815e6cab80d3b1320e393b1e"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"cf5190bda49a26788552ee732aefcf3d11eb261a","unresolved":false,"context_lines":[{"line_number":1818,"context_line":"        \"\"\""},{"line_number":1819,"context_line":"        write_metadata(self._fd, metadata)"},{"line_number":1820,"context_line":"        fsync(self._fd)"},{"line_number":1821,"context_line":"        # drop_buffer_cache(self._fd, 0, self._upload_size)"},{"line_number":1822,"context_line":""},{"line_number":1823,"context_line":"    def commit(self, timestamp):"},{"line_number":1824,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":25,"id":"1f9dbf25_bf058ac4","line":1821,"updated":"2018-03-03 01:37:58.000000000","message":"... so here we do it the *other way*? Why?","commit_id":"79fec861768bb219815e6cab80d3b1320e393b1e"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"cf5190bda49a26788552ee732aefcf3d11eb261a","unresolved":false,"context_lines":[{"line_number":2812,"context_line":"                              bytes_per_sync\u003dself._bytes_per_sync,"},{"line_number":2813,"context_line":"                              diskfile\u003dself,"},{"line_number":2814,"context_line":"                              next_part_power\u003dself.next_part_power)"},{"line_number":2815,"context_line":"        return dfw"},{"line_number":2816,"context_line":""},{"line_number":2817,"context_line":"    def write_metadata(self, metadata):"},{"line_number":2818,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":25,"id":"1f9dbf25_ff0232b4","line":2815,"updated":"2018-03-03 01:37:58.000000000","message":"It seems like we could do\n\n try:\n     yield dfw\n finally:\n     self.close()\n\nand maintain our context manager interface here... am I missing something?","commit_id":"79fec861768bb219815e6cab80d3b1320e393b1e"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"3a4e11c77137716276fc6c095aa1ad508299c7a0","unresolved":false,"context_lines":[{"line_number":2709,"context_line":"            fd, tmppath \u003d mkstemp(dir\u003dself._tmpdir)"},{"line_number":2710,"context_line":"        return fd, tmppath"},{"line_number":2711,"context_line":""},{"line_number":2712,"context_line":"    @contextmanager"},{"line_number":2713,"context_line":"    def create(self, size\u003dNone):"},{"line_number":2714,"context_line":"        \"\"\""},{"line_number":2715,"context_line":"        Context manager to create a file. We create a temporary file first, and"}],"source_content_type":"text/x-python","patch_set":32,"id":"3f79a3b5_e3209c4c","side":"PARENT","line":2712,"range":{"start_line":2712,"start_character":4,"end_line":2712,"end_character":19},"updated":"2018-08-04 00:48:58.000000000","message":"This seems like a sizable diskfile API change... how much do we care about backwards compat for diskfile API? Could we preserve the existing API with something like\n\n return closing(dfw)\n\n?","commit_id":"791009a6ff9c85556ccd1a7e7fee312f1d32abd7"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"3a4e11c77137716276fc6c095aa1ad508299c7a0","unresolved":false,"context_lines":[{"line_number":2762,"context_line":"                        os.unlink(tmppath)"},{"line_number":2763,"context_line":"                except OSError:"},{"line_number":2764,"context_line":"                    self._logger.exception(\u0027Error removing tempfile: %s\u0027 %"},{"line_number":2765,"context_line":"                                           tmppath)"},{"line_number":2766,"context_line":""},{"line_number":2767,"context_line":"    def write_metadata(self, metadata):"},{"line_number":2768,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":32,"id":"3f79a3b5_c3c5605d","side":"PARENT","line":2765,"updated":"2018-08-04 00:48:58.000000000","message":"So we lose the logging? I feel like https://bugs.launchpad.net/swift/+bug/1647804 (for example) would\u0027ve been harder to diagnose without it...\n\n*shrug*\n\nMaybe it doesn\u0027t matter so much now that https://review.openstack.org/#/c/518783/ landed...","commit_id":"791009a6ff9c85556ccd1a7e7fee312f1d32abd7"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"3a4e11c77137716276fc6c095aa1ad508299c7a0","unresolved":false,"context_lines":[{"line_number":632,"context_line":""},{"line_number":633,"context_line":"class DiskFileCache(object):"},{"line_number":634,"context_line":"    \"\"\""},{"line_number":635,"context_line":"    A collection of open DiskFile(-Writer) instances lingering between"},{"line_number":636,"context_line":"    requests."},{"line_number":637,"context_line":"    \"\"\""},{"line_number":638,"context_line":"    def __init__(self, reclaim_time\u003d100.0):"}],"source_content_type":"text/x-python","patch_set":32,"id":"3f79a3b5_63048cc7","line":635,"range":{"start_line":635,"start_character":25,"end_line":635,"end_character":52},"updated":"2018-08-04 00:48:58.000000000","message":"Wait, so are these DiskFiles or DiskFileWriters? It seems like all the _suppress_file_closing manipulations would only have an effect on writers, yeah?\n\nLooks like we only insert into the cache in the one spot in obj/server.py:\n\n self.writer_cache.save(writer)\n\nIs that ever *not* a DiskFileWriter?","commit_id":"83256db01ae98b902df7691cad155bbdc6024080"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"ade2f0775d29c70a287a89a39524a57cdc5bb005","unresolved":false,"context_lines":[{"line_number":634,"context_line":"    \"\"\""},{"line_number":635,"context_line":"    A collection of open DiskFile(-Writer) instances lingering between"},{"line_number":636,"context_line":"    requests."},{"line_number":637,"context_line":"    \"\"\""},{"line_number":638,"context_line":"    def __init__(self, reclaim_time\u003d100.0):"},{"line_number":639,"context_line":"        self.reclaim_time \u003d reclaim_time"},{"line_number":640,"context_line":"        self.explist \u003d deque()"}],"source_content_type":"text/x-python","patch_set":32,"id":"3f79a3b5_fbfb4712","line":637,"updated":"2018-09-11 18:11:40.000000000","message":"It wasn\u0027t super obvious to me why we needed to do this - but because we use linkat we have to leave the fd open between the individual WSGI requests.\n\nI mean I guess we *could* linkat to a \u0027staged-data-temp-id.tmp\u0027 before closing in STAGE then open /hashpath/staged-data-\u003ctemp-id\u003e.tmp in UPDATE/COMMIT ... maybe that would let us move clean up to rehash/ssync?","commit_id":"83256db01ae98b902df7691cad155bbdc6024080"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"3a4e11c77137716276fc6c095aa1ad508299c7a0","unresolved":false,"context_lines":[{"line_number":635,"context_line":"    A collection of open DiskFile(-Writer) instances lingering between"},{"line_number":636,"context_line":"    requests."},{"line_number":637,"context_line":"    \"\"\""},{"line_number":638,"context_line":"    def __init__(self, reclaim_time\u003d100.0):"},{"line_number":639,"context_line":"        self.reclaim_time \u003d reclaim_time"},{"line_number":640,"context_line":"        self.explist \u003d deque()"},{"line_number":641,"context_line":"        self.index \u003d dict()"}],"source_content_type":"text/x-python","patch_set":32,"id":"3f79a3b5_230e94e5","line":638,"range":{"start_line":638,"start_character":36,"end_line":638,"end_character":41},"updated":"2018-08-04 00:48:58.000000000","message":"Why 100s? I guess, 60s for client read timeout, plus some \"reasonable\" slop?","commit_id":"83256db01ae98b902df7691cad155bbdc6024080"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"3a4e11c77137716276fc6c095aa1ad508299c7a0","unresolved":false,"context_lines":[{"line_number":644,"context_line":"        while len(self.explist) !\u003d 0:"},{"line_number":645,"context_line":"            if self.explist[0][1] + self.reclaim_time \u003e now:"},{"line_number":646,"context_line":"                return"},{"line_number":647,"context_line":"            p \u003d self.explist.popleft()"},{"line_number":648,"context_line":"            self.index.pop(p[0])"},{"line_number":649,"context_line":"            p[2]._suppress_file_closing \u003d False"},{"line_number":650,"context_line":"            p[2].close()"}],"source_content_type":"text/x-python","patch_set":32,"id":"3f79a3b5_8388c871","line":647,"range":{"start_line":647,"start_character":12,"end_line":647,"end_character":13},"updated":"2018-08-04 00:48:58.000000000","message":"nit: Might be worth unpacking here for readability.","commit_id":"83256db01ae98b902df7691cad155bbdc6024080"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"ade2f0775d29c70a287a89a39524a57cdc5bb005","unresolved":false,"context_lines":[{"line_number":649,"context_line":"            p[2]._suppress_file_closing \u003d False"},{"line_number":650,"context_line":"            p[2].close()"},{"line_number":651,"context_line":""},{"line_number":652,"context_line":"    def save(self, df):"},{"line_number":653,"context_line":"        # Note that you can have 2 clients PUT-ting into one object."},{"line_number":654,"context_line":"        # They will not conflict until the later one wins. But this"},{"line_number":655,"context_line":"        # result is only known when one or both commit with a POST."}],"source_content_type":"text/x-python","patch_set":32,"id":"3f79a3b5_fb8b4714","line":652,"updated":"2018-09-11 18:11:40.000000000","message":"I don\u0027t think this is part of the DiskFile API, I think the object-server should have a thing that can stash (writer, metadata_so_far) tuples?","commit_id":"83256db01ae98b902df7691cad155bbdc6024080"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"3a4e11c77137716276fc6c095aa1ad508299c7a0","unresolved":false,"context_lines":[{"line_number":668,"context_line":"            return None"},{"line_number":669,"context_line":"        return p[2]"},{"line_number":670,"context_line":""},{"line_number":671,"context_line":"    def popx(self, oid):"},{"line_number":672,"context_line":"        try:"},{"line_number":673,"context_line":"            p \u003d self.index.pop(oid)"},{"line_number":674,"context_line":"        except KeyError:"}],"source_content_type":"text/x-python","patch_set":32,"id":"3f79a3b5_296f55c6","line":671,"range":{"start_line":671,"start_character":8,"end_line":671,"end_character":12},"updated":"2018-08-04 00:48:58.000000000","message":"nit: Why popx? Why not just pop?","commit_id":"83256db01ae98b902df7691cad155bbdc6024080"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"2f7cd3091471486155fa7aecc3fc6d56988f5933","unresolved":false,"context_lines":[{"line_number":776,"context_line":""},{"line_number":777,"context_line":"        :param timestamp: the object timestamp, an instance of"},{"line_number":778,"context_line":"                          :class:`~swift.common.utils.Timestamp`"},{"line_number":779,"context_line":"        :param ext: an optional string representing a file extension to be"},{"line_number":780,"context_line":"                    appended to the returned file name"},{"line_number":781,"context_line":"        :param ctype_timestamp: an optional content-type timestamp, an instance"},{"line_number":782,"context_line":"                                of :class:`~swift.common.utils.Timestamp`"}],"source_content_type":"text/x-python","patch_set":32,"id":"5f7c97a3_08959394","line":779,"range":{"start_line":779,"start_character":20,"end_line":779,"end_character":31},"updated":"2018-08-02 07:32:10.000000000","message":"This doesn\u0027t seem optional anymore.","commit_id":"83256db01ae98b902df7691cad155bbdc6024080"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"5d5354c06967a495b5bff02ba2c161bcab1f1873","unresolved":false,"context_lines":[{"line_number":776,"context_line":""},{"line_number":777,"context_line":"        :param timestamp: the object timestamp, an instance of"},{"line_number":778,"context_line":"                          :class:`~swift.common.utils.Timestamp`"},{"line_number":779,"context_line":"        :param ext: an optional string representing a file extension to be"},{"line_number":780,"context_line":"                    appended to the returned file name"},{"line_number":781,"context_line":"        :param ctype_timestamp: an optional content-type timestamp, an instance"},{"line_number":782,"context_line":"                                of :class:`~swift.common.utils.Timestamp`"}],"source_content_type":"text/x-python","patch_set":32,"id":"5f7c97a3_236208dc","line":779,"range":{"start_line":779,"start_character":20,"end_line":779,"end_character":31},"in_reply_to":"5f7c97a3_08959394","updated":"2018-08-02 23:19:01.000000000","message":"Done","commit_id":"83256db01ae98b902df7691cad155bbdc6024080"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"3a4e11c77137716276fc6c095aa1ad508299c7a0","unresolved":false,"context_lines":[{"line_number":776,"context_line":""},{"line_number":777,"context_line":"        :param timestamp: the object timestamp, an instance of"},{"line_number":778,"context_line":"                          :class:`~swift.common.utils.Timestamp`"},{"line_number":779,"context_line":"        :param ext: an optional string representing a file extension to be"},{"line_number":780,"context_line":"                    appended to the returned file name"},{"line_number":781,"context_line":"        :param ctype_timestamp: an optional content-type timestamp, an instance"},{"line_number":782,"context_line":"                                of :class:`~swift.common.utils.Timestamp`"}],"source_content_type":"text/x-python","patch_set":32,"id":"3f79a3b5_03d91882","line":779,"range":{"start_line":779,"start_character":20,"end_line":779,"end_character":31},"in_reply_to":"5f7c97a3_236208dc","updated":"2018-08-04 00:48:58.000000000","message":"... *why* isn\u0027t it optional anymore?","commit_id":"83256db01ae98b902df7691cad155bbdc6024080"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"ade2f0775d29c70a287a89a39524a57cdc5bb005","unresolved":false,"context_lines":[{"line_number":1723,"context_line":""},{"line_number":1724,"context_line":"    def close(self):"},{"line_number":1725,"context_line":"        if self._suppress_file_closing:"},{"line_number":1726,"context_line":"            return"},{"line_number":1727,"context_line":"        try:"},{"line_number":1728,"context_line":"            os.close(self._fd)"},{"line_number":1729,"context_line":"        except OSError:"}],"source_content_type":"text/x-python","patch_set":32,"id":"3f79a3b5_78697d24","line":1726,"updated":"2018-09-11 18:11:40.000000000","message":"this smells, this looks like this method would be named \"maybe_close\"","commit_id":"83256db01ae98b902df7691cad155bbdc6024080"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"3a4e11c77137716276fc6c095aa1ad508299c7a0","unresolved":false,"context_lines":[{"line_number":1839,"context_line":"        # fsync(self._fd)"},{"line_number":1840,"context_line":"        drop_buffer_cache(self._fd, 0, self._upload_size)"},{"line_number":1841,"context_line":"        self._staged_metadata \u003d metadata"},{"line_number":1842,"context_line":"        self._suppress_file_closing \u003d True"},{"line_number":1843,"context_line":""},{"line_number":1844,"context_line":"    def orphan(self):"},{"line_number":1845,"context_line":"        self._diskfile \u003d None"}],"source_content_type":"text/x-python","patch_set":32,"id":"3f79a3b5_43d07051","line":1842,"range":{"start_line":1842,"start_character":8,"end_line":1842,"end_character":42},"updated":"2018-08-04 00:48:58.000000000","message":"I was kind of expecting this to happen in DiskFileCache.save... since DiskFileCache does a bunch of managing of _suppress_file_closing anyway... but I could also see it being nice to have DiskFileWriter managing its own _* attrs...\n\nMaybe we could shore up the DiskFileWriter stage/commit APIs to make it so DiskFileCache doesn\u0027t need to think about _suppress_file_closing? Might need to add one extra method like\n\ndef abort(self):\n    self._suppress_file_closing \u003d True\n    self.close()\n\nthat DiskFileCache.expire could use...","commit_id":"83256db01ae98b902df7691cad155bbdc6024080"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"ade2f0775d29c70a287a89a39524a57cdc5bb005","unresolved":false,"context_lines":[{"line_number":1839,"context_line":"        # fsync(self._fd)"},{"line_number":1840,"context_line":"        drop_buffer_cache(self._fd, 0, self._upload_size)"},{"line_number":1841,"context_line":"        self._staged_metadata \u003d metadata"},{"line_number":1842,"context_line":"        self._suppress_file_closing \u003d True"},{"line_number":1843,"context_line":""},{"line_number":1844,"context_line":"    def orphan(self):"},{"line_number":1845,"context_line":"        self._diskfile \u003d None"}],"source_content_type":"text/x-python","patch_set":32,"id":"3f79a3b5_18640949","line":1842,"range":{"start_line":1842,"start_character":8,"end_line":1842,"end_character":42},"in_reply_to":"3f79a3b5_43d07051","updated":"2018-09-11 18:11:40.000000000","message":"+1","commit_id":"83256db01ae98b902df7691cad155bbdc6024080"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"3a4e11c77137716276fc6c095aa1ad508299c7a0","unresolved":false,"context_lines":[{"line_number":1842,"context_line":"        self._suppress_file_closing \u003d True"},{"line_number":1843,"context_line":""},{"line_number":1844,"context_line":"    def orphan(self):"},{"line_number":1845,"context_line":"        self._diskfile \u003d None"},{"line_number":1846,"context_line":""},{"line_number":1847,"context_line":"    def reparent(self, diskfile):"},{"line_number":1848,"context_line":"        self._diskfile \u003d diskfile"}],"source_content_type":"text/x-python","patch_set":32,"id":"3f79a3b5_ee23a3bd","line":1845,"updated":"2018-08-04 00:48:58.000000000","message":"I wonder if it would be worth having this be a side-effect of stage... and maybe have reparent() return self._staged_metadata so obj/server.py doesn\u0027t have to go grubbing around for writer._staged_metadata.","commit_id":"83256db01ae98b902df7691cad155bbdc6024080"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"3a4e11c77137716276fc6c095aa1ad508299c7a0","unresolved":false,"context_lines":[{"line_number":1847,"context_line":"    def reparent(self, diskfile):"},{"line_number":1848,"context_line":"        self._diskfile \u003d diskfile"},{"line_number":1849,"context_line":""},{"line_number":1850,"context_line":"    def write_metadata(self, metadata):"},{"line_number":1851,"context_line":"        \"\"\""},{"line_number":1852,"context_line":"        Write metadata like put would do it."},{"line_number":1853,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":32,"id":"3f79a3b5_694c6d76","line":1850,"updated":"2018-08-04 00:48:58.000000000","message":"Assuming DiskFileCache only ever holds DiskFileWriters (which I\u0027m *prety sure* is the case), do we actually need this addition?","commit_id":"83256db01ae98b902df7691cad155bbdc6024080"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"3a4e11c77137716276fc6c095aa1ad508299c7a0","unresolved":false,"context_lines":[{"line_number":1849,"context_line":""},{"line_number":1850,"context_line":"    def write_metadata(self, metadata):"},{"line_number":1851,"context_line":"        \"\"\""},{"line_number":1852,"context_line":"        Write metadata like put would do it."},{"line_number":1853,"context_line":"        \"\"\""},{"line_number":1854,"context_line":"        self._staged_metadata \u003d metadata"},{"line_number":1855,"context_line":"        write_metadata(self._fd, metadata)"}],"source_content_type":"text/x-python","patch_set":32,"id":"3f79a3b5_ee0c0344","line":1852,"updated":"2018-08-04 00:48:58.000000000","message":"Given that put() just raises NotImplementedError, this seems like strange documentation...","commit_id":"83256db01ae98b902df7691cad155bbdc6024080"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"3a4e11c77137716276fc6c095aa1ad508299c7a0","unresolved":false,"context_lines":[{"line_number":1854,"context_line":"        self._staged_metadata \u003d metadata"},{"line_number":1855,"context_line":"        write_metadata(self._fd, metadata)"},{"line_number":1856,"context_line":"        fsync(self._fd)"},{"line_number":1857,"context_line":"        # drop_buffer_cache(self._fd, 0, self._upload_size)"},{"line_number":1858,"context_line":""},{"line_number":1859,"context_line":"    def commit(self, timestamp):"},{"line_number":1860,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":32,"id":"3f79a3b5_ae2daba9","line":1857,"updated":"2018-08-04 00:48:58.000000000","message":"??","commit_id":"83256db01ae98b902df7691cad155bbdc6024080"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"3a4e11c77137716276fc6c095aa1ad508299c7a0","unresolved":false,"context_lines":[{"line_number":2853,"context_line":"                if err.errno in (errno.ENOSPC, errno.EDQUOT):"},{"line_number":2854,"context_line":"                    raise DiskFileNoSpace()"},{"line_number":2855,"context_line":"                raise"},{"line_number":2856,"context_line":"        dfw \u003d self.writer_cls(self._name, self._datadir, fd, tmppath,"},{"line_number":2857,"context_line":"                              bytes_per_sync\u003dself._bytes_per_sync,"},{"line_number":2858,"context_line":"                              diskfile\u003dself,"},{"line_number":2859,"context_line":"                              next_part_power\u003dself.next_part_power)"}],"source_content_type":"text/x-python","patch_set":32,"id":"3f79a3b5_23aa1485","line":2856,"range":{"start_line":2856,"start_character":14,"end_line":2856,"end_character":29},"updated":"2018-08-04 00:48:58.000000000","message":"This should basically never raise an error, yeah? Otherwise I\u0027d worry about us leaking the temp file...","commit_id":"83256db01ae98b902df7691cad155bbdc6024080"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"3a4e11c77137716276fc6c095aa1ad508299c7a0","unresolved":false,"context_lines":[{"line_number":3134,"context_line":"        if frag_index is None:"},{"line_number":3135,"context_line":"            frag_index \u003d self._diskfile._frag_index"},{"line_number":3136,"context_line":"            if frag_index is None:"},{"line_number":3137,"context_line":"                raise DiskFileUseError()"},{"line_number":3138,"context_line":"        data_file_path \u003d join("},{"line_number":3139,"context_line":"            self._datadir, self.manager.make_on_disk_filename("},{"line_number":3140,"context_line":"                timestamp, \u0027.data\u0027, frag_index\u003dfrag_index))"}],"source_content_type":"text/x-python","patch_set":32,"id":"3f79a3b5_4339503f","line":3137,"updated":"2018-08-04 00:48:58.000000000","message":"Off-topic: I find it mildly infuriating that we already have the frag index written down on disk (repeatedly, if there\u0027s more than one fragment in the fragment archive) but we\u0027re too stupid to look for it...","commit_id":"83256db01ae98b902df7691cad155bbdc6024080"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"3a4e11c77137716276fc6c095aa1ad508299c7a0","unresolved":false,"context_lines":[{"line_number":3533,"context_line":"            results[\u0027chosen_frag_set\u0027] \u003d chosen_frag_set"},{"line_number":3534,"context_line":"            # Do not filter meta if we POST to durable, and it\u0027s not there yet."},{"line_number":3535,"context_line":"            # Killing this meta will mysteriously misplace it for EC objects."},{"line_number":3536,"context_line":"            if durable_frag_set is not None and \\"},{"line_number":3537,"context_line":"                    chosen_frag_set !\u003d durable_frag_set:"},{"line_number":3538,"context_line":"                # hide meta files older than data file but newer than durable"},{"line_number":3539,"context_line":"                # file so they don\u0027t get marked as obsolete (we already threw"}],"source_content_type":"text/x-python","patch_set":32,"id":"3f79a3b5_a3062420","line":3536,"updated":"2018-08-04 00:48:58.000000000","message":"I need to look more at what\u0027s going on here and why this is necessary...\n\nHow can we POST to a durable that isn\u0027t there yet?","commit_id":"83256db01ae98b902df7691cad155bbdc6024080"}],"swift/obj/mem_diskfile.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"cf5190bda49a26788552ee732aefcf3d11eb261a","unresolved":false,"context_lines":[{"line_number":422,"context_line":"        :raises DiskFileNoSpace: if a size is specified and allocation fails"},{"line_number":423,"context_line":"        \"\"\""},{"line_number":424,"context_line":"        fp \u003d moves.cStringIO()"},{"line_number":425,"context_line":"        return DiskFileWriter(self._filesystem, self._name, fp)"},{"line_number":426,"context_line":""},{"line_number":427,"context_line":"    def write_metadata(self, metadata):"},{"line_number":428,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":25,"id":"1f9dbf25_421d940d","line":425,"updated":"2018-03-03 01:37:58.000000000","message":"This seems like a fairly sizable API break -- do we have any feeling for how (or whether) a third-party diskfile implementation might be able to be compatible with both swift 2.17.0 (say) and this?","commit_id":"79fec861768bb219815e6cab80d3b1320e393b1e"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"2916044a01da4770a045ad09d15287c44263d3c1","unresolved":false,"context_lines":[{"line_number":422,"context_line":"        :raises DiskFileNoSpace: if a size is specified and allocation fails"},{"line_number":423,"context_line":"        \"\"\""},{"line_number":424,"context_line":"        fp \u003d moves.cStringIO()"},{"line_number":425,"context_line":"        return DiskFileWriter(self._filesystem, self._name, fp)"},{"line_number":426,"context_line":""},{"line_number":427,"context_line":"    def write_metadata(self, metadata):"},{"line_number":428,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":25,"id":"df7087c5_d241463b","line":425,"in_reply_to":"1f9dbf25_421d940d","updated":"2018-03-12 04:07:16.000000000","message":"The people like Swift-on-File only do major versions. So it\u0027s not going to be a big deal of them, I hope. That said, maybe you can ask who supports Kinetic, if anyone.","commit_id":"79fec861768bb219815e6cab80d3b1320e393b1e"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"ade2f0775d29c70a287a89a39524a57cdc5bb005","unresolved":false,"context_lines":[{"line_number":116,"context_line":"        return self._upload_size"},{"line_number":117,"context_line":""},{"line_number":118,"context_line":"    def stage(self, metadata):"},{"line_number":119,"context_line":"        self._staged_metadata \u003d metadata"},{"line_number":120,"context_line":""},{"line_number":121,"context_line":"    def orphan(self):"},{"line_number":122,"context_line":"        self._diskfile \u003d None"}],"source_content_type":"text/x-python","patch_set":32,"id":"3f79a3b5_b8a715bd","line":119,"updated":"2018-09-11 18:11:40.000000000","message":"oh, we already added a STAGE and it\u0027s for metadata instead of data...","commit_id":"83256db01ae98b902df7691cad155bbdc6024080"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"ade2f0775d29c70a287a89a39524a57cdc5bb005","unresolved":false,"context_lines":[{"line_number":122,"context_line":"        self._diskfile \u003d None"},{"line_number":123,"context_line":""},{"line_number":124,"context_line":"    def reparent(self, diskfile):"},{"line_number":125,"context_line":"        self._diskfile \u003d diskfile"},{"line_number":126,"context_line":""},{"line_number":127,"context_line":"    def put(self, metadata):"},{"line_number":128,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":32,"id":"3f79a3b5_d8ac51a1","line":125,"updated":"2018-09-11 18:11:40.000000000","message":"I don\u0027t like either of these methods...","commit_id":"83256db01ae98b902df7691cad155bbdc6024080"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"ade2f0775d29c70a287a89a39524a57cdc5bb005","unresolved":false,"context_lines":[{"line_number":429,"context_line":"        self._fp \u003d None"},{"line_number":430,"context_line":"        return dr"},{"line_number":431,"context_line":""},{"line_number":432,"context_line":"    def create(self, size\u003dNone):"},{"line_number":433,"context_line":"        \"\"\""},{"line_number":434,"context_line":"        Create a file. We create a temporary file first, and then return a"},{"line_number":435,"context_line":"        DiskFileWriter object to encapsulate the state."}],"source_content_type":"text/x-python","patch_set":32,"id":"3f79a3b5_98c9f97a","line":432,"updated":"2018-09-11 18:11:40.000000000","message":"Can we just leave the API alone?  Leave this a context manager.  And make a new method (e.g. \"writer\" to play of the existing \"reader\" method) that will just create DiskFileWriter that doesn\u0027t close for use in STAGE?","commit_id":"83256db01ae98b902df7691cad155bbdc6024080"}],"swift/obj/server.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"8898ed7e81094158d38b5056b6697345f6712ea5","unresolved":false,"context_lines":[{"line_number":510,"context_line":"            get_name_and_placement(request, 5, 5, True)"},{"line_number":511,"context_line":"        to_durable \u003d False"},{"line_number":512,"context_line":"        if request.headers.get(\u0027X-Backend-Post-To-Durable\u0027) is not None:"},{"line_number":513,"context_line":"            to_durable \u003d True"},{"line_number":514,"context_line":"        req_timestamp \u003d valid_timestamp(request)"},{"line_number":515,"context_line":"        new_delete_at \u003d int(request.headers.get(\u0027X-Delete-At\u0027) or 0)"},{"line_number":516,"context_line":"        if new_delete_at and new_delete_at \u003c time.time():"}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_5fe5c6db","line":513,"updated":"2017-05-02 17:16:37.000000000","message":"Maybe just\n\n to_durable \u003d (request.headers.get(\u0027X-Backend-Post-To-Durable\u0027) is not None)\n\n? Eh... doesn\u0027t really matter...","commit_id":"78f47b42db89987c611761d87ef3ff2e2c28b491"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"9c98977a75d350825c281216b4e93da4986b1b00","unresolved":false,"context_lines":[{"line_number":510,"context_line":"            get_name_and_placement(request, 5, 5, True)"},{"line_number":511,"context_line":"        to_durable \u003d False"},{"line_number":512,"context_line":"        if request.headers.get(\u0027X-Backend-Post-To-Durable\u0027) is not None:"},{"line_number":513,"context_line":"            to_durable \u003d True"},{"line_number":514,"context_line":"        req_timestamp \u003d valid_timestamp(request)"},{"line_number":515,"context_line":"        new_delete_at \u003d int(request.headers.get(\u0027X-Delete-At\u0027) or 0)"},{"line_number":516,"context_line":"        if new_delete_at and new_delete_at \u003c time.time():"}],"source_content_type":"text/x-python","patch_set":8,"id":"3f044301_194abc26","line":513,"in_reply_to":"5ff73747_5fe5c6db","updated":"2017-05-04 03:29:09.000000000","message":"I mostly did this because a oneliner didn\u0027t fit into 79 characters. But once I start splitting it up, all the beauty dissipates.","commit_id":"78f47b42db89987c611761d87ef3ff2e2c28b491"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"8898ed7e81094158d38b5056b6697345f6712ea5","unresolved":false,"context_lines":[{"line_number":532,"context_line":"            orig_metadata \u003d {}"},{"line_number":533,"context_line":""},{"line_number":534,"context_line":"        # Not sure if this is the right thing to do, but... If we post as"},{"line_number":535,"context_line":"        # as the seconf half of PUT+POST, and there was no existing object,"},{"line_number":536,"context_line":"        # we substitute the newly uploaded object\u0027s metadata for orig_metadata."},{"line_number":537,"context_line":"        # Else, the rest of POST() blows up everywhere it wants old timestamp"},{"line_number":538,"context_line":"        # or content-type. Really, our actions call for PUT-like semantics."}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_3f031a76","line":535,"range":{"start_line":535,"start_character":17,"end_line":535,"end_character":23},"updated":"2017-05-02 17:16:37.000000000","message":"s/seconf/second/","commit_id":"78f47b42db89987c611761d87ef3ff2e2c28b491"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"9c98977a75d350825c281216b4e93da4986b1b00","unresolved":false,"context_lines":[{"line_number":532,"context_line":"            orig_metadata \u003d {}"},{"line_number":533,"context_line":""},{"line_number":534,"context_line":"        # Not sure if this is the right thing to do, but... If we post as"},{"line_number":535,"context_line":"        # as the seconf half of PUT+POST, and there was no existing object,"},{"line_number":536,"context_line":"        # we substitute the newly uploaded object\u0027s metadata for orig_metadata."},{"line_number":537,"context_line":"        # Else, the rest of POST() blows up everywhere it wants old timestamp"},{"line_number":538,"context_line":"        # or content-type. Really, our actions call for PUT-like semantics."}],"source_content_type":"text/x-python","patch_set":8,"id":"3f044301_b95068d6","line":535,"range":{"start_line":535,"start_character":17,"end_line":535,"end_character":23},"in_reply_to":"5ff73747_3f031a76","updated":"2017-05-04 03:29:09.000000000","message":"Done","commit_id":"78f47b42db89987c611761d87ef3ff2e2c28b491"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"8898ed7e81094158d38b5056b6697345f6712ea5","unresolved":false,"context_lines":[{"line_number":535,"context_line":"        # as the seconf half of PUT+POST, and there was no existing object,"},{"line_number":536,"context_line":"        # we substitute the newly uploaded object\u0027s metadata for orig_metadata."},{"line_number":537,"context_line":"        # Else, the rest of POST() blows up everywhere it wants old timestamp"},{"line_number":538,"context_line":"        # or content-type. Really, our actions call for PUT-like semantics."},{"line_number":539,"context_line":"        if to_durable:"},{"line_number":540,"context_line":"            temp_id \u003d request.headers.get(\u0027X-Backend-Temp-Id\u0027)"},{"line_number":541,"context_line":"            if not temp_id:"}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_df2c36ae","line":538,"updated":"2017-05-02 17:16:37.000000000","message":"Wait, so what does this mean for POSTs during partitions? If there\u0027s a split brain and one side sees\n\n t1 PUT  obj\n\nwhile the other side only sees\n\n t2 POST obj\n\nwe still write down the .meta so we can sort it out later, right? Only the existing\n\n except (DiskFileNotExist, DiskFileQuarantined):\n     return HTTPNotFound(request\u003drequest)\n\nkinda implies we don\u0027t...","commit_id":"78f47b42db89987c611761d87ef3ff2e2c28b491"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"9c98977a75d350825c281216b4e93da4986b1b00","unresolved":false,"context_lines":[{"line_number":535,"context_line":"        # as the seconf half of PUT+POST, and there was no existing object,"},{"line_number":536,"context_line":"        # we substitute the newly uploaded object\u0027s metadata for orig_metadata."},{"line_number":537,"context_line":"        # Else, the rest of POST() blows up everywhere it wants old timestamp"},{"line_number":538,"context_line":"        # or content-type. Really, our actions call for PUT-like semantics."},{"line_number":539,"context_line":"        if to_durable:"},{"line_number":540,"context_line":"            temp_id \u003d request.headers.get(\u0027X-Backend-Temp-Id\u0027)"},{"line_number":541,"context_line":"            if not temp_id:"}],"source_content_type":"text/x-python","patch_set":8,"id":"3f044301_7909c0db","line":538,"in_reply_to":"5ff73747_df2c36ae","updated":"2017-05-04 03:29:09.000000000","message":"Hopefuly we reject the POST because if the only side only sees it, the tmp ID will be invalid. I need to verify.","commit_id":"78f47b42db89987c611761d87ef3ff2e2c28b491"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"8898ed7e81094158d38b5056b6697345f6712ea5","unresolved":false,"context_lines":[{"line_number":537,"context_line":"        # Else, the rest of POST() blows up everywhere it wants old timestamp"},{"line_number":538,"context_line":"        # or content-type. Really, our actions call for PUT-like semantics."},{"line_number":539,"context_line":"        if to_durable:"},{"line_number":540,"context_line":"            temp_id \u003d request.headers.get(\u0027X-Backend-Temp-Id\u0027)"},{"line_number":541,"context_line":"            if not temp_id:"},{"line_number":542,"context_line":"                return HTTPBadRequest(body\u003d\u0027Bad Temp-ID\u0027, request\u003drequest,"},{"line_number":543,"context_line":"                                      content_type\u003d\u0027text/plain\u0027)"}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_9f934e01","line":540,"updated":"2017-05-02 17:16:37.000000000","message":"So we send both\n\n X-Backend-Post-To-Durable\n\nand\n\n X-Backend-Temp-Id\n\n... would it be worth just sending the latter, and have that imply to_durable should be True? What possible messages could we want to send about temp ids *besides* \"finalize this\"?","commit_id":"78f47b42db89987c611761d87ef3ff2e2c28b491"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"9c98977a75d350825c281216b4e93da4986b1b00","unresolved":false,"context_lines":[{"line_number":537,"context_line":"        # Else, the rest of POST() blows up everywhere it wants old timestamp"},{"line_number":538,"context_line":"        # or content-type. Really, our actions call for PUT-like semantics."},{"line_number":539,"context_line":"        if to_durable:"},{"line_number":540,"context_line":"            temp_id \u003d request.headers.get(\u0027X-Backend-Temp-Id\u0027)"},{"line_number":541,"context_line":"            if not temp_id:"},{"line_number":542,"context_line":"                return HTTPBadRequest(body\u003d\u0027Bad Temp-ID\u0027, request\u003drequest,"},{"line_number":543,"context_line":"                                      content_type\u003d\u0027text/plain\u0027)"}],"source_content_type":"text/x-python","patch_set":8,"id":"3f044301_39133810","line":540,"in_reply_to":"5ff73747_9f934e01","updated":"2017-05-04 03:29:09.000000000","message":"Yes, using just one would remove a possibility of inconsistency. Added to TODO.","commit_id":"78f47b42db89987c611761d87ef3ff2e2c28b491"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"8898ed7e81094158d38b5056b6697345f6712ea5","unresolved":false,"context_lines":[{"line_number":545,"context_line":"            if not frag_index:"},{"line_number":546,"context_line":"                return HTTPBadRequest(body\u003d\u0027Bad Frag-Index\u0027, request\u003drequest,"},{"line_number":547,"context_line":"                                      content_type\u003d\u0027text/plain\u0027)"},{"line_number":548,"context_line":"            # try:"},{"line_number":549,"context_line":"            orig_metadata \u003d disk_file.hold_commit(temp_id, frag_index)"},{"line_number":550,"context_line":""},{"line_number":551,"context_line":"        orig_timestamp \u003d Timestamp(orig_metadata.get(\u0027X-Timestamp\u0027, 0))"}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_da5e8411","line":548,"updated":"2017-05-02 17:16:37.000000000","message":"Yeah, definitely seems like we need to be tolerant of some exceptions here... what happens when the temp id doesn\u0027t match anything? Or if there\u0027s some corruption between when we wrote it and when we read it back?","commit_id":"78f47b42db89987c611761d87ef3ff2e2c28b491"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"9c98977a75d350825c281216b4e93da4986b1b00","unresolved":false,"context_lines":[{"line_number":545,"context_line":"            if not frag_index:"},{"line_number":546,"context_line":"                return HTTPBadRequest(body\u003d\u0027Bad Frag-Index\u0027, request\u003drequest,"},{"line_number":547,"context_line":"                                      content_type\u003d\u0027text/plain\u0027)"},{"line_number":548,"context_line":"            # try:"},{"line_number":549,"context_line":"            orig_metadata \u003d disk_file.hold_commit(temp_id, frag_index)"},{"line_number":550,"context_line":""},{"line_number":551,"context_line":"        orig_timestamp \u003d Timestamp(orig_metadata.get(\u0027X-Timestamp\u0027, 0))"}],"source_content_type":"text/x-python","patch_set":8,"id":"3f044301_59272466","line":548,"in_reply_to":"5ff73747_da5e8411","updated":"2017-05-04 03:29:09.000000000","message":"Awwww. That corruption question is important and I didn\u0027t think it through properly. I was going to keep the etag that PUT supplied -or- calculated, make the object durable with POST, and then hope that auditor catches any issues.","commit_id":"78f47b42db89987c611761d87ef3ff2e2c28b491"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"8898ed7e81094158d38b5056b6697345f6712ea5","unresolved":false,"context_lines":[{"line_number":591,"context_line":"            metadata \u003d orig_metadata"},{"line_number":592,"context_line":"            metadata.update(val for val in request.headers.items()"},{"line_number":593,"context_line":"                            if (is_sys_or_user_meta(\u0027object\u0027, val[0]) or"},{"line_number":594,"context_line":"                                is_object_transient_sysmeta(val[0])))"},{"line_number":595,"context_line":"        else:"},{"line_number":596,"context_line":"            # preserve existing metadata, only content-type may be updated"},{"line_number":597,"context_line":"            metadata \u003d dict(disk_file.get_metafile_metadata())"}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_3a49e831","line":594,"updated":"2017-05-02 17:16:37.000000000","message":"Why not just\n\n metadata.update(request.headers)\n\n? I thought the footers idea previously allowed for arbitrary headers...","commit_id":"78f47b42db89987c611761d87ef3ff2e2c28b491"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"8898ed7e81094158d38b5056b6697345f6712ea5","unresolved":false,"context_lines":[{"line_number":854,"context_line":"                    # if the PUT requires a two-phase commit (a data and"},{"line_number":855,"context_line":"                    # a commit phase) send the proxy server another"},{"line_number":856,"context_line":"                    # 100-continue response to indicate that we are finished"},{"line_number":857,"context_line":"                    # writing object data"},{"line_number":858,"context_line":"                    if use_multiphase_commit:"},{"line_number":859,"context_line":"                        request.environ[\u0027wsgi.input\u0027].\\"},{"line_number":860,"context_line":"                            send_hundred_continue_response()"}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_ba8df82b","line":857,"updated":"2017-05-02 17:16:37.000000000","message":"Right, we continue to support the old way to allow rolling upgrades...","commit_id":"78f47b42db89987c611761d87ef3ff2e2c28b491"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"46c4035a8817184b73b4a0e75e6fd2c9fdbb0af5","unresolved":false,"context_lines":[{"line_number":604,"context_line":"        # will get filtered out to client as a result of PUT, and the"},{"line_number":605,"context_line":"        # ETag that we calculated during PUT."},{"line_number":606,"context_line":"        if to_durable:"},{"line_number":607,"context_line":"            disk_file.commit(req_timestamp)"},{"line_number":608,"context_line":"            return HTTPCreated(request\u003drequest, etag\u003dorig_metadata.get(\u0027ETag\u0027))"},{"line_number":609,"context_line":""},{"line_number":610,"context_line":"        orig_timestamp \u003d Timestamp(orig_metadata.get(\u0027X-Timestamp\u0027, 0))"}],"source_content_type":"text/x-python","patch_set":19,"id":"5f4e5783_b56f27c4","line":607,"updated":"2017-10-11 01:39:37.000000000","message":"I don\u0027t think this diskfile has it\u0027s self._frag_index set at this point - which will make it blow up when it tries to make a file name","commit_id":"69ef1ba4da42fb6b3d2a6feca8ec39215db6b0a9"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"1c736b4fd62f145cee3cb2d6098812ddb2faf3dc","unresolved":false,"context_lines":[{"line_number":604,"context_line":"        # will get filtered out to client as a result of PUT, and the"},{"line_number":605,"context_line":"        # ETag that we calculated during PUT."},{"line_number":606,"context_line":"        if to_durable:"},{"line_number":607,"context_line":"            disk_file.commit(req_timestamp)"},{"line_number":608,"context_line":"            return HTTPCreated(request\u003drequest, etag\u003dorig_metadata.get(\u0027ETag\u0027))"},{"line_number":609,"context_line":""},{"line_number":610,"context_line":"        orig_timestamp \u003d Timestamp(orig_metadata.get(\u0027X-Timestamp\u0027, 0))"}],"source_content_type":"text/x-python","patch_set":19,"id":"5f4e5783_036f4ac5","line":607,"in_reply_to":"5f4e5783_b56f27c4","updated":"2017-10-17 01:51:27.000000000","message":"See, this is why I wanted you to have a look: you have a mental model of how all this is supposed to work. I noticed that _frag_index was missing if we just get a diskfile and immediately call commit(). So I added a \"fix\" into _construct_from_data_file(). Normally commit() inherits _frag_index from a put(). IOW, we have diskfile instances that are \"good to commit because they\u0027ve been put-ed\" and those that are not. I\u0027m open to general suggestions about how  to fix this better.","commit_id":"69ef1ba4da42fb6b3d2a6feca8ec39215db6b0a9"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"cf5190bda49a26788552ee732aefcf3d11eb261a","unresolved":false,"context_lines":[{"line_number":732,"context_line":"        writer \u003d self.writer_cache.get(temp_id)"},{"line_number":733,"context_line":"        if writer is None:"},{"line_number":734,"context_line":"            return HTTPBadRequest(body\u003d\u0027Bad Temp-Id\u0027, request\u003drequest)"},{"line_number":735,"context_line":"        writer.reparent(disk_file)"},{"line_number":736,"context_line":"        # XXX Do we need to \"orphan\" a writer forcefuly upon the exit?"},{"line_number":737,"context_line":""},{"line_number":738,"context_line":"        # This could be a good time to process X-Delete-At, but we left"}],"source_content_type":"text/x-python","patch_set":25,"id":"1f9dbf25_3f22da53","line":735,"updated":"2018-03-03 01:37:58.000000000","message":"Out of curiosity, why do we need to reparent? What new information are we getting?","commit_id":"79fec861768bb219815e6cab80d3b1320e393b1e"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"2916044a01da4770a045ad09d15287c44263d3c1","unresolved":false,"context_lines":[{"line_number":732,"context_line":"        writer \u003d self.writer_cache.get(temp_id)"},{"line_number":733,"context_line":"        if writer is None:"},{"line_number":734,"context_line":"            return HTTPBadRequest(body\u003d\u0027Bad Temp-Id\u0027, request\u003drequest)"},{"line_number":735,"context_line":"        writer.reparent(disk_file)"},{"line_number":736,"context_line":"        # XXX Do we need to \"orphan\" a writer forcefuly upon the exit?"},{"line_number":737,"context_line":""},{"line_number":738,"context_line":"        # This could be a good time to process X-Delete-At, but we left"}],"source_content_type":"text/x-python","patch_set":25,"id":"df7087c5_915af04c","line":735,"in_reply_to":"1f9dbf25_3f22da53","updated":"2018-03-12 04:07:16.000000000","message":"If I read you right, your question has 2 sides: 1) why reparent at all, instead of using an \"orphan\" DiskFileWrater or keeping whole DiskFile, and/or 2) why the preservation in the first place?\n\n(2) The preservation allows us to keep the anonymous temp file. Now, I don\u0027t remember _why exactly_ it was so good. Your XFS must collect all the unnamed tempfiles on a crash, you know. But apparently it\u0027s faster? And there\u0027s less junk in /srv/node/DEV/object/tempfiles. Also, once we have it, the  persistent DiskFileWriter allows to keep the metadata that is essential, most importantly the fragment index. Before, saving the fragment index into metadata and restoring it required insane gyrations, gone from this version.\n\n(1) I tried each way and this was less disruptive to assumptions embedded into the code. Running with 2 DiskFiles was just plain confusing. Running without was a crash city all around and weird workarounds.","commit_id":"79fec861768bb219815e6cab80d3b1320e393b1e"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"cf5190bda49a26788552ee732aefcf3d11eb261a","unresolved":false,"context_lines":[{"line_number":735,"context_line":"        writer.reparent(disk_file)"},{"line_number":736,"context_line":"        # XXX Do we need to \"orphan\" a writer forcefuly upon the exit?"},{"line_number":737,"context_line":""},{"line_number":738,"context_line":"        # This could be a good time to process X-Delete-At, but we left"},{"line_number":739,"context_line":"        # all that at Stage 1, PUT."},{"line_number":740,"context_line":""},{"line_number":741,"context_line":"        frag_index \u003d request.headers.get(\u0027X-Object-Sysmeta-Ec-Frag-Index\u0027)"}],"source_content_type":"text/x-python","patch_set":25,"id":"1f9dbf25_ff1b1218","line":738,"range":{"start_line":738,"start_character":24,"end_line":738,"end_character":58},"updated":"2018-03-03 01:37:58.000000000","message":"?\n\nDid this recently change with all of Sam\u0027s X-Delete-At/After patches?","commit_id":"79fec861768bb219815e6cab80d3b1320e393b1e"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"2916044a01da4770a045ad09d15287c44263d3c1","unresolved":false,"context_lines":[{"line_number":735,"context_line":"        writer.reparent(disk_file)"},{"line_number":736,"context_line":"        # XXX Do we need to \"orphan\" a writer forcefuly upon the exit?"},{"line_number":737,"context_line":""},{"line_number":738,"context_line":"        # This could be a good time to process X-Delete-At, but we left"},{"line_number":739,"context_line":"        # all that at Stage 1, PUT."},{"line_number":740,"context_line":""},{"line_number":741,"context_line":"        frag_index \u003d request.headers.get(\u0027X-Object-Sysmeta-Ec-Frag-Index\u0027)"}],"source_content_type":"text/x-python","patch_set":25,"id":"df7087c5_313f64d7","line":738,"range":{"start_line":738,"start_character":24,"end_line":738,"end_character":58},"in_reply_to":"1f9dbf25_ff1b1218","updated":"2018-03-12 04:07:16.000000000","message":"You may be right, I\u0027ll re-check.","commit_id":"79fec861768bb219815e6cab80d3b1320e393b1e"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"cf5190bda49a26788552ee732aefcf3d11eb261a","unresolved":false,"context_lines":[{"line_number":768,"context_line":"        # object POST updates are PUT to the container server"},{"line_number":769,"context_line":"        self.container_update("},{"line_number":770,"context_line":"            \u0027PUT\u0027, account, container, obj, request, update_headers,"},{"line_number":771,"context_line":"            device, policy)"},{"line_number":772,"context_line":""},{"line_number":773,"context_line":"        # Add sysmeta to response"},{"line_number":774,"context_line":"        resp_headers \u003d {}"}],"source_content_type":"text/x-python","patch_set":25,"id":"1f9dbf25_fff0f2cb","line":771,"updated":"2018-03-03 01:37:58.000000000","message":"So we do (or save off) the container update before we\u0027re actually durable... interesting...","commit_id":"79fec861768bb219815e6cab80d3b1320e393b1e"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"2916044a01da4770a045ad09d15287c44263d3c1","unresolved":false,"context_lines":[{"line_number":768,"context_line":"        # object POST updates are PUT to the container server"},{"line_number":769,"context_line":"        self.container_update("},{"line_number":770,"context_line":"            \u0027PUT\u0027, account, container, obj, request, update_headers,"},{"line_number":771,"context_line":"            device, policy)"},{"line_number":772,"context_line":""},{"line_number":773,"context_line":"        # Add sysmeta to response"},{"line_number":774,"context_line":"        resp_headers \u003d {}"}],"source_content_type":"text/x-python","patch_set":25,"id":"df7087c5_f1200cb3","line":771,"in_reply_to":"1f9dbf25_fff0f2cb","updated":"2018-03-12 04:07:16.000000000","message":"I think the container happens in the exact same place as before, relatively to phases. And yes, now that non-durable object does not return 404, doesn\u0027t it make sense to have it accounted in the container server? Surely it counts against one\u0027s quota.","commit_id":"79fec861768bb219815e6cab80d3b1320e393b1e"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"cf5190bda49a26788552ee732aefcf3d11eb261a","unresolved":false,"context_lines":[{"line_number":829,"context_line":"        temp_id \u003d None"},{"line_number":830,"context_line":"        temp_stage \u003d False"},{"line_number":831,"context_line":"        if request.headers.get(\u0027X-Backend-Temp-Hold\u0027) is not None:"},{"line_number":832,"context_line":"            temp_stage \u003d True"},{"line_number":833,"context_line":"        req_timestamp \u003d valid_timestamp(request)"},{"line_number":834,"context_line":"        error_response \u003d check_object_creation(request, obj)"},{"line_number":835,"context_line":"        if error_response:"}],"source_content_type":"text/x-python","patch_set":25,"id":"1f9dbf25_df02d609","line":832,"updated":"2018-03-03 01:37:58.000000000","message":"Better to do something like\n\n temp_stage \u003d config_true_value(request.headers.get(\u0027X-Backend-Temp-Hold\u0027))\n\nsimilar to what we do for the older \"flag\" headers?","commit_id":"79fec861768bb219815e6cab80d3b1320e393b1e"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"2916044a01da4770a045ad09d15287c44263d3c1","unresolved":false,"context_lines":[{"line_number":829,"context_line":"        temp_id \u003d None"},{"line_number":830,"context_line":"        temp_stage \u003d False"},{"line_number":831,"context_line":"        if request.headers.get(\u0027X-Backend-Temp-Hold\u0027) is not None:"},{"line_number":832,"context_line":"            temp_stage \u003d True"},{"line_number":833,"context_line":"        req_timestamp \u003d valid_timestamp(request)"},{"line_number":834,"context_line":"        error_response \u003d check_object_creation(request, obj)"},{"line_number":835,"context_line":"        if error_response:"}],"source_content_type":"text/x-python","patch_set":25,"id":"df7087c5_cc13eb5b","line":832,"in_reply_to":"1f9dbf25_df02d609","updated":"2018-03-12 04:07:16.000000000","message":"This is consistent with the same in the POST method, above. But either way is fine by me, so I changed it. Note that the API for \"X-Backend-Temp-Hold: 0\" means \"yes hold\" in my old code and \"no hold\" in yours. Not like it matters, but heh.","commit_id":"79fec861768bb219815e6cab80d3b1320e393b1e"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"cf5190bda49a26788552ee732aefcf3d11eb261a","unresolved":false,"context_lines":[{"line_number":982,"context_line":"                metadata.update(val for val in request.headers.items()"},{"line_number":983,"context_line":"                                if (is_sys_or_user_meta(\u0027object\u0027, val[0]) or"},{"line_number":984,"context_line":"                                    is_object_transient_sysmeta(val[0])))"},{"line_number":985,"context_line":"                metadata.update(val for val in footer_meta.items()"},{"line_number":986,"context_line":"                                if (is_sys_or_user_meta(\u0027object\u0027, val[0]) or"},{"line_number":987,"context_line":"                                    is_object_transient_sysmeta(val[0])))"},{"line_number":988,"context_line":"                headers_to_copy \u003d ("}],"source_content_type":"text/x-python","patch_set":25,"id":"1f9dbf25_1f311eed","line":985,"updated":"2018-03-03 01:37:58.000000000","message":"Just to make sure: temp_stage implies no footer data, yes?\n\nSanity check -- there are four configurations of headers that may be sent:\n\n- X-Backend-Obj-Multiphase-Commit and X-Backend-Obj-Metadata-Footer (for old-style EC)\n- Just X-Backend-Obj-Metadata-Footer (for old-style encryption)\n- Just X-Backend-Temp-Hold (for new-style EC and encryption)\n- None of the above three headers (for replicated, unencrypted objects)","commit_id":"79fec861768bb219815e6cab80d3b1320e393b1e"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"2916044a01da4770a045ad09d15287c44263d3c1","unresolved":false,"context_lines":[{"line_number":982,"context_line":"                metadata.update(val for val in request.headers.items()"},{"line_number":983,"context_line":"                                if (is_sys_or_user_meta(\u0027object\u0027, val[0]) or"},{"line_number":984,"context_line":"                                    is_object_transient_sysmeta(val[0])))"},{"line_number":985,"context_line":"                metadata.update(val for val in footer_meta.items()"},{"line_number":986,"context_line":"                                if (is_sys_or_user_meta(\u0027object\u0027, val[0]) or"},{"line_number":987,"context_line":"                                    is_object_transient_sysmeta(val[0])))"},{"line_number":988,"context_line":"                headers_to_copy \u003d ("}],"source_content_type":"text/x-python","patch_set":25,"id":"df7087c5_ac69d78b","line":985,"in_reply_to":"1f9dbf25_1f311eed","updated":"2018-03-12 04:07:16.000000000","message":"That enumeration sounds correct to me.","commit_id":"79fec861768bb219815e6cab80d3b1320e393b1e"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"c842cb5ed37538269282bb08f3652c0bc749970a","unresolved":false,"context_lines":[{"line_number":705,"context_line":"                       policy, request, req_timestamp, temp_id):"},{"line_number":706,"context_line":"        \"\"\"The break-out for 3-stage PUT+POST, Stage 2: apply trailers\"\"\""},{"line_number":707,"context_line":""},{"line_number":708,"context_line":"        if policy.policy_type !\u003d EC_POLICY:"},{"line_number":709,"context_line":"            return HTTPBadRequest("},{"line_number":710,"context_line":"                body\u003d\u0027PUT+POST must use an EC policy\u0027,"},{"line_number":711,"context_line":"                request\u003drequest, content_type\u003d\u0027text/plain\u0027)"},{"line_number":712,"context_line":"        if len(temp_id) \u003d\u003d 0 or \u0027/\u0027 in temp_id:"},{"line_number":713,"context_line":"            return HTTPBadRequest("},{"line_number":714,"context_line":"                body\u003d\u0027PUT+POST given bad X-Backend-Temp-Id\u0027,"}],"source_content_type":"text/x-python","patch_set":27,"id":"5f7c97a3_17fcfd6f","line":711,"range":{"start_line":708,"start_character":8,"end_line":711,"end_character":59},"updated":"2018-06-14 17:43:39.000000000","message":"At some point we are going to need this new protocol to work with replication policies, for encrypted objects. I need to study more to understand the requirement for EC policy.","commit_id":"061690bbc467be6a017aaddd5c3cc79632ec55b8"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"c842cb5ed37538269282bb08f3652c0bc749970a","unresolved":false,"context_lines":[{"line_number":709,"context_line":"            return HTTPBadRequest("},{"line_number":710,"context_line":"                body\u003d\u0027PUT+POST must use an EC policy\u0027,"},{"line_number":711,"context_line":"                request\u003drequest, content_type\u003d\u0027text/plain\u0027)"},{"line_number":712,"context_line":"        if len(temp_id) \u003d\u003d 0 or \u0027/\u0027 in temp_id:"},{"line_number":713,"context_line":"            return HTTPBadRequest("},{"line_number":714,"context_line":"                body\u003d\u0027PUT+POST given bad X-Backend-Temp-Id\u0027,"},{"line_number":715,"context_line":"                request\u003drequest, content_type\u003d\u0027text/plain\u0027)"}],"source_content_type":"text/x-python","patch_set":27,"id":"5f7c97a3_774f7191","line":712,"range":{"start_line":712,"start_character":8,"end_line":712,"end_character":46},"updated":"2018-06-14 17:43:39.000000000","message":"I think it would be good to encapsulate this validation in the DiskFileCache which is where temp ids are generated.\n\nAlso, this check does not occur in _post_commit, so perhaps should be done in POST() before either _post_metadata or _post_commit are called.\n\nIn fact, why do this check at all - the outcome will be the same if we just call self.writer_cache.get(temp_id) with a bad temp_id - or is there some subtlety I am missing?","commit_id":"061690bbc467be6a017aaddd5c3cc79632ec55b8"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"c492dc9f4e7cc1ce5e7ba1e00fcdb6589b351be1","unresolved":false,"context_lines":[{"line_number":709,"context_line":"            return HTTPBadRequest("},{"line_number":710,"context_line":"                body\u003d\u0027PUT+POST must use an EC policy\u0027,"},{"line_number":711,"context_line":"                request\u003drequest, content_type\u003d\u0027text/plain\u0027)"},{"line_number":712,"context_line":"        if len(temp_id) \u003d\u003d 0 or \u0027/\u0027 in temp_id:"},{"line_number":713,"context_line":"            return HTTPBadRequest("},{"line_number":714,"context_line":"                body\u003d\u0027PUT+POST given bad X-Backend-Temp-Id\u0027,"},{"line_number":715,"context_line":"                request\u003drequest, content_type\u003d\u0027text/plain\u0027)"}],"source_content_type":"text/x-python","patch_set":27,"id":"5f7c97a3_47601e36","line":712,"range":{"start_line":712,"start_character":8,"end_line":712,"end_character":46},"in_reply_to":"5f7c97a3_774f7191","updated":"2018-06-20 01:09:29.000000000","message":"You\u0027re right, the validation of IDs are vestiges from the time when IDs were simply filenames, from before DiskFileCache. Will drop and rely on the lookup.","commit_id":"061690bbc467be6a017aaddd5c3cc79632ec55b8"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"c842cb5ed37538269282bb08f3652c0bc749970a","unresolved":false,"context_lines":[{"line_number":714,"context_line":"                body\u003d\u0027PUT+POST given bad X-Backend-Temp-Id\u0027,"},{"line_number":715,"context_line":"                request\u003drequest, content_type\u003d\u0027text/plain\u0027)"},{"line_number":716,"context_line":""},{"line_number":717,"context_line":"        next_part_power \u003d request.headers.get(\u0027X-Backend-Next-Part-Power\u0027)"},{"line_number":718,"context_line":"        try:"},{"line_number":719,"context_line":"            disk_file \u003d self.get_diskfile("},{"line_number":720,"context_line":"                device, partition, account, container, obj, policy\u003dpolicy,"},{"line_number":721,"context_line":"                next_part_power\u003dnext_part_power)"},{"line_number":722,"context_line":"        except DiskFileDeviceUnavailable:"},{"line_number":723,"context_line":"            return HTTPInsufficientStorage(drive\u003ddevice, request\u003drequest)"},{"line_number":724,"context_line":""},{"line_number":725,"context_line":"        writer \u003d self.writer_cache.get(temp_id)"},{"line_number":726,"context_line":"        if writer is None:"}],"source_content_type":"text/x-python","patch_set":27,"id":"5f7c97a3_b88bb5be","line":723,"range":{"start_line":717,"start_character":8,"end_line":723,"end_character":73},"updated":"2018-06-14 17:43:39.000000000","message":"this could probably be done upfront in the main POST method","commit_id":"061690bbc467be6a017aaddd5c3cc79632ec55b8"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"c492dc9f4e7cc1ce5e7ba1e00fcdb6589b351be1","unresolved":false,"context_lines":[{"line_number":714,"context_line":"                body\u003d\u0027PUT+POST given bad X-Backend-Temp-Id\u0027,"},{"line_number":715,"context_line":"                request\u003drequest, content_type\u003d\u0027text/plain\u0027)"},{"line_number":716,"context_line":""},{"line_number":717,"context_line":"        next_part_power \u003d request.headers.get(\u0027X-Backend-Next-Part-Power\u0027)"},{"line_number":718,"context_line":"        try:"},{"line_number":719,"context_line":"            disk_file \u003d self.get_diskfile("},{"line_number":720,"context_line":"                device, partition, account, container, obj, policy\u003dpolicy,"},{"line_number":721,"context_line":"                next_part_power\u003dnext_part_power)"},{"line_number":722,"context_line":"        except DiskFileDeviceUnavailable:"},{"line_number":723,"context_line":"            return HTTPInsufficientStorage(drive\u003ddevice, request\u003drequest)"},{"line_number":724,"context_line":""},{"line_number":725,"context_line":"        writer \u003d self.writer_cache.get(temp_id)"},{"line_number":726,"context_line":"        if writer is None:"}],"source_content_type":"text/x-python","patch_set":27,"id":"5f7c97a3_67c8ba20","line":723,"range":{"start_line":717,"start_character":8,"end_line":723,"end_character":73},"in_reply_to":"5f7c97a3_b88bb5be","updated":"2018-06-20 01:09:29.000000000","message":"parameters to DiskFile() are not the same on all branches (phases), and it\u0027s clumsy to make this determination once, and then repeat it when choosing the branch. Although, let me verify that parameters differ anymore.","commit_id":"061690bbc467be6a017aaddd5c3cc79632ec55b8"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"c842cb5ed37538269282bb08f3652c0bc749970a","unresolved":false,"context_lines":[{"line_number":738,"context_line":"            return HTTPBadRequest(body\u003d\u0027Bad Frag-Index\u0027, request\u003drequest,"},{"line_number":739,"context_line":"                                  content_type\u003d\u0027text/plain\u0027)"},{"line_number":740,"context_line":""},{"line_number":741,"context_line":"        writer.unstage(frag_index)"},{"line_number":742,"context_line":""},{"line_number":743,"context_line":"        metadata \u003d orig_metadata"},{"line_number":744,"context_line":"        metadata.update(val for val in request.headers.items()"}],"source_content_type":"text/x-python","patch_set":27,"id":"5f7c97a3_98ecb9f5","line":741,"range":{"start_line":741,"start_character":8,"end_line":741,"end_character":34},"updated":"2018-06-14 17:43:39.000000000","message":"this renames the tmp file to the .data file, then at line 748 the metadata is written to the data file. This is the opposite order w.r.t. the legacy PUT path where the metadata is written to the tmp file and then the rename happens - that way if the metadata write fails we don\u0027t end up with a \u0027partially\u0027 written .data file.","commit_id":"061690bbc467be6a017aaddd5c3cc79632ec55b8"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"c842cb5ed37538269282bb08f3652c0bc749970a","unresolved":false,"context_lines":[{"line_number":745,"context_line":"                        if (is_sys_or_user_meta(\u0027object\u0027, val[0]) or"},{"line_number":746,"context_line":"                            is_object_transient_sysmeta(val[0])))"},{"line_number":747,"context_line":"        try:"},{"line_number":748,"context_line":"            writer.write_metadata(metadata)"},{"line_number":749,"context_line":"        except (DiskFileXattrNotSupported, DiskFileNoSpace):"},{"line_number":750,"context_line":"            return HTTPInsufficientStorage(drive\u003ddevice, request\u003drequest)"},{"line_number":751,"context_line":""}],"source_content_type":"text/x-python","patch_set":27,"id":"5f7c97a3_f88dad35","line":748,"range":{"start_line":748,"start_character":12,"end_line":748,"end_character":43},"updated":"2018-06-14 17:43:39.000000000","message":"is it possible we could just call writer.put(metadata) here and not need the unstage method? If i understand correctly, stage is caching the metadata received with the PUT, then here we update it with metadata received with the POST, and then we can finalize the PUT.","commit_id":"061690bbc467be6a017aaddd5c3cc79632ec55b8"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"c492dc9f4e7cc1ce5e7ba1e00fcdb6589b351be1","unresolved":false,"context_lines":[{"line_number":745,"context_line":"                        if (is_sys_or_user_meta(\u0027object\u0027, val[0]) or"},{"line_number":746,"context_line":"                            is_object_transient_sysmeta(val[0])))"},{"line_number":747,"context_line":"        try:"},{"line_number":748,"context_line":"            writer.write_metadata(metadata)"},{"line_number":749,"context_line":"        except (DiskFileXattrNotSupported, DiskFileNoSpace):"},{"line_number":750,"context_line":"            return HTTPInsufficientStorage(drive\u003ddevice, request\u003drequest)"},{"line_number":751,"context_line":""}],"source_content_type":"text/x-python","patch_set":27,"id":"5f7c97a3_30ac3a9e","line":748,"range":{"start_line":748,"start_character":12,"end_line":748,"end_character":43},"in_reply_to":"5f7c97a3_f88dad35","updated":"2018-06-20 01:09:29.000000000","message":"Yes, this should be just a put(), but there was something different about it that prevented me from doing it. Some information was not available or whatnot, perhaps frag index. \u0027ll have another look.","commit_id":"061690bbc467be6a017aaddd5c3cc79632ec55b8"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"c842cb5ed37538269282bb08f3652c0bc749970a","unresolved":false,"context_lines":[{"line_number":753,"context_line":"            \u0027x-size\u0027: orig_metadata[\u0027Content-Length\u0027],"},{"line_number":754,"context_line":"            \u0027x-content-type\u0027: orig_metadata[\u0027Content-Type\u0027],"},{"line_number":755,"context_line":"            \u0027x-timestamp\u0027: orig_metadata[\u0027X-Timestamp\u0027],"},{"line_number":756,"context_line":"            \u0027x-content-type-timestamp\u0027: orig_metadata[\u0027X-Timestamp\u0027],"},{"line_number":757,"context_line":"            \u0027x-meta-timestamp\u0027: metadata[\u0027X-Timestamp\u0027],"},{"line_number":758,"context_line":"            \u0027x-etag\u0027: orig_metadata[\u0027ETag\u0027]})"},{"line_number":759,"context_line":""},{"line_number":760,"context_line":"        self._check_container_override(update_headers, orig_metadata)"}],"source_content_type":"text/x-python","patch_set":27,"id":"5f7c97a3_2dff6666","line":757,"range":{"start_line":756,"start_character":12,"end_line":757,"end_character":56},"updated":"2018-06-14 17:43:39.000000000","message":"It\u0027s not strictly necessary to include these headers when their value is equal to x-timestamp.","commit_id":"061690bbc467be6a017aaddd5c3cc79632ec55b8"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"c842cb5ed37538269282bb08f3652c0bc749970a","unresolved":false,"context_lines":[{"line_number":757,"context_line":"            \u0027x-meta-timestamp\u0027: metadata[\u0027X-Timestamp\u0027],"},{"line_number":758,"context_line":"            \u0027x-etag\u0027: orig_metadata[\u0027ETag\u0027]})"},{"line_number":759,"context_line":""},{"line_number":760,"context_line":"        self._check_container_override(update_headers, orig_metadata)"},{"line_number":761,"context_line":""},{"line_number":762,"context_line":"        # object POST updates are PUT to the container server"},{"line_number":763,"context_line":"        self.container_update("}],"source_content_type":"text/x-python","patch_set":27,"id":"5f7c97a3_38e805b2","line":760,"range":{"start_line":760,"start_character":8,"end_line":760,"end_character":69},"updated":"2018-06-14 17:43:39.000000000","message":"request.headers (or some subset) should be passed as the third arg (footers) similar to line 1048. These request headers are equivalent to the MIME footers.","commit_id":"061690bbc467be6a017aaddd5c3cc79632ec55b8"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"c842cb5ed37538269282bb08f3652c0bc749970a","unresolved":false,"context_lines":[{"line_number":759,"context_line":""},{"line_number":760,"context_line":"        self._check_container_override(update_headers, orig_metadata)"},{"line_number":761,"context_line":""},{"line_number":762,"context_line":"        # object POST updates are PUT to the container server"},{"line_number":763,"context_line":"        self.container_update("},{"line_number":764,"context_line":"            \u0027PUT\u0027, account, container, obj, request, update_headers,"},{"line_number":765,"context_line":"            device, policy)"},{"line_number":766,"context_line":""},{"line_number":767,"context_line":"        # Add sysmeta to response"},{"line_number":768,"context_line":"        resp_headers \u003d {}"}],"source_content_type":"text/x-python","patch_set":27,"id":"5f7c97a3_f81dad77","line":765,"range":{"start_line":762,"start_character":8,"end_line":765,"end_character":27},"updated":"2018-06-14 17:43:39.000000000","message":"comparing with the legacy PUT path, I think the container server update should wait until the commit is done.","commit_id":"061690bbc467be6a017aaddd5c3cc79632ec55b8"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"c842cb5ed37538269282bb08f3652c0bc749970a","unresolved":false,"context_lines":[{"line_number":764,"context_line":"            \u0027PUT\u0027, account, container, obj, request, update_headers,"},{"line_number":765,"context_line":"            device, policy)"},{"line_number":766,"context_line":""},{"line_number":767,"context_line":"        # Add sysmeta to response"},{"line_number":768,"context_line":"        resp_headers \u003d {}"},{"line_number":769,"context_line":"        for key, value in orig_metadata.items():"},{"line_number":770,"context_line":"            if is_sys_meta(\u0027object\u0027, key):"},{"line_number":771,"context_line":"                resp_headers[key] \u003d value"},{"line_number":772,"context_line":""},{"line_number":773,"context_line":"        # Yes, Proxy wants ETag out of Stage 2. It even crashes outright"},{"line_number":774,"context_line":"        # if this is not supplied."}],"source_content_type":"text/x-python","patch_set":27,"id":"5f7c97a3_b8009504","line":771,"range":{"start_line":767,"start_character":8,"end_line":771,"end_character":41},"updated":"2018-06-14 17:43:39.000000000","message":"is this necessary in the context of _post_metadata() ?","commit_id":"061690bbc467be6a017aaddd5c3cc79632ec55b8"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"c842cb5ed37538269282bb08f3652c0bc749970a","unresolved":false,"context_lines":[{"line_number":779,"context_line":"                     policy, request, req_timestamp, temp_id):"},{"line_number":780,"context_line":"        \"\"\"The break-out for 3-stage PUT+POST, Stage 3: mark as durable\"\"\""},{"line_number":781,"context_line":""},{"line_number":782,"context_line":"        # Without frag_prefs, the DiskFile.read_metadata() below throws"},{"line_number":783,"context_line":"        # a DiskFileNotExist. See how ECDiskFile._get_ondisk_files chooses"},{"line_number":784,"context_line":"        # fragments and fills [\u0027data_info\u0027]."},{"line_number":785,"context_line":"        frag_prefs \u003d [{\u0027timestamp\u0027: req_timestamp.internal, \u0027exclude\u0027: []}]"},{"line_number":786,"context_line":""},{"line_number":787,"context_line":"        next_part_power \u003d request.headers.get(\u0027X-Backend-Next-Part-Power\u0027)"}],"source_content_type":"text/x-python","patch_set":27,"id":"5f7c97a3_cde6f216","line":784,"range":{"start_line":782,"start_character":8,"end_line":784,"end_character":44},"updated":"2018-06-14 17:43:39.000000000","message":"I wonder if this is actually an existing bug, or at least potentiall improvement - the implication is that a POST to a nondurable object will return 404. I\u0027m not sure there is any reason we could not use default frag_prefs so that any POST would be accepted.","commit_id":"061690bbc467be6a017aaddd5c3cc79632ec55b8"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"c842cb5ed37538269282bb08f3652c0bc749970a","unresolved":false,"context_lines":[{"line_number":782,"context_line":"        # Without frag_prefs, the DiskFile.read_metadata() below throws"},{"line_number":783,"context_line":"        # a DiskFileNotExist. See how ECDiskFile._get_ondisk_files chooses"},{"line_number":784,"context_line":"        # fragments and fills [\u0027data_info\u0027]."},{"line_number":785,"context_line":"        frag_prefs \u003d [{\u0027timestamp\u0027: req_timestamp.internal, \u0027exclude\u0027: []}]"},{"line_number":786,"context_line":""},{"line_number":787,"context_line":"        next_part_power \u003d request.headers.get(\u0027X-Backend-Next-Part-Power\u0027)"},{"line_number":788,"context_line":"        try:"}],"source_content_type":"text/x-python","patch_set":27,"id":"5f7c97a3_2ddc4643","line":785,"range":{"start_line":785,"start_character":22,"end_line":785,"end_character":74},"updated":"2018-06-14 17:43:39.000000000","message":"ah, ok, so we need to make sure the matching timestamp fragment is  selected, not just *any* fragment","commit_id":"061690bbc467be6a017aaddd5c3cc79632ec55b8"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"c842cb5ed37538269282bb08f3652c0bc749970a","unresolved":false,"context_lines":[{"line_number":792,"context_line":"                next_part_power\u003dnext_part_power)"},{"line_number":793,"context_line":"        except DiskFileDeviceUnavailable:"},{"line_number":794,"context_line":"            return HTTPInsufficientStorage(drive\u003ddevice, request\u003drequest)"},{"line_number":795,"context_line":"        try:"},{"line_number":796,"context_line":"            orig_metadata \u003d disk_file.read_metadata()"},{"line_number":797,"context_line":"        except DiskFileXattrNotSupported:"},{"line_number":798,"context_line":"            return HTTPInsufficientStorage(drive\u003ddevice, request\u003drequest)"},{"line_number":799,"context_line":"        except (DiskFileNotExist, DiskFileQuarantined):"},{"line_number":800,"context_line":"            return HTTPNotFound(request\u003drequest)"},{"line_number":801,"context_line":""},{"line_number":802,"context_line":"        writer \u003d self.writer_cache.get(temp_id)"},{"line_number":803,"context_line":"        if writer is None:"}],"source_content_type":"text/x-python","patch_set":27,"id":"5f7c97a3_78bf5d60","line":800,"range":{"start_line":795,"start_character":8,"end_line":800,"end_character":48},"updated":"2018-06-14 17:43:39.000000000","message":"doesn\u0027t the write have a cache of the relevant metadata i.e. the etag? if we use that then the diskfile isn\u0027t read so the frag_prefs trickery isn\u0027t needed.","commit_id":"061690bbc467be6a017aaddd5c3cc79632ec55b8"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"c842cb5ed37538269282bb08f3652c0bc749970a","unresolved":false,"context_lines":[{"line_number":1024,"context_line":""},{"line_number":1025,"context_line":"        except (DiskFileXattrNotSupported, DiskFileNoSpace):"},{"line_number":1026,"context_line":"            return HTTPInsufficientStorage(drive\u003ddevice, request\u003drequest)"},{"line_number":1027,"context_line":"        if orig_delete_at !\u003d new_delete_at:"},{"line_number":1028,"context_line":"            if new_delete_at:"},{"line_number":1029,"context_line":"                self.delete_at_update("},{"line_number":1030,"context_line":"                    \u0027PUT\u0027, new_delete_at, account, container, obj, request,"},{"line_number":1031,"context_line":"                    device, policy)"},{"line_number":1032,"context_line":"            if orig_delete_at:"},{"line_number":1033,"context_line":"                self.delete_at_update("},{"line_number":1034,"context_line":"                    \u0027DELETE\u0027, orig_delete_at, account, container, obj,"},{"line_number":1035,"context_line":"                    request, device, policy)"},{"line_number":1036,"context_line":""},{"line_number":1037,"context_line":"        if temp_id:"},{"line_number":1038,"context_line":"            return HTTPCreated(request\u003drequest,"}],"source_content_type":"text/x-python","patch_set":27,"id":"5f7c97a3_f7c62149","line":1035,"range":{"start_line":1027,"start_character":0,"end_line":1035,"end_character":44},"updated":"2018-06-14 17:43:39.000000000","message":"should this also be deferred until the object has been committed? at this point in time the PUT may not succeed so we shouldn\u0027t remove previous entry from the expirer queue?","commit_id":"061690bbc467be6a017aaddd5c3cc79632ec55b8"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"c492dc9f4e7cc1ce5e7ba1e00fcdb6589b351be1","unresolved":false,"context_lines":[{"line_number":1024,"context_line":""},{"line_number":1025,"context_line":"        except (DiskFileXattrNotSupported, DiskFileNoSpace):"},{"line_number":1026,"context_line":"            return HTTPInsufficientStorage(drive\u003ddevice, request\u003drequest)"},{"line_number":1027,"context_line":"        if orig_delete_at !\u003d new_delete_at:"},{"line_number":1028,"context_line":"            if new_delete_at:"},{"line_number":1029,"context_line":"                self.delete_at_update("},{"line_number":1030,"context_line":"                    \u0027PUT\u0027, new_delete_at, account, container, obj, request,"},{"line_number":1031,"context_line":"                    device, policy)"},{"line_number":1032,"context_line":"            if orig_delete_at:"},{"line_number":1033,"context_line":"                self.delete_at_update("},{"line_number":1034,"context_line":"                    \u0027DELETE\u0027, orig_delete_at, account, container, obj,"},{"line_number":1035,"context_line":"                    request, device, policy)"},{"line_number":1036,"context_line":""},{"line_number":1037,"context_line":"        if temp_id:"},{"line_number":1038,"context_line":"            return HTTPCreated(request\u003drequest,"}],"source_content_type":"text/x-python","patch_set":27,"id":"5f7c97a3_9bc76f65","line":1035,"range":{"start_line":1027,"start_character":0,"end_line":1035,"end_character":44},"in_reply_to":"5f7c97a3_f7c62149","updated":"2018-06-20 01:09:29.000000000","message":"I\u0027ll check once again, what the original does about the expiration.","commit_id":"061690bbc467be6a017aaddd5c3cc79632ec55b8"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"3a4e11c77137716276fc6c095aa1ad508299c7a0","unresolved":false,"context_lines":[{"line_number":181,"context_line":"            socket._fileobject.default_bufsize \u003d self.network_chunk_size"},{"line_number":182,"context_line":"        # TODO: find a way to enable similar functionality in py3"},{"line_number":183,"context_line":""},{"line_number":184,"context_line":"        self.writer_cache \u003d DiskFileCache()"},{"line_number":185,"context_line":""},{"line_number":186,"context_line":"        # Provide further setup specific to an object server implementation."},{"line_number":187,"context_line":"        self.setup(conf)"}],"source_content_type":"text/x-python","patch_set":32,"id":"3f79a3b5_4e8737d9","line":184,"updated":"2018-08-04 00:48:58.000000000","message":"So that reclaim_time arg isn\u0027t actually configurable?","commit_id":"83256db01ae98b902df7691cad155bbdc6024080"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"15157f07ee0caffa1281eb922e8fecac6a7ffd95","unresolved":false,"context_lines":[{"line_number":181,"context_line":"            socket._fileobject.default_bufsize \u003d self.network_chunk_size"},{"line_number":182,"context_line":"        # TODO: find a way to enable similar functionality in py3"},{"line_number":183,"context_line":""},{"line_number":184,"context_line":"        self.writer_cache \u003d DiskFileCache()"},{"line_number":185,"context_line":""},{"line_number":186,"context_line":"        # Provide further setup specific to an object server implementation."},{"line_number":187,"context_line":"        self.setup(conf)"}],"source_content_type":"text/x-python","patch_set":32,"id":"3f79a3b5_581ef60d","line":184,"in_reply_to":"3f79a3b5_4e8737d9","updated":"2018-09-11 21:58:05.000000000","message":"It can easily be made configurable. But I hate adding knobs. The 100 seconds ought to be enough for everybody and all that. Timeout counts from the time server accepts the body, so length of the transfer should not be an issue. Basically it\u0027s a time for proxy to wait for slow nodes to complete (so, maybe node_timeout*2?). Normally you\u0027d think 10 seconds were enough. But jacking it up is not that bad, right? So, 100 should be fine, right?","commit_id":"83256db01ae98b902df7691cad155bbdc6024080"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"ade2f0775d29c70a287a89a39524a57cdc5bb005","unresolved":false,"context_lines":[{"line_number":615,"context_line":"        temp_id \u003d request.headers.get(\u0027X-Backend-Temp-Id\u0027)"},{"line_number":616,"context_line":"        if temp_id is not None:"},{"line_number":617,"context_line":"            return self._post_temp("},{"line_number":618,"context_line":"                device, disk_file, account, container, obj, policy,"},{"line_number":619,"context_line":"                request, req_timestamp, temp_id)"},{"line_number":620,"context_line":""},{"line_number":621,"context_line":"        try:"}],"source_content_type":"text/x-python","patch_set":32,"id":"3f79a3b5_d89a1161","line":618,"updated":"2018-09-11 18:11:40.000000000","message":"Why not check for the X-Backend-Temp-Id *before* we open the diskfile?  Then we could ditch reparent?","commit_id":"83256db01ae98b902df7691cad155bbdc6024080"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"15157f07ee0caffa1281eb922e8fecac6a7ffd95","unresolved":false,"context_lines":[{"line_number":615,"context_line":"        temp_id \u003d request.headers.get(\u0027X-Backend-Temp-Id\u0027)"},{"line_number":616,"context_line":"        if temp_id is not None:"},{"line_number":617,"context_line":"            return self._post_temp("},{"line_number":618,"context_line":"                device, disk_file, account, container, obj, policy,"},{"line_number":619,"context_line":"                request, req_timestamp, temp_id)"},{"line_number":620,"context_line":""},{"line_number":621,"context_line":"        try:"}],"source_content_type":"text/x-python","patch_set":32,"id":"3f79a3b5_589fd650","line":618,"in_reply_to":"3f79a3b5_d89a1161","updated":"2018-09-11 21:58:05.000000000","message":"Well yea duh. Alasdair made me do it, when he forced these refactored functions. The downside is only when an error happens though, e.g. someone manages to post an ID which will not eventually be found. So, we can fail faster, but maybe it\u0027s not important?","commit_id":"83256db01ae98b902df7691cad155bbdc6024080"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"3a4e11c77137716276fc6c095aa1ad508299c7a0","unresolved":false,"context_lines":[{"line_number":742,"context_line":""},{"line_number":743,"context_line":"    def _post_temp(self, device, disk_file, account, container, obj,"},{"line_number":744,"context_line":"                   policy, request, req_timestamp, temp_id):"},{"line_number":745,"context_line":"        \"\"\"The break-out for 3-stage PUT+POST, Stage 2: apply trailers\"\"\""},{"line_number":746,"context_line":"        self.logger.debug(\u0027POST temp id %s, headers %s\u0027 %"},{"line_number":747,"context_line":"                          (temp_id, dict(request.headers)))"},{"line_number":748,"context_line":"        writer \u003d self.writer_cache.get(temp_id)"}],"source_content_type":"text/x-python","patch_set":32,"id":"3f79a3b5_ae204bf9","line":745,"range":{"start_line":745,"start_character":47,"end_line":745,"end_character":70},"updated":"2018-08-04 00:48:58.000000000","message":"This covers both stage 2 and stage 3, yeah?","commit_id":"83256db01ae98b902df7691cad155bbdc6024080"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"1ed0dd252026e225621493b8cc2d4493c675ee73","unresolved":false,"context_lines":[{"line_number":742,"context_line":""},{"line_number":743,"context_line":"    def _post_temp(self, device, disk_file, account, container, obj,"},{"line_number":744,"context_line":"                   policy, request, req_timestamp, temp_id):"},{"line_number":745,"context_line":"        \"\"\"The break-out for 3-stage PUT+POST, Stage 2: apply trailers\"\"\""},{"line_number":746,"context_line":"        self.logger.debug(\u0027POST temp id %s, headers %s\u0027 %"},{"line_number":747,"context_line":"                          (temp_id, dict(request.headers)))"},{"line_number":748,"context_line":"        writer \u003d self.writer_cache.get(temp_id)"}],"source_content_type":"text/x-python","patch_set":32,"id":"3f79a3b5_e16e38ff","line":745,"range":{"start_line":745,"start_character":47,"end_line":745,"end_character":70},"in_reply_to":"3f79a3b5_ae204bf9","updated":"2018-09-11 21:05:09.000000000","message":"Done.","commit_id":"83256db01ae98b902df7691cad155bbdc6024080"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"ade2f0775d29c70a287a89a39524a57cdc5bb005","unresolved":false,"context_lines":[{"line_number":749,"context_line":"        if writer is None:"},{"line_number":750,"context_line":"            return HTTPBadRequest(body\u003d\u0027Bad Temp-Id\u0027, request\u003drequest)"},{"line_number":751,"context_line":"        writer.reparent(disk_file)"},{"line_number":752,"context_line":"        metadata \u003d writer._staged_metadata"},{"line_number":753,"context_line":""},{"line_number":754,"context_line":"        if config_true_value(request.headers.get(\u0027X-Backend-Finalize\u0027)):"},{"line_number":755,"context_line":"            # this POST may have metadata to add to that sent with the PUT"}],"source_content_type":"text/x-python","patch_set":32,"id":"3f79a3b5_5b159be4","line":752,"updated":"2018-09-11 18:11:40.000000000","message":"XXX We should expand the DiskFile API to \"require\" this sort of access to a \"private\" attribute.","commit_id":"83256db01ae98b902df7691cad155bbdc6024080"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"15157f07ee0caffa1281eb922e8fecac6a7ffd95","unresolved":false,"context_lines":[{"line_number":749,"context_line":"        if writer is None:"},{"line_number":750,"context_line":"            return HTTPBadRequest(body\u003d\u0027Bad Temp-Id\u0027, request\u003drequest)"},{"line_number":751,"context_line":"        writer.reparent(disk_file)"},{"line_number":752,"context_line":"        metadata \u003d writer._staged_metadata"},{"line_number":753,"context_line":""},{"line_number":754,"context_line":"        if config_true_value(request.headers.get(\u0027X-Backend-Finalize\u0027)):"},{"line_number":755,"context_line":"            # this POST may have metadata to add to that sent with the PUT"}],"source_content_type":"text/x-python","patch_set":32,"id":"3f79a3b5_58c83655","line":752,"in_reply_to":"3f79a3b5_5b159be4","updated":"2018-09-11 21:58:05.000000000","message":"Indeed. You\u0027re doing it, right?","commit_id":"83256db01ae98b902df7691cad155bbdc6024080"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"2f7cd3091471486155fa7aecc3fc6d56988f5933","unresolved":false,"context_lines":[{"line_number":752,"context_line":"        metadata \u003d writer._staged_metadata"},{"line_number":753,"context_line":""},{"line_number":754,"context_line":"        if config_true_value(request.headers.get(\u0027X-Backend-Finalize\u0027)):"},{"line_number":755,"context_line":"            # this POST may have metadata to add to that sent with the PUT"},{"line_number":756,"context_line":"            request_etag \u003d request.headers.get(\u0027etag\u0027, \u0027\u0027).lower()"},{"line_number":757,"context_line":"            etag \u003d metadata[\u0027ETag\u0027]"},{"line_number":758,"context_line":"            if request_etag and request_etag !\u003d etag:"}],"source_content_type":"text/x-python","patch_set":32,"id":"5f7c97a3_590f1137","line":755,"updated":"2018-08-02 07:32:10.000000000","message":"# this POST may have extra metadata to add to the metadata that was originally send in the PUT","commit_id":"83256db01ae98b902df7691cad155bbdc6024080"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"5d5354c06967a495b5bff02ba2c161bcab1f1873","unresolved":false,"context_lines":[{"line_number":752,"context_line":"        metadata \u003d writer._staged_metadata"},{"line_number":753,"context_line":""},{"line_number":754,"context_line":"        if config_true_value(request.headers.get(\u0027X-Backend-Finalize\u0027)):"},{"line_number":755,"context_line":"            # this POST may have metadata to add to that sent with the PUT"},{"line_number":756,"context_line":"            request_etag \u003d request.headers.get(\u0027etag\u0027, \u0027\u0027).lower()"},{"line_number":757,"context_line":"            etag \u003d metadata[\u0027ETag\u0027]"},{"line_number":758,"context_line":"            if request_etag and request_etag !\u003d etag:"}],"source_content_type":"text/x-python","patch_set":32,"id":"5f7c97a3_231a0819","line":755,"in_reply_to":"5f7c97a3_590f1137","updated":"2018-08-02 23:19:01.000000000","message":"Done","commit_id":"83256db01ae98b902df7691cad155bbdc6024080"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"ade2f0775d29c70a287a89a39524a57cdc5bb005","unresolved":false,"context_lines":[{"line_number":971,"context_line":"                                               request.headers,"},{"line_number":972,"context_line":"                                               footer_meta)"},{"line_number":973,"context_line":"                if temp_stage:"},{"line_number":974,"context_line":"                    writer.stage(metadata)"},{"line_number":975,"context_line":"                    writer.orphan()"},{"line_number":976,"context_line":"                    temp_id \u003d self.writer_cache.save(writer)"},{"line_number":977,"context_line":"                else:"}],"source_content_type":"text/x-python","patch_set":32,"id":"3f79a3b5_9bf253e3","line":974,"updated":"2018-09-11 18:11:40.000000000","message":"why is this metadata... \"staged\" - isn\u0027t it just \"the metadata we have so far\"?  How do we combine it with update later?","commit_id":"83256db01ae98b902df7691cad155bbdc6024080"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"15157f07ee0caffa1281eb922e8fecac6a7ffd95","unresolved":false,"context_lines":[{"line_number":971,"context_line":"                                               request.headers,"},{"line_number":972,"context_line":"                                               footer_meta)"},{"line_number":973,"context_line":"                if temp_stage:"},{"line_number":974,"context_line":"                    writer.stage(metadata)"},{"line_number":975,"context_line":"                    writer.orphan()"},{"line_number":976,"context_line":"                    temp_id \u003d self.writer_cache.save(writer)"},{"line_number":977,"context_line":"                else:"}],"source_content_type":"text/x-python","patch_set":32,"id":"3f79a3b5_7838923a","line":974,"in_reply_to":"3f79a3b5_9bf253e3","updated":"2018-09-11 21:58:05.000000000","message":"\"stage data with given metadata (mostly Etag)\". When we combine though, the newer one wins, including any etags...","commit_id":"83256db01ae98b902df7691cad155bbdc6024080"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"ade2f0775d29c70a287a89a39524a57cdc5bb005","unresolved":false,"context_lines":[{"line_number":972,"context_line":"                                               footer_meta)"},{"line_number":973,"context_line":"                if temp_stage:"},{"line_number":974,"context_line":"                    writer.stage(metadata)"},{"line_number":975,"context_line":"                    writer.orphan()"},{"line_number":976,"context_line":"                    temp_id \u003d self.writer_cache.save(writer)"},{"line_number":977,"context_line":"                else:"},{"line_number":978,"context_line":"                    writer.put(metadata)"}],"source_content_type":"text/x-python","patch_set":32,"id":"3f79a3b5_b8e395d8","line":975,"updated":"2018-09-11 18:11:40.000000000","message":"What is the goal of calling orphan here really?  Is the DiskFile holding open some reference we need to release to the GC?","commit_id":"83256db01ae98b902df7691cad155bbdc6024080"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"15157f07ee0caffa1281eb922e8fecac6a7ffd95","unresolved":false,"context_lines":[{"line_number":972,"context_line":"                                               footer_meta)"},{"line_number":973,"context_line":"                if temp_stage:"},{"line_number":974,"context_line":"                    writer.stage(metadata)"},{"line_number":975,"context_line":"                    writer.orphan()"},{"line_number":976,"context_line":"                    temp_id \u003d self.writer_cache.save(writer)"},{"line_number":977,"context_line":"                else:"},{"line_number":978,"context_line":"                    writer.put(metadata)"}],"source_content_type":"text/x-python","patch_set":32,"id":"3f79a3b5_188f1ece","line":975,"in_reply_to":"3f79a3b5_b8e395d8","updated":"2018-09-11 21:58:05.000000000","message":"Mostly taking dishes to the counter here. I didn\u0027t have a specific issue with hanging references, but I don\u0027t know what\u0027s hiding in depths of GC. Normally DiskFile is not hanging along, so I didn\u0027t want to pin something bad by accident for 100 seconds. You can say I have no idea what I\u0027m doing and this is safer.","commit_id":"83256db01ae98b902df7691cad155bbdc6024080"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a56346d93124dea05e7aded037ac4fce3fad9eb3","unresolved":false,"context_lines":[{"line_number":1015,"context_line":"            if orig_delete_at:"},{"line_number":1016,"context_line":"                self.delete_at_update("},{"line_number":1017,"context_line":"                    \u0027DELETE\u0027, orig_delete_at, account, container, obj,"},{"line_number":1018,"context_line":"                    request, device, policy)"},{"line_number":1019,"context_line":""},{"line_number":1020,"context_line":"        if temp_id:"},{"line_number":1021,"context_line":"            return HTTPCreated(request\u003drequest,"}],"source_content_type":"text/x-python","patch_set":32,"id":"3f79a3b5_26ae982f","line":1018,"updated":"2018-09-10 22:38:27.000000000","message":"I don\u0027t think we should let expir-ery updates go out until we  actually have a successful PUT","commit_id":"83256db01ae98b902df7691cad155bbdc6024080"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"1ed0dd252026e225621493b8cc2d4493c675ee73","unresolved":false,"context_lines":[{"line_number":1015,"context_line":"            if orig_delete_at:"},{"line_number":1016,"context_line":"                self.delete_at_update("},{"line_number":1017,"context_line":"                    \u0027DELETE\u0027, orig_delete_at, account, container, obj,"},{"line_number":1018,"context_line":"                    request, device, policy)"},{"line_number":1019,"context_line":""},{"line_number":1020,"context_line":"        if temp_id:"},{"line_number":1021,"context_line":"            return HTTPCreated(request\u003drequest,"}],"source_content_type":"text/x-python","patch_set":32,"id":"3f79a3b5_bfcdb4b8","line":1018,"in_reply_to":"3f79a3b5_26ae982f","updated":"2018-09-11 21:05:09.000000000","message":"Done (by bailing a bit earlier)","commit_id":"83256db01ae98b902df7691cad155bbdc6024080"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a56346d93124dea05e7aded037ac4fce3fad9eb3","unresolved":false,"context_lines":[{"line_number":1018,"context_line":"                    request, device, policy)"},{"line_number":1019,"context_line":""},{"line_number":1020,"context_line":"        if temp_id:"},{"line_number":1021,"context_line":"            return HTTPCreated(request\u003drequest,"},{"line_number":1022,"context_line":"                               headers\u003d{\u0027X-Backend-Temp-Id\u0027: temp_id})"},{"line_number":1023,"context_line":""},{"line_number":1024,"context_line":"        self.container_update("}],"source_content_type":"text/x-python","patch_set":32,"id":"3f79a3b5_89b3ffe9","line":1021,"updated":"2018-09-10 22:38:27.000000000","message":"this should probably be \"Accepted\"","commit_id":"83256db01ae98b902df7691cad155bbdc6024080"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"1ed0dd252026e225621493b8cc2d4493c675ee73","unresolved":false,"context_lines":[{"line_number":1018,"context_line":"                    request, device, policy)"},{"line_number":1019,"context_line":""},{"line_number":1020,"context_line":"        if temp_id:"},{"line_number":1021,"context_line":"            return HTTPCreated(request\u003drequest,"},{"line_number":1022,"context_line":"                               headers\u003d{\u0027X-Backend-Temp-Id\u0027: temp_id})"},{"line_number":1023,"context_line":""},{"line_number":1024,"context_line":"        self.container_update("}],"source_content_type":"text/x-python","patch_set":32,"id":"3f79a3b5_0781044e","line":1021,"in_reply_to":"3f79a3b5_89b3ffe9","updated":"2018-09-11 21:05:09.000000000","message":"Okay, I changed it. I have no preference.","commit_id":"83256db01ae98b902df7691cad155bbdc6024080"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a56346d93124dea05e7aded037ac4fce3fad9eb3","unresolved":false,"context_lines":[{"line_number":1266,"context_line":"    def __call__(self, env, start_response):"},{"line_number":1267,"context_line":"        \"\"\"WSGI Application entry point for the Swift Object Server.\"\"\""},{"line_number":1268,"context_line":"        start_time \u003d time.time()"},{"line_number":1269,"context_line":"        self.writer_cache.expire(start_time)"},{"line_number":1270,"context_line":"        req \u003d Request(env)"},{"line_number":1271,"context_line":"        self.logger.txn_id \u003d req.headers.get(\u0027x-trans-id\u0027, None)"},{"line_number":1272,"context_line":""}],"source_content_type":"text/x-python","patch_set":32,"id":"3f79a3b5_263bf839","line":1269,"updated":"2018-09-10 22:38:27.000000000","message":"it would be better to have a background coro do this on some sort of period - as is this can hold cache arbitrarily long or call it it extremely frequently (every REPLICATE)","commit_id":"83256db01ae98b902df7691cad155bbdc6024080"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"1ed0dd252026e225621493b8cc2d4493c675ee73","unresolved":false,"context_lines":[{"line_number":1266,"context_line":"    def __call__(self, env, start_response):"},{"line_number":1267,"context_line":"        \"\"\"WSGI Application entry point for the Swift Object Server.\"\"\""},{"line_number":1268,"context_line":"        start_time \u003d time.time()"},{"line_number":1269,"context_line":"        self.writer_cache.expire(start_time)"},{"line_number":1270,"context_line":"        req \u003d Request(env)"},{"line_number":1271,"context_line":"        self.logger.txn_id \u003d req.headers.get(\u0027x-trans-id\u0027, None)"},{"line_number":1272,"context_line":""}],"source_content_type":"text/x-python","patch_set":32,"id":"3f79a3b5_c17f7cdd","line":1269,"in_reply_to":"3f79a3b5_263bf839","updated":"2018-09-11 21:05:09.000000000","message":"My suggestion about holding long was to always process no more than 1. See, even if all of requests are staging PUTs, the cache will not grow in that case. Lemme fix this, I just didn\u0027t want to obscure what\u0027s going on.","commit_id":"83256db01ae98b902df7691cad155bbdc6024080"}],"swift/obj/ssync_receiver.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"cf5190bda49a26788552ee732aefcf3d11eb261a","unresolved":false,"context_lines":[{"line_number":280,"context_line":"            # durable now. Try this just once to avoid looping if it fails."},{"line_number":281,"context_line":"            try:"},{"line_number":282,"context_line":"                writer \u003d df.create()"},{"line_number":283,"context_line":"                with contextlib.closing(writer):"},{"line_number":284,"context_line":"                    writer.commit(remote[\u0027ts_data\u0027])"},{"line_number":285,"context_line":"                return self._check_local(remote, make_durable\u003dFalse)"},{"line_number":286,"context_line":"            except Exception:"}],"source_content_type":"text/x-python","patch_set":25,"id":"1f9dbf25_e7763207","line":283,"updated":"2018-03-03 01:37:58.000000000","message":"Why was this necessary? What\u0027s the difference between\n\n close(writer)\n\nand the finally block on master around https://github.com/openstack/swift/blob/2.17.0/swift/obj/diskfile.py#L2660-L2675 ?","commit_id":"79fec861768bb219815e6cab80d3b1320e393b1e"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"2916044a01da4770a045ad09d15287c44263d3c1","unresolved":false,"context_lines":[{"line_number":280,"context_line":"            # durable now. Try this just once to avoid looping if it fails."},{"line_number":281,"context_line":"            try:"},{"line_number":282,"context_line":"                writer \u003d df.create()"},{"line_number":283,"context_line":"                with contextlib.closing(writer):"},{"line_number":284,"context_line":"                    writer.commit(remote[\u0027ts_data\u0027])"},{"line_number":285,"context_line":"                return self._check_local(remote, make_durable\u003dFalse)"},{"line_number":286,"context_line":"            except Exception:"}],"source_content_type":"text/x-python","patch_set":25,"id":"df7087c5_116ee000","line":283,"in_reply_to":"1f9dbf25_e7763207","updated":"2018-03-12 04:07:16.000000000","message":"Okay, let me answer this without punting it onto Sam completely, since I own the issue now. His original assertion from patch 509607 was that getting rid of yield (iterator) was necessary to have a class that can stick around open. If we take it as true, then we have to replace dogmatically all places where the \"with\" is used, including this one.\n\nAs for the difference, well, there\u0027s not much, except the yield.\n\nhttps://review.openstack.org/509607","commit_id":"79fec861768bb219815e6cab80d3b1320e393b1e"}],"swift/proxy/controllers/obj.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"cebc4f2009f5c91a67d19103054b32c6c1ae8589","unresolved":false,"context_lines":[{"line_number":1857,"context_line":""},{"line_number":1858,"context_line":"        headers \u003d self.footer_meta"},{"line_number":1859,"context_line":"        headers[\u0027Content-Length\u0027] \u003d \u00270\u0027"},{"line_number":1860,"context_line":"        self.conn.putrequest(\u0027POST\u0027, self.path, skip_host\u003d(\u0027Host\u0027 in headers))"},{"line_number":1861,"context_line":"        for header, value in headers.items():"},{"line_number":1862,"context_line":"            self.conn.putheader(header, str(value))"},{"line_number":1863,"context_line":"        self.conn.endheaders()"}],"source_content_type":"text/x-python","patch_set":1,"id":"3a461143_6f5d1048","line":1860,"updated":"2017-02-02 03:50:09.000000000","message":"what *is* it about this POST that makes it a commit with footers instead of just a regular old metadata update?","commit_id":"442f483da525b9f7c9f6bdd35f412a1fb92c8974"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"8898ed7e81094158d38b5056b6697345f6712ea5","unresolved":false,"context_lines":[{"line_number":1813,"context_line":"        # Normally it\u0027s bad to post with the same timestamp as the object,"},{"line_number":1814,"context_line":"        # since it confuses the replication, but we\u0027re posting a non-durable"},{"line_number":1815,"context_line":"        # object, and such objects are ignored by reconstructor (are they?"},{"line_number":1816,"context_line":"        # XXX prove that it\u0027s safe)"},{"line_number":1817,"context_line":"        headers[\u0027X-Timestamp\u0027] \u003d self.nheaders[\u0027X-Timestamp\u0027]"},{"line_number":1818,"context_line":"        # Not sure if we want both headers, they are never separate. XXX"},{"line_number":1819,"context_line":"        headers[\u0027X-Backend-Post-To-Durable\u0027] \u003d \u0027true\u0027"}],"source_content_type":"text/x-python","patch_set":8,"id":"5ff73747_9a4fdc91","line":1816,"updated":"2017-05-02 17:16:37.000000000","message":"I thought the reconstructor might look to neighbors and make non-durables durable (at least, under some conditions)...","commit_id":"78f47b42db89987c611761d87ef3ff2e2c28b491"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"9c98977a75d350825c281216b4e93da4986b1b00","unresolved":false,"context_lines":[{"line_number":1813,"context_line":"        # Normally it\u0027s bad to post with the same timestamp as the object,"},{"line_number":1814,"context_line":"        # since it confuses the replication, but we\u0027re posting a non-durable"},{"line_number":1815,"context_line":"        # object, and such objects are ignored by reconstructor (are they?"},{"line_number":1816,"context_line":"        # XXX prove that it\u0027s safe)"},{"line_number":1817,"context_line":"        headers[\u0027X-Timestamp\u0027] \u003d self.nheaders[\u0027X-Timestamp\u0027]"},{"line_number":1818,"context_line":"        # Not sure if we want both headers, they are never separate. XXX"},{"line_number":1819,"context_line":"        headers[\u0027X-Backend-Post-To-Durable\u0027] \u003d \u0027true\u0027"}],"source_content_type":"text/x-python","patch_set":8,"id":"3f044301_59d044ca","line":1816,"in_reply_to":"5ff73747_9a4fdc91","updated":"2017-05-04 03:29:09.000000000","message":"I removed this. The comment was about the original code that made a half-baked object visible after PUT. With the temporary file, this XXX item is no longer pertinent.","commit_id":"78f47b42db89987c611761d87ef3ff2e2c28b491"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"46c4035a8817184b73b4a0e75e6fd2c9fdbb0af5","unresolved":false,"context_lines":[{"line_number":396,"context_line":"                    {\u0027status\u0027: response.status,"},{"line_number":397,"context_line":"                     \u0027body\u0027: body[:1024], \u0027path\u0027: req.path})"},{"line_number":398,"context_line":"            elif is_success(response.status):"},{"line_number":399,"context_line":"                etags.add(response.getheader(\u0027etag\u0027).strip(\u0027\"\u0027))"},{"line_number":400,"context_line":""},{"line_number":401,"context_line":"        for (putter, response) in pile:"},{"line_number":402,"context_line":"            if response:"}],"source_content_type":"text/x-python","patch_set":19,"id":"5f4e5783_9e244dc8","line":399,"updated":"2017-10-11 01:39:37.000000000","message":"I think there\u0027s some contexts (e.g. a POST 202 response) where we might hit this branch and find a response with no etag...","commit_id":"69ef1ba4da42fb6b3d2a6feca8ec39215db6b0a9"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"bd5632b6780c1c4a23977eda420215ff05a35de6","unresolved":false,"context_lines":[{"line_number":396,"context_line":"                    {\u0027status\u0027: response.status,"},{"line_number":397,"context_line":"                     \u0027body\u0027: body[:1024], \u0027path\u0027: req.path})"},{"line_number":398,"context_line":"            elif is_success(response.status):"},{"line_number":399,"context_line":"                etags.add(response.getheader(\u0027etag\u0027).strip(\u0027\"\u0027))"},{"line_number":400,"context_line":""},{"line_number":401,"context_line":"        for (putter, response) in pile:"},{"line_number":402,"context_line":"            if response:"}],"source_content_type":"text/x-python","patch_set":19,"id":"3f4b6375_0c4722bf","line":399,"in_reply_to":"5f4e5783_9e244dc8","updated":"2017-10-30 03:17:33.000000000","message":"It does not seem to hurt anything in the 3rd phase (commit), but I added etag there anyway. You can see that we discard those empty etags later (where it\u0027s called _etags). In the 2nd phase (footer meta), I made sure to have etags.","commit_id":"69ef1ba4da42fb6b3d2a6feca8ec39215db6b0a9"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"46c4035a8817184b73b4a0e75e6fd2c9fdbb0af5","unresolved":false,"context_lines":[{"line_number":1833,"context_line":"    def __init__(self, conn, node, resp, path, connect_duration,"},{"line_number":1834,"context_line":"                 logger, headers, node_timeout\u003d60.0):"},{"line_number":1835,"context_line":"        super(DoublePutter, self).__init__(conn, node, resp, path,"},{"line_number":1836,"context_line":"                                           connect_duration, logger)"},{"line_number":1837,"context_line":"        self.nheaders \u003d headers"},{"line_number":1838,"context_line":"        self.node_timeout \u003d node_timeout"},{"line_number":1839,"context_line":""}],"source_content_type":"text/x-python","patch_set":19,"id":"5f4e5783_fe99011f","line":1836,"updated":"2017-10-11 01:39:37.000000000","message":"we probably ought plumb the chunked kwarg","commit_id":"69ef1ba4da42fb6b3d2a6feca8ec39215db6b0a9"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"bd5632b6780c1c4a23977eda420215ff05a35de6","unresolved":false,"context_lines":[{"line_number":1833,"context_line":"    def __init__(self, conn, node, resp, path, connect_duration,"},{"line_number":1834,"context_line":"                 logger, headers, node_timeout\u003d60.0):"},{"line_number":1835,"context_line":"        super(DoublePutter, self).__init__(conn, node, resp, path,"},{"line_number":1836,"context_line":"                                           connect_duration, logger)"},{"line_number":1837,"context_line":"        self.nheaders \u003d headers"},{"line_number":1838,"context_line":"        self.node_timeout \u003d node_timeout"},{"line_number":1839,"context_line":""}],"source_content_type":"text/x-python","patch_set":19,"id":"3f4b6375_ec43a6ae","line":1836,"in_reply_to":"5f4e5783_fe99011f","updated":"2017-10-30 03:17:33.000000000","message":"Done","commit_id":"69ef1ba4da42fb6b3d2a6feca8ec39215db6b0a9"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"46c4035a8817184b73b4a0e75e6fd2c9fdbb0af5","unresolved":false,"context_lines":[{"line_number":1927,"context_line":"                  not supported by the node."},{"line_number":1928,"context_line":"        \"\"\""},{"line_number":1929,"context_line":"        headers \u003d HeaderKeyDict(headers_)"},{"line_number":1930,"context_line":"        headers[\u0027Expect\u0027] \u003d \u0027100-continue\u0027"},{"line_number":1931,"context_line":""},{"line_number":1932,"context_line":"        # XXX It may be more proper to use a query parameter. A query is seen"},{"line_number":1933,"context_line":"        # in the logs and possibly violates ReST model less. But it\u0027s easier"}],"source_content_type":"text/x-python","patch_set":19,"id":"5f4e5783_7e52b169","line":1930,"updated":"2017-10-11 01:39:37.000000000","message":"seems like somewhere around here we need to either content-length or transfer-encoding: chunked","commit_id":"69ef1ba4da42fb6b3d2a6feca8ec39215db6b0a9"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"bd5632b6780c1c4a23977eda420215ff05a35de6","unresolved":false,"context_lines":[{"line_number":1927,"context_line":"                  not supported by the node."},{"line_number":1928,"context_line":"        \"\"\""},{"line_number":1929,"context_line":"        headers \u003d HeaderKeyDict(headers_)"},{"line_number":1930,"context_line":"        headers[\u0027Expect\u0027] \u003d \u0027100-continue\u0027"},{"line_number":1931,"context_line":""},{"line_number":1932,"context_line":"        # XXX It may be more proper to use a query parameter. A query is seen"},{"line_number":1933,"context_line":"        # in the logs and possibly violates ReST model less. But it\u0027s easier"}],"source_content_type":"text/x-python","patch_set":19,"id":"3f4b6375_2c23fe07","line":1930,"in_reply_to":"5f4e5783_7e52b169","updated":"2017-10-30 03:17:33.000000000","message":"I think I addressed this problem by plumping the length down into putter. I think there\u0027s no ambiguity anymore.","commit_id":"69ef1ba4da42fb6b3d2a6feca8ec39215db6b0a9"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"46c4035a8817184b73b4a0e75e6fd2c9fdbb0af5","unresolved":false,"context_lines":[{"line_number":2737,"context_line":"                    raise HTTPServiceUnavailable(request\u003dreq)"},{"line_number":2738,"context_line":""},{"line_number":2739,"context_line":"                elif not self._have_adequate_informational("},{"line_number":2740,"context_line":"                        statuses, min_conns):"},{"line_number":2741,"context_line":"                    resp \u003d self.best_response(req, statuses, reasons, bodies,"},{"line_number":2742,"context_line":"                                              _(\u0027Object PUT\u0027),"},{"line_number":2743,"context_line":"                                              quorum_size\u003dmin_conns)"}],"source_content_type":"text/x-python","patch_set":19,"id":"5f4e5783_32ab4d1e","line":2740,"updated":"2017-10-11 01:39:37.000000000","message":"I think after the metadata POST response these statuses are 202 not 100 continue","commit_id":"69ef1ba4da42fb6b3d2a6feca8ec39215db6b0a9"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"bd5632b6780c1c4a23977eda420215ff05a35de6","unresolved":false,"context_lines":[{"line_number":2737,"context_line":"                    raise HTTPServiceUnavailable(request\u003dreq)"},{"line_number":2738,"context_line":""},{"line_number":2739,"context_line":"                elif not self._have_adequate_informational("},{"line_number":2740,"context_line":"                        statuses, min_conns):"},{"line_number":2741,"context_line":"                    resp \u003d self.best_response(req, statuses, reasons, bodies,"},{"line_number":2742,"context_line":"                                              _(\u0027Object PUT\u0027),"},{"line_number":2743,"context_line":"                                              quorum_size\u003dmin_conns)"}],"source_content_type":"text/x-python","patch_set":19,"id":"3f4b6375_0c2042f7","line":2740,"in_reply_to":"5f4e5783_32ab4d1e","updated":"2017-10-30 03:17:33.000000000","message":"Done","commit_id":"69ef1ba4da42fb6b3d2a6feca8ec39215db6b0a9"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"46c4035a8817184b73b4a0e75e6fd2c9fdbb0af5","unresolved":false,"context_lines":[{"line_number":2750,"context_line":"                        raise HTTPServiceUnavailable(request\u003dreq)"},{"line_number":2751,"context_line":"                    else:"},{"line_number":2752,"context_line":"                        # Other errors should use raw best_response"},{"line_number":2753,"context_line":"                        raise resp"},{"line_number":2754,"context_line":""},{"line_number":2755,"context_line":"                # quorum achieved, start 2nd phase - send commit"},{"line_number":2756,"context_line":"                # confirmation to participating object servers"}],"source_content_type":"text/x-python","patch_set":19,"id":"5f4e5783_12a8511d","line":2753,"updated":"2017-10-11 01:39:37.000000000","message":"basically we raise a 200 here...","commit_id":"69ef1ba4da42fb6b3d2a6feca8ec39215db6b0a9"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"46c4035a8817184b73b4a0e75e6fd2c9fdbb0af5","unresolved":false,"context_lines":[{"line_number":2757,"context_line":"                # so they write a .durable state file indicating"},{"line_number":2758,"context_line":"                # a successful PUT"},{"line_number":2759,"context_line":"                for putter in putters:"},{"line_number":2760,"context_line":"                    putter.send_commit_confirmation()"},{"line_number":2761,"context_line":"                for putter in putters:"},{"line_number":2762,"context_line":"                    putter.wait()"},{"line_number":2763,"context_line":"        except ChunkReadTimeout as err:"}],"source_content_type":"text/x-python","patch_set":19,"id":"5f4e5783_f2625516","line":2760,"updated":"2017-10-11 01:39:37.000000000","message":"... instead of sending the third POST","commit_id":"69ef1ba4da42fb6b3d2a6feca8ec39215db6b0a9"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"c842cb5ed37538269282bb08f3652c0bc749970a","unresolved":false,"context_lines":[{"line_number":1946,"context_line":"        headers[\u0027X-Backend-Temp-Id\u0027] \u003d temp_id"},{"line_number":1947,"context_line":"        self.conn.putrequest(\u0027POST\u0027, self.conn.path,"},{"line_number":1948,"context_line":"                             skip_host\u003d(\u0027Host\u0027 in headers))"},{"line_number":1949,"context_line":"        for header, value in headers.items():"},{"line_number":1950,"context_line":"            self.conn.putheader(header, str(value))"},{"line_number":1951,"context_line":"        self.conn.endheaders()"},{"line_number":1952,"context_line":""},{"line_number":1953,"context_line":"        self.state \u003d DATA_SENT"}],"source_content_type":"text/x-python","patch_set":27,"id":"5f7c97a3_58ece191","line":1950,"range":{"start_line":1949,"start_character":8,"end_line":1950,"end_character":51},"updated":"2018-06-14 17:43:39.000000000","message":"the MIMEPutter sends a header with the md5 of the \u0027footers\u0027. IDK if we need to do similar here? there\u0027s no hash sent for the regular PUT headers.","commit_id":"061690bbc467be6a017aaddd5c3cc79632ec55b8"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"c842cb5ed37538269282bb08f3652c0bc749970a","unresolved":false,"context_lines":[{"line_number":2608,"context_line":"                \u0027X-Object-Sysmeta-Ec-Content-Length\u0027]"},{"line_number":2609,"context_line":""},{"line_number":2610,"context_line":"    def _make_putter(self, node, part, req, length, headers):"},{"line_number":2611,"context_line":"        if self.app.use_put_v1:"},{"line_number":2612,"context_line":"            return DoublePutter.connect("},{"line_number":2613,"context_line":"                node, part, req.swift_entity_path, length, headers,"},{"line_number":2614,"context_line":"                self.app.conn_timeout, self.app.node_timeout,"}],"source_content_type":"text/x-python","patch_set":27,"id":"5f7c97a3_9bb4b338","line":2611,"range":{"start_line":2611,"start_character":8,"end_line":2611,"end_character":30},"updated":"2018-06-14 17:43:39.000000000","message":"It would be useful for DoublePutter to be the default, at least during development, so that the functional tests exercise the new path","commit_id":"061690bbc467be6a017aaddd5c3cc79632ec55b8"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"adc80765639e411e7ccdc8cd708fc549e0803eee","unresolved":false,"context_lines":[{"line_number":1976,"context_line":""},{"line_number":1977,"context_line":"        headers \u003d footer_metadata"},{"line_number":1978,"context_line":"        headers[\u0027Content-Length\u0027] \u003d \u00270\u0027"},{"line_number":1979,"context_line":"        for k in [\u0027X-Backend-Storage-Policy-Index\u0027, \u0027Referrer\u0027, \u0027User-Agent\u0027]:"},{"line_number":1980,"context_line":"            if k in self.nheaders:"},{"line_number":1981,"context_line":"                headers[k] \u003d self.nheaders[k]"},{"line_number":1982,"context_line":"        # Normally it\u0027s bad to post with the same timestamp as the object,"}],"source_content_type":"text/x-python","patch_set":28,"id":"5f7c97a3_1434a4de","line":1979,"range":{"start_line":1979,"start_character":53,"end_line":1979,"end_character":61},"updated":"2018-06-19 14:57:43.000000000","message":"should be (mis)spelt \u0027Referer\u0027 - the HTTP header has always been misspelt\n\nhttps://www.w3.org/Protocols/HTTP/HTRQ_Headers.html#z14","commit_id":"46ee580e707c3b8bcdcd5de49391f025a2016433"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"adc80765639e411e7ccdc8cd708fc549e0803eee","unresolved":false,"context_lines":[{"line_number":1986,"context_line":"        headers[\u0027X-Backend-Temp-Id\u0027] \u003d temp_id"},{"line_number":1987,"context_line":"        self.conn.putrequest(\u0027POST\u0027, self.conn.path,"},{"line_number":1988,"context_line":"                             skip_host\u003d(\u0027Host\u0027 in headers))"},{"line_number":1989,"context_line":"        for header, value in headers.items():"},{"line_number":1990,"context_line":"            self.conn.putheader(header, str(value))"},{"line_number":1991,"context_line":"        self.conn.endheaders()"},{"line_number":1992,"context_line":""}],"source_content_type":"text/x-python","patch_set":28,"id":"5f7c97a3_7c1e3bbc","line":1989,"range":{"start_line":1989,"start_character":29,"end_line":1989,"end_character":36},"updated":"2018-06-19 14:57:43.000000000","message":"these headers do not container the X-Container-* headers to direct a container update, so no contaienr updates are being issued from the object server.","commit_id":"46ee580e707c3b8bcdcd5de49391f025a2016433"},{"author":{"_account_id":9625,"name":"Thiago da Silva","email":"thiagodasilva@gmail.com","username":"thiago"},"change_message_id":"04a3c3dd769d7521372336fffcda15b841e5e8f5","unresolved":false,"context_lines":[{"line_number":1934,"context_line":"                   mime_boundary, multiphase\u003dneed_multiphase)"},{"line_number":1935,"context_line":""},{"line_number":1936,"context_line":""},{"line_number":1937,"context_line":"# Comments throughout the code refers to this as a \"2-phase\" scheme."},{"line_number":1938,"context_line":"# In reality, it involves 3 phases, not 2: data, metadata, commit."},{"line_number":1939,"context_line":"class DoublePutter(Putter):"},{"line_number":1940,"context_line":"    \"\"\""}],"source_content_type":"text/x-python","patch_set":32,"id":"5f7c97a3_f300e3b3","line":1937,"range":{"start_line":1937,"start_character":2,"end_line":1937,"end_character":67},"updated":"2018-08-02 20:50:49.000000000","message":"why not fix this?","commit_id":"83256db01ae98b902df7691cad155bbdc6024080"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"3a4e11c77137716276fc6c095aa1ad508299c7a0","unresolved":false,"context_lines":[{"line_number":1934,"context_line":"                   mime_boundary, multiphase\u003dneed_multiphase)"},{"line_number":1935,"context_line":""},{"line_number":1936,"context_line":""},{"line_number":1937,"context_line":"# Comments throughout the code refers to this as a \"2-phase\" scheme."},{"line_number":1938,"context_line":"# In reality, it involves 3 phases, not 2: data, metadata, commit."},{"line_number":1939,"context_line":"class DoublePutter(Putter):"},{"line_number":1940,"context_line":"    \"\"\""}],"source_content_type":"text/x-python","patch_set":32,"id":"3f79a3b5_2eb75b89","line":1937,"range":{"start_line":1937,"start_character":2,"end_line":1937,"end_character":67},"in_reply_to":"5f7c97a3_f300e3b3","updated":"2018-08-04 00:48:58.000000000","message":"Well... sometimes it *is* two-phase. Or, the second request combines phases two and three.","commit_id":"83256db01ae98b902df7691cad155bbdc6024080"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"c4b0816434ae5f4b7237d7f53180dfcc86190c68","unresolved":false,"context_lines":[{"line_number":1953,"context_line":"    def _send_post_to_temp(self, finalize, commit, extra_headers\u003dNone):"},{"line_number":1954,"context_line":"        # Normally it\u0027s bad to post with the same timestamp as the object,"},{"line_number":1955,"context_line":"        # since it confuses the replication, but we\u0027re posting to an invisible"},{"line_number":1956,"context_line":"        # object with Temp-ID, so it\u0027s safe."},{"line_number":1957,"context_line":"        headers \u003d HeaderKeyDict()"},{"line_number":1958,"context_line":"        headers_to_copy \u003d [\u0027X-Backend-Storage-Policy-Index\u0027, \u0027Referer\u0027,"},{"line_number":1959,"context_line":"                           \u0027User-Agent\u0027, \u0027X-Timestamp\u0027]"}],"source_content_type":"text/x-python","patch_set":32,"id":"3f79a3b5_c9a713bd","line":1956,"updated":"2018-09-10 15:51:15.000000000","message":"And importantly, we aren\u0027t generating any .meta file -- it\u0027s all going into the .data","commit_id":"83256db01ae98b902df7691cad155bbdc6024080"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"75867a9c210c01e910797df485d336bb97fd2337","unresolved":false,"context_lines":[{"line_number":1953,"context_line":"    def _send_post_to_temp(self, finalize, commit, extra_headers\u003dNone):"},{"line_number":1954,"context_line":"        # Normally it\u0027s bad to post with the same timestamp as the object,"},{"line_number":1955,"context_line":"        # since it confuses the replication, but we\u0027re posting to an invisible"},{"line_number":1956,"context_line":"        # object with Temp-ID, so it\u0027s safe."},{"line_number":1957,"context_line":"        headers \u003d HeaderKeyDict()"},{"line_number":1958,"context_line":"        headers_to_copy \u003d [\u0027X-Backend-Storage-Policy-Index\u0027, \u0027Referer\u0027,"},{"line_number":1959,"context_line":"                           \u0027User-Agent\u0027, \u0027X-Timestamp\u0027]"}],"source_content_type":"text/x-python","patch_set":32,"id":"3f79a3b5_86b3ece9","line":1956,"in_reply_to":"3f79a3b5_c9a713bd","updated":"2018-09-10 22:56:26.000000000","message":"I don\u0027t understand what you\u0027re implying here. I meant that if the post only reaches some objects, one otherwise might end with some of them having new meta but same timestamp, which would be confusing the replication. It would still be bad even if we don\u0027t write any meta files... I think.\n\nSorry if I\u0027m stupid here, but how would you amend the comment?","commit_id":"83256db01ae98b902df7691cad155bbdc6024080"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"2cf6bb3adb999e06dc371db9111998134a1661f7","unresolved":false,"context_lines":[{"line_number":1956,"context_line":"        # object with Temp-ID, so it\u0027s safe."},{"line_number":1957,"context_line":"        headers \u003d HeaderKeyDict()"},{"line_number":1958,"context_line":"        headers_to_copy \u003d [\u0027X-Backend-Storage-Policy-Index\u0027, \u0027Referer\u0027,"},{"line_number":1959,"context_line":"                           \u0027User-Agent\u0027, \u0027X-Timestamp\u0027]"},{"line_number":1960,"context_line":"        if commit:"},{"line_number":1961,"context_line":"            # copy any container update routing headers from original PUT"},{"line_number":1962,"context_line":"            headers_to_copy.extend([\u0027X-Container-Host\u0027, \u0027X-Container-Device\u0027,"}],"source_content_type":"text/x-python","patch_set":32,"id":"3f79a3b5_3eecf0f1","line":1959,"updated":"2018-08-10 01:37:12.000000000","message":"Add \u0027X-Trans-Id\u0027 to this list to pass the transaction id along. Tested this locally. http://paste.openstack.org/show/727783/","commit_id":"83256db01ae98b902df7691cad155bbdc6024080"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"75867a9c210c01e910797df485d336bb97fd2337","unresolved":false,"context_lines":[{"line_number":1956,"context_line":"        # object with Temp-ID, so it\u0027s safe."},{"line_number":1957,"context_line":"        headers \u003d HeaderKeyDict()"},{"line_number":1958,"context_line":"        headers_to_copy \u003d [\u0027X-Backend-Storage-Policy-Index\u0027, \u0027Referer\u0027,"},{"line_number":1959,"context_line":"                           \u0027User-Agent\u0027, \u0027X-Timestamp\u0027]"},{"line_number":1960,"context_line":"        if commit:"},{"line_number":1961,"context_line":"            # copy any container update routing headers from original PUT"},{"line_number":1962,"context_line":"            headers_to_copy.extend([\u0027X-Container-Host\u0027, \u0027X-Container-Device\u0027,"}],"source_content_type":"text/x-python","patch_set":32,"id":"3f79a3b5_6956a3d1","line":1959,"in_reply_to":"3f79a3b5_3eecf0f1","updated":"2018-09-10 22:56:26.000000000","message":"Done but I need you to verify.","commit_id":"83256db01ae98b902df7691cad155bbdc6024080"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a56346d93124dea05e7aded037ac4fce3fad9eb3","unresolved":false,"context_lines":[{"line_number":1956,"context_line":"        # object with Temp-ID, so it\u0027s safe."},{"line_number":1957,"context_line":"        headers \u003d HeaderKeyDict()"},{"line_number":1958,"context_line":"        headers_to_copy \u003d [\u0027X-Backend-Storage-Policy-Index\u0027, \u0027Referer\u0027,"},{"line_number":1959,"context_line":"                           \u0027User-Agent\u0027, \u0027X-Timestamp\u0027]"},{"line_number":1960,"context_line":"        if commit:"},{"line_number":1961,"context_line":"            # copy any container update routing headers from original PUT"},{"line_number":1962,"context_line":"            headers_to_copy.extend([\u0027X-Container-Host\u0027, \u0027X-Container-Device\u0027,"}],"source_content_type":"text/x-python","patch_set":32,"id":"3f79a3b5_ccea650d","line":1959,"in_reply_to":"3f79a3b5_3eecf0f1","updated":"2018-09-10 22:38:27.000000000","message":"XXX please patch this","commit_id":"83256db01ae98b902df7691cad155bbdc6024080"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"c4b0816434ae5f4b7237d7f53180dfcc86190c68","unresolved":false,"context_lines":[{"line_number":1961,"context_line":"            # copy any container update routing headers from original PUT"},{"line_number":1962,"context_line":"            headers_to_copy.extend([\u0027X-Container-Host\u0027, \u0027X-Container-Device\u0027,"},{"line_number":1963,"context_line":"                                    \u0027X-Container-Partition\u0027,"},{"line_number":1964,"context_line":"                                    \u0027X-Backend-Container-Path\u0027])"},{"line_number":1965,"context_line":"        for k in headers_to_copy:"},{"line_number":1966,"context_line":"            if k in self.nheaders:"},{"line_number":1967,"context_line":"                headers[k] \u003d self.nheaders[k]"}],"source_content_type":"text/x-python","patch_set":32,"id":"3f79a3b5_09998b77","line":1964,"updated":"2018-09-10 15:51:15.000000000","message":"Should there be expirer headers here, too?","commit_id":"83256db01ae98b902df7691cad155bbdc6024080"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"75867a9c210c01e910797df485d336bb97fd2337","unresolved":false,"context_lines":[{"line_number":1961,"context_line":"            # copy any container update routing headers from original PUT"},{"line_number":1962,"context_line":"            headers_to_copy.extend([\u0027X-Container-Host\u0027, \u0027X-Container-Device\u0027,"},{"line_number":1963,"context_line":"                                    \u0027X-Container-Partition\u0027,"},{"line_number":1964,"context_line":"                                    \u0027X-Backend-Container-Path\u0027])"},{"line_number":1965,"context_line":"        for k in headers_to_copy:"},{"line_number":1966,"context_line":"            if k in self.nheaders:"},{"line_number":1967,"context_line":"                headers[k] \u003d self.nheaders[k]"}],"source_content_type":"text/x-python","patch_set":32,"id":"3f79a3b5_86b9ac7a","line":1964,"in_reply_to":"3f79a3b5_09998b77","updated":"2018-09-10 22:56:26.000000000","message":"I was sure that they are included into the footer_metadata. I renamed the extra_headers to underscore this.","commit_id":"83256db01ae98b902df7691cad155bbdc6024080"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"2f7cd3091471486155fa7aecc3fc6d56988f5933","unresolved":false,"context_lines":[{"line_number":1984,"context_line":"        after object data."},{"line_number":1985,"context_line":""},{"line_number":1986,"context_line":"        :param footer_metadata: dictionary of metadata items"},{"line_number":1987,"context_line":"                                to be sent as footers."},{"line_number":1988,"context_line":"        \"\"\""},{"line_number":1989,"context_line":"        if self.state \u003d\u003d DATA_SENT:"},{"line_number":1990,"context_line":"            raise ValueError(\"called end_of_object_data twice\")"}],"source_content_type":"text/x-python","patch_set":32,"id":"5f7c97a3_54eef81f","line":1987,"updated":"2018-08-02 07:32:10.000000000","message":"We should we add the commit param to this doc string","commit_id":"83256db01ae98b902df7691cad155bbdc6024080"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"5d5354c06967a495b5bff02ba2c161bcab1f1873","unresolved":false,"context_lines":[{"line_number":1984,"context_line":"        after object data."},{"line_number":1985,"context_line":""},{"line_number":1986,"context_line":"        :param footer_metadata: dictionary of metadata items"},{"line_number":1987,"context_line":"                                to be sent as footers."},{"line_number":1988,"context_line":"        \"\"\""},{"line_number":1989,"context_line":"        if self.state \u003d\u003d DATA_SENT:"},{"line_number":1990,"context_line":"            raise ValueError(\"called end_of_object_data twice\")"}],"source_content_type":"text/x-python","patch_set":32,"id":"5f7c97a3_23fc483f","line":1987,"in_reply_to":"5f7c97a3_54eef81f","updated":"2018-08-02 23:19:01.000000000","message":"I added this, although not sure the new comment expresses the nature of the parameter.","commit_id":"83256db01ae98b902df7691cad155bbdc6024080"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a56346d93124dea05e7aded037ac4fce3fad9eb3","unresolved":false,"context_lines":[{"line_number":2029,"context_line":"        self.state \u003d DATA_ACKED"},{"line_number":2030,"context_line":"        self._send_post_to_temp(False, True)"},{"line_number":2031,"context_line":"        self.queue.put(\u0027\u0027)"},{"line_number":2032,"context_line":"        self.wait()"},{"line_number":2033,"context_line":"        self.state \u003d COMMIT_SENT"},{"line_number":2034,"context_line":""},{"line_number":2035,"context_line":"    @classmethod"}],"source_content_type":"text/x-python","patch_set":32,"id":"3f79a3b5_8c7f8d65","line":2032,"updated":"2018-09-10 22:38:27.000000000","message":"AFAICT the DoublePutter is the only one that calls .wait on itself - do we really need to force the queue to drain?","commit_id":"83256db01ae98b902df7691cad155bbdc6024080"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"1ed0dd252026e225621493b8cc2d4493c675ee73","unresolved":false,"context_lines":[{"line_number":2029,"context_line":"        self.state \u003d DATA_ACKED"},{"line_number":2030,"context_line":"        self._send_post_to_temp(False, True)"},{"line_number":2031,"context_line":"        self.queue.put(\u0027\u0027)"},{"line_number":2032,"context_line":"        self.wait()"},{"line_number":2033,"context_line":"        self.state \u003d COMMIT_SENT"},{"line_number":2034,"context_line":""},{"line_number":2035,"context_line":"    @classmethod"}],"source_content_type":"text/x-python","patch_set":32,"id":"3f79a3b5_1fc6286a","line":2032,"in_reply_to":"3f79a3b5_8c7f8d65","updated":"2018-09-11 21:05:09.000000000","message":"Well, it was there before, so... I suppose when you\u0027re waiting for replies, the greenthread still progresses. Is there a harm though? There\u0027s no useful pipelining happening.\n\nBTW, that self.queue.put(\u0027\u0027) was harmful and I removed it.","commit_id":"83256db01ae98b902df7691cad155bbdc6024080"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"a37021642dfe43c1b138023ff8b0bd2616bca57e","unresolved":false,"context_lines":[{"line_number":2046,"context_line":"        a 400 error in the old code on purpose. When this occurs, proxy knows"},{"line_number":2047,"context_line":"        that it talks to old storage node and can fall back. For efficiency,"},{"line_number":2048,"context_line":"        once the fallback is initiated, it remains in effect for an extended"},{"line_number":2049,"context_line":"        period of time. This way we aren\u0027t resending every PUT twice."},{"line_number":2050,"context_line":""},{"line_number":2051,"context_line":"        :returns: Unlike the previous Putter.connect() implementations,"},{"line_number":2052,"context_line":"                  this can return None iff the DoublePutter protocol is"}],"source_content_type":"text/x-python","patch_set":32,"id":"5f7c97a3_57e25e9e","line":2049,"updated":"2018-08-03 06:27:10.000000000","message":"this comment mentions that on a 400 from the object server, meaning it\u0027s a old one, it\u0027ll fall back to the MIMEPutter. But in my testing this didn\u0027t happen.\n\nHas this been implemented yet or am I missing something obvious?","commit_id":"83256db01ae98b902df7691cad155bbdc6024080"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"76782e5bcd51d06b67169efe197ae16bbf2b57ff","unresolved":false,"context_lines":[{"line_number":2046,"context_line":"        a 400 error in the old code on purpose. When this occurs, proxy knows"},{"line_number":2047,"context_line":"        that it talks to old storage node and can fall back. For efficiency,"},{"line_number":2048,"context_line":"        once the fallback is initiated, it remains in effect for an extended"},{"line_number":2049,"context_line":"        period of time. This way we aren\u0027t resending every PUT twice."},{"line_number":2050,"context_line":""},{"line_number":2051,"context_line":"        :returns: Unlike the previous Putter.connect() implementations,"},{"line_number":2052,"context_line":"                  this can return None iff the DoublePutter protocol is"}],"source_content_type":"text/x-python","patch_set":32,"id":"3f79a3b5_c6faf273","line":2049,"in_reply_to":"3f79a3b5_9e51fa11","updated":"2018-08-09 07:24:06.000000000","message":"ahh, good to know I\u0027m not going crazy then :P","commit_id":"83256db01ae98b902df7691cad155bbdc6024080"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"6449a1bd6604153b4b42cab2871e1732bffe880d","unresolved":false,"context_lines":[{"line_number":2046,"context_line":"        a 400 error in the old code on purpose. When this occurs, proxy knows"},{"line_number":2047,"context_line":"        that it talks to old storage node and can fall back. For efficiency,"},{"line_number":2048,"context_line":"        once the fallback is initiated, it remains in effect for an extended"},{"line_number":2049,"context_line":"        period of time. This way we aren\u0027t resending every PUT twice."},{"line_number":2050,"context_line":""},{"line_number":2051,"context_line":"        :returns: Unlike the previous Putter.connect() implementations,"},{"line_number":2052,"context_line":"                  this can return None iff the DoublePutter protocol is"}],"source_content_type":"text/x-python","patch_set":32,"id":"3f79a3b5_9e51fa11","line":2049,"in_reply_to":"5f7c97a3_57e25e9e","updated":"2018-08-04 05:42:15.000000000","message":"Oops. The comment is incorrect. The last version that implemented the fallback was #19. Look here if you\u0027re curious how that worked, see _make_putter():\n https://review.openstack.org/#/c/427911/19/swift/proxy/controllers/obj.py","commit_id":"83256db01ae98b902df7691cad155bbdc6024080"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"a37021642dfe43c1b138023ff8b0bd2616bca57e","unresolved":false,"context_lines":[{"line_number":2073,"context_line":"        headers[\u0027X-Backend-Temp-Hold\u0027] \u003d \"1\""},{"line_number":2074,"context_line":"        conn, resp, final_resp, connect_duration \u003d cls._make_connection("},{"line_number":2075,"context_line":"            node, part, path, headers, conn_timeout, node_timeout,"},{"line_number":2076,"context_line":"            version\u003d\u0027v1\u0027)"},{"line_number":2077,"context_line":"        # Believe it or not, the Putter protocol is to return a perfectly"},{"line_number":2078,"context_line":"        # good instance even when PUT throws an error. So, no checking"},{"line_number":2079,"context_line":"        # of resp.status here."}],"source_content_type":"text/x-python","patch_set":32,"id":"5f7c97a3_77b51a7d","line":2076,"updated":"2018-08-03 06:27:10.000000000","message":"This is the connection where we should receive the 400 if it\u0027s and older object server. So how should we be falling back?\n\nMaybe catch a HTTPBadRequest back in the _make_putter functions when a DoublePutter is created and fallback to the MIMEPutter?\n\nOr should we just return a MIMEPutter in this case?","commit_id":"83256db01ae98b902df7691cad155bbdc6024080"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"1ed0dd252026e225621493b8cc2d4493c675ee73","unresolved":false,"context_lines":[{"line_number":2073,"context_line":"        headers[\u0027X-Backend-Temp-Hold\u0027] \u003d \"1\""},{"line_number":2074,"context_line":"        conn, resp, final_resp, connect_duration \u003d cls._make_connection("},{"line_number":2075,"context_line":"            node, part, path, headers, conn_timeout, node_timeout,"},{"line_number":2076,"context_line":"            version\u003d\u0027v1\u0027)"},{"line_number":2077,"context_line":"        # Believe it or not, the Putter protocol is to return a perfectly"},{"line_number":2078,"context_line":"        # good instance even when PUT throws an error. So, no checking"},{"line_number":2079,"context_line":"        # of resp.status here."}],"source_content_type":"text/x-python","patch_set":32,"id":"3f79a3b5_7fc5dc3c","line":2076,"in_reply_to":"5f7c97a3_77b51a7d","updated":"2018-09-11 21:05:09.000000000","message":"We discussed it at Denver PTG (Stein) again and decided not to screw with fallbacks for now.","commit_id":"83256db01ae98b902df7691cad155bbdc6024080"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a56346d93124dea05e7aded037ac4fce3fad9eb3","unresolved":false,"context_lines":[{"line_number":2981,"context_line":"            headers.pop(\u0027Content-Length\u0027, None)"},{"line_number":2982,"context_line":"            headers.pop(\u0027Etag\u0027, None)"},{"line_number":2983,"context_line":"            # the transfer will need to be chunked"},{"line_number":2984,"context_line":"            headers[\u0027Transfer-Encoding\u0027] \u003d \u0027chunked\u0027"},{"line_number":2985,"context_line":""},{"line_number":2986,"context_line":"        # Since the request body sent from client -\u003e proxy is not"},{"line_number":2987,"context_line":"        # the same as the request body sent proxy -\u003e object, we"}],"source_content_type":"text/x-python","patch_set":32,"id":"3f79a3b5_ac8b6915","line":2984,"updated":"2018-09-10 22:38:27.000000000","message":"what?  why.  Only the MIMEPutter should require chunked on a client Content-Length transfer and it already forces it.","commit_id":"83256db01ae98b902df7691cad155bbdc6024080"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"4483570c3af2600c4a19fef1befd37785ec793a5","unresolved":false,"context_lines":[{"line_number":2981,"context_line":"            headers.pop(\u0027Content-Length\u0027, None)"},{"line_number":2982,"context_line":"            headers.pop(\u0027Etag\u0027, None)"},{"line_number":2983,"context_line":"            # the transfer will need to be chunked"},{"line_number":2984,"context_line":"            headers[\u0027Transfer-Encoding\u0027] \u003d \u0027chunked\u0027"},{"line_number":2985,"context_line":""},{"line_number":2986,"context_line":"        # Since the request body sent from client -\u003e proxy is not"},{"line_number":2987,"context_line":"        # the same as the request body sent proxy -\u003e object, we"}],"source_content_type":"text/x-python","patch_set":32,"id":"3f79a3b5_1b52cd69","line":2984,"in_reply_to":"3f79a3b5_ac8b6915","updated":"2018-09-11 05:16:03.000000000","message":"Interesting... yeah, it should be possible to set\n\n headers[\u0027Content-Length\u0027] \u003d expected_frag_size\n\nnow...\n\nI suppose we can\u0027t set ETag yet though, since we\u0027ve got footers that (may) have an updated ETag that should be used instead.\n\nQuick reminder that a nice follow-up to this would be to address https://bugs.launchpad.net/swift/+bug/1632839 ... I definitely think it can be out-of-scope for this change though.","commit_id":"83256db01ae98b902df7691cad155bbdc6024080"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"1ed0dd252026e225621493b8cc2d4493c675ee73","unresolved":false,"context_lines":[{"line_number":2981,"context_line":"            headers.pop(\u0027Content-Length\u0027, None)"},{"line_number":2982,"context_line":"            headers.pop(\u0027Etag\u0027, None)"},{"line_number":2983,"context_line":"            # the transfer will need to be chunked"},{"line_number":2984,"context_line":"            headers[\u0027Transfer-Encoding\u0027] \u003d \u0027chunked\u0027"},{"line_number":2985,"context_line":""},{"line_number":2986,"context_line":"        # Since the request body sent from client -\u003e proxy is not"},{"line_number":2987,"context_line":"        # the same as the request body sent proxy -\u003e object, we"}],"source_content_type":"text/x-python","patch_set":32,"id":"3f79a3b5_42b92713","line":2984,"in_reply_to":"3f79a3b5_ac8b6915","updated":"2018-09-11 21:05:09.000000000","message":"Okay, I don\u0027t remember the logic behind it too well. As you can see, content-length is forcefully removed. When you do that, the transfer encoding must be chunked, because we do not want to close the connection (where the POST is going to be sent). Previously, MIME had its own chunkization going on, so this replaces that mechanism. But... why? Why the length is never known for EC? Because encoding changes the sizes... with fragment headers of varying size? Maybe?","commit_id":"83256db01ae98b902df7691cad155bbdc6024080"}],"swift/proxy/server.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"46c4035a8817184b73b4a0e75e6fd2c9fdbb0af5","unresolved":false,"context_lines":[{"line_number":692,"context_line":"    def is_put_v1(self):"},{"line_number":693,"context_line":"        \"\"\""},{"line_number":694,"context_line":"        Return True if the v1 PUT is to be tried. Return False for legacy."},{"line_number":695,"context_line":"        \"\"\""},{"line_number":696,"context_line":"        if self.failed_time is None:"},{"line_number":697,"context_line":"            return True"},{"line_number":698,"context_line":"        if time() \u003e\u003d self.failed_time + PUT_V1_TIME:"}],"source_content_type":"text/x-python","patch_set":19,"id":"5f4e5783_5e76d508","line":695,"updated":"2017-10-11 01:39:37.000000000","message":"so... the legacy protocol was the first one... and this current PUT+POST+POST.... the new one... the second one... is v1?","commit_id":"69ef1ba4da42fb6b3d2a6feca8ec39215db6b0a9"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"1c736b4fd62f145cee3cb2d6098812ddb2faf3dc","unresolved":false,"context_lines":[{"line_number":692,"context_line":"    def is_put_v1(self):"},{"line_number":693,"context_line":"        \"\"\""},{"line_number":694,"context_line":"        Return True if the v1 PUT is to be tried. Return False for legacy."},{"line_number":695,"context_line":"        \"\"\""},{"line_number":696,"context_line":"        if self.failed_time is None:"},{"line_number":697,"context_line":"            return True"},{"line_number":698,"context_line":"        if time() \u003e\u003d self.failed_time + PUT_V1_TIME:"}],"source_content_type":"text/x-python","patch_set":19,"id":"5f4e5783_638cbed6","line":695,"in_reply_to":"5f4e5783_5e76d508","updated":"2017-10-17 01:51:27.000000000","message":"The legacy is \"v0\", if you wish.","commit_id":"69ef1ba4da42fb6b3d2a6feca8ec39215db6b0a9"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"46c4035a8817184b73b4a0e75e6fd2c9fdbb0af5","unresolved":false,"context_lines":[{"line_number":704,"context_line":"        \"\"\""},{"line_number":705,"context_line":"        Report a failure of PUT+POST, so that a fallback can be implemented."},{"line_number":706,"context_line":"        \"\"\""},{"line_number":707,"context_line":"        self.failed_time \u003d time()"},{"line_number":708,"context_line":""},{"line_number":709,"context_line":""},{"line_number":710,"context_line":"def parse_per_policy_config(conf):"}],"source_content_type":"text/x-python","patch_set":19,"id":"5f4e5783_9ecb2d52","line":707,"updated":"2017-10-11 01:39:37.000000000","message":"I\u0027m pretty sure I don\u0027t like any of this - although it might just be I don\u0027t understand it.","commit_id":"69ef1ba4da42fb6b3d2a6feca8ec39215db6b0a9"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"1c736b4fd62f145cee3cb2d6098812ddb2faf3dc","unresolved":false,"context_lines":[{"line_number":704,"context_line":"        \"\"\""},{"line_number":705,"context_line":"        Report a failure of PUT+POST, so that a fallback can be implemented."},{"line_number":706,"context_line":"        \"\"\""},{"line_number":707,"context_line":"        self.failed_time \u003d time()"},{"line_number":708,"context_line":""},{"line_number":709,"context_line":""},{"line_number":710,"context_line":"def parse_per_policy_config(conf):"}],"source_content_type":"text/x-python","patch_set":19,"id":"5f4e5783_e36e6efe","line":707,"in_reply_to":"5f4e5783_9ecb2d52","updated":"2017-10-17 01:51:27.000000000","message":"Ironically, the bugs that I inserted prove that the fallback mechanism works perfectly. If you just take this patch and run it, the cluster will continue with no issue, using the legacy protocol (because an attempt to use v1 every 20 minutes causes \"411 Length Required\").\n\nBut it\u0027s okay, I\u0027ll kill it, in order to save reviewers\u0027 time at this stage.","commit_id":"69ef1ba4da42fb6b3d2a6feca8ec39215db6b0a9"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"76782e5bcd51d06b67169efe197ae16bbf2b57ff","unresolved":false,"context_lines":[{"line_number":285,"context_line":"        self.disallowed_sections \u003d list_from_csv("},{"line_number":286,"context_line":"            conf.get(\u0027disallowed_sections\u0027, \u0027swift.valid_api_versions\u0027))"},{"line_number":287,"context_line":"        self.admin_key \u003d conf.get(\u0027admin_key\u0027, None)"},{"line_number":288,"context_line":"        # default to using the new object put+post+post protocol, at least for"},{"line_number":289,"context_line":"        # dev so that functional tests exercise this XXX"},{"line_number":290,"context_line":"        # XXX unit tests fail? 499 all over"},{"line_number":291,"context_line":"        self.use_put_v1 \u003d config_true_value(conf.get(\u0027put_api_v1\u0027, \u0027yes\u0027))"}],"source_content_type":"text/x-python","patch_set":32,"id":"3f79a3b5_e6684e22","line":288,"range":{"start_line":288,"start_character":42,"end_line":288,"end_character":55},"updated":"2018-08-09 07:24:06.000000000","message":"Maybe we need a shorter term for this at least as a team because writing PUT+POST+POST is getting annoying, maybe PPP or P3 or P^3 :P","commit_id":"83256db01ae98b902df7691cad155bbdc6024080"}],"test/unit/obj/test_diskfile.py":[{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"a37021642dfe43c1b138023ff8b0bd2616bca57e","unresolved":false,"context_lines":[{"line_number":8249,"context_line":"        iut.expire(time())"},{"line_number":8250,"context_line":""},{"line_number":8251,"context_line":"        closed \u003d [0]"},{"line_number":8252,"context_line":"        df1 \u003d FakeDiskFileWriter(closed)"},{"line_number":8253,"context_line":"        id1 \u003d iut.save(df1)"},{"line_number":8254,"context_line":""},{"line_number":8255,"context_line":"        # This little trick makes this test stable w.r.t the short"}],"source_content_type":"text/x-python","patch_set":32,"id":"5f7c97a3_32a49005","line":8252,"updated":"2018-08-03 06:27:10.000000000","message":"You could also force time.time() to return something you wanted using mock. Something like:\n\n  with mock.patch(\u0027time.time\u0027, return_value\u003d\u00270\u0027):\n      id1 \u003d iut.save(df1)\n\n  iut.expire(1)\n  \u003ctest\u003e\n  iut.expire(3)\n  \u003ctest\u003e\n  ...\n\nBut meh your way is readable :)","commit_id":"83256db01ae98b902df7691cad155bbdc6024080"}],"test/unit/obj/test_server.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"cebc4f2009f5c91a67d19103054b32c6c1ae8589","unresolved":false,"context_lines":[{"line_number":423,"context_line":"                   \u0027Content-Type\u0027: \u0027application/x-test\u0027}"},{"line_number":424,"context_line":"        req \u003d Request.blank(\u0027/sda1/p/a/c/o\u0027,"},{"line_number":425,"context_line":"                            environ\u003d{\u0027REQUEST_METHOD\u0027: \u0027POST\u0027,"},{"line_number":426,"context_line":"                                     \u0027QUERY_STRING\u0027: \u0027durable\u003dyes\u0027},"},{"line_number":427,"context_line":"                            headers\u003dheaders)"},{"line_number":428,"context_line":"        resp \u003d req.get_response(self.object_controller)"},{"line_number":429,"context_line":"        self.assertEqual(resp.status_int, 202)"}],"source_content_type":"text/x-python","patch_set":1,"id":"3a461143_4f3f74d6","line":426,"updated":"2017-02-02 03:50:09.000000000","message":"I sorta like having it in the qs - good call!","commit_id":"442f483da525b9f7c9f6bdd35f412a1fb92c8974"}]}