)]}'
{"setup.cfg":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"d367e1cb6a08adab3b50266a3f5d512955358269","unresolved":false,"context_lines":[{"line_number":78,"context_line":"[entry_points]"},{"line_number":79,"context_line":"console_scripts \u003d"},{"line_number":80,"context_line":"    swift-manage-shard-ranges \u003d swift.cli.manage_shard_ranges:main"},{"line_number":81,"context_line":"    swift-container-deleter \u003d swift.cli.container_deleter:main"},{"line_number":82,"context_line":""},{"line_number":83,"context_line":"paste.app_factory \u003d"},{"line_number":84,"context_line":"    proxy \u003d swift.proxy.server:app_factory"}],"source_content_type":"text/x-ttcn-cfg","patch_set":5,"id":"bfb3d3c7_364d5a72","line":81,"updated":"2019-05-20 18:42:41.000000000","message":"I\u0027m not sure using entry_points for console_scripts is more virtuous than all the bin scripts above...\n\nIf I\u0027d have noticed swift-manage-shard-ranges is doing something different than the other 30-some-odd scripts in this file I might have suggested we still with \"one obvious way to do it\"\n\nNot sure if this is just adding to the \"problem\" or if we plan to migrate everyone to use console scripts?","commit_id":"84b63da021dfb8c719e1c4616ea0aa426844cba4"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"94f1537cdf4eca55b0be63a25358f0f87d5e9ca7","unresolved":false,"context_lines":[{"line_number":78,"context_line":"[entry_points]"},{"line_number":79,"context_line":"console_scripts \u003d"},{"line_number":80,"context_line":"    swift-manage-shard-ranges \u003d swift.cli.manage_shard_ranges:main"},{"line_number":81,"context_line":"    swift-container-deleter \u003d swift.cli.container_deleter:main"},{"line_number":82,"context_line":""},{"line_number":83,"context_line":"paste.app_factory \u003d"},{"line_number":84,"context_line":"    proxy \u003d swift.proxy.server:app_factory"}],"source_content_type":"text/x-ttcn-cfg","patch_set":5,"id":"bfb3d3c7_96568629","line":81,"in_reply_to":"bfb3d3c7_364d5a72","updated":"2019-05-20 19:07:11.000000000","message":"¯\\_(ツ)_/¯\n\nIDK, scripts like https://github.com/openstack/swift/blob/2.21.0/bin/swift-form-signature just seem... unnecessary. Not sure what we gain by it (beyond consistency).","commit_id":"84b63da021dfb8c719e1c4616ea0aa426844cba4"}],"swift/cli/container_deleter.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"297e2c5320209880baa7d361541d8cf62326a474","unresolved":false,"context_lines":[{"line_number":22,"context_line":"def make_delete_jobs(account, container, objects, timestamp):"},{"line_number":23,"context_line":"    return ["},{"line_number":24,"context_line":"        {"},{"line_number":25,"context_line":"            \u0027name\u0027: \u0027%d-%s/%s/%s\u0027 % ("},{"line_number":26,"context_line":"                int(timestamp), account, container,"},{"line_number":27,"context_line":"                obj[\u0027name\u0027]),"},{"line_number":28,"context_line":"            \u0027deleted\u0027: 0,"},{"line_number":29,"context_line":"            \u0027created_at\u0027: timestamp.internal,"}],"source_content_type":"text/x-python","patch_set":1,"id":"dfbec78f_60805dc0","line":26,"range":{"start_line":25,"start_character":20,"end_line":26,"end_character":50},"updated":"2019-05-14 21:27:26.000000000","message":"This should surely be factored out into a build_task_obj() that lives in obj/expirer.py or something, to complement parse_task_obj().","commit_id":"0ff4fe3d248627b987f589584608139365944dce"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"06ebc89d8abc5edfe8eac9026c0e271c8d6bd7f1","unresolved":false,"context_lines":[{"line_number":22,"context_line":"def make_delete_jobs(account, container, objects, timestamp):"},{"line_number":23,"context_line":"    return ["},{"line_number":24,"context_line":"        {"},{"line_number":25,"context_line":"            \u0027name\u0027: \u0027%d-%s/%s/%s\u0027 % ("},{"line_number":26,"context_line":"                int(timestamp), account, container,"},{"line_number":27,"context_line":"                obj[\u0027name\u0027]),"},{"line_number":28,"context_line":"            \u0027deleted\u0027: 0,"},{"line_number":29,"context_line":"            \u0027created_at\u0027: timestamp.internal,"}],"source_content_type":"text/x-python","patch_set":1,"id":"dfbec78f_1b40f4a4","line":26,"range":{"start_line":25,"start_character":20,"end_line":26,"end_character":50},"in_reply_to":"dfbec78f_60805dc0","updated":"2019-05-14 23:49:01.000000000","message":"Done","commit_id":"0ff4fe3d248627b987f589584608139365944dce"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"297e2c5320209880baa7d361541d8cf62326a474","unresolved":false,"context_lines":[{"line_number":41,"context_line":"    deleted \u003d 0"},{"line_number":42,"context_line":"    obj_iter \u003d swift.iter_objects(account, container, prefix\u003dprefix)"},{"line_number":43,"context_line":"    while True:"},{"line_number":44,"context_line":"        to_delete \u003d list(itertools.islice(obj_iter, 10000))"},{"line_number":45,"context_line":"        if not to_delete:"},{"line_number":46,"context_line":"            break"},{"line_number":47,"context_line":"        delete_jobs \u003d make_delete_jobs("}],"source_content_type":"text/x-python","patch_set":1,"id":"dfbec78f_60affd7c","line":44,"range":{"start_line":44,"start_character":52,"end_line":44,"end_character":57},"updated":"2019-05-14 21:27:26.000000000","message":"Maybe should be configurable? Should probably at least be a constant.","commit_id":"0ff4fe3d248627b987f589584608139365944dce"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"06ebc89d8abc5edfe8eac9026c0e271c8d6bd7f1","unresolved":false,"context_lines":[{"line_number":41,"context_line":"    deleted \u003d 0"},{"line_number":42,"context_line":"    obj_iter \u003d swift.iter_objects(account, container, prefix\u003dprefix)"},{"line_number":43,"context_line":"    while True:"},{"line_number":44,"context_line":"        to_delete \u003d list(itertools.islice(obj_iter, 10000))"},{"line_number":45,"context_line":"        if not to_delete:"},{"line_number":46,"context_line":"            break"},{"line_number":47,"context_line":"        delete_jobs \u003d make_delete_jobs("}],"source_content_type":"text/x-python","patch_set":1,"id":"dfbec78f_fb42c0aa","line":44,"range":{"start_line":44,"start_character":52,"end_line":44,"end_character":57},"in_reply_to":"dfbec78f_60affd7c","updated":"2019-05-14 23:49:01.000000000","message":"Converted to a constant.","commit_id":"0ff4fe3d248627b987f589584608139365944dce"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"297e2c5320209880baa7d361541d8cf62326a474","unresolved":false,"context_lines":[{"line_number":48,"context_line":"            account, container, to_delete, timestamp)"},{"line_number":49,"context_line":"        swift.make_request("},{"line_number":50,"context_line":"            \u0027UPDATE\u0027,"},{"line_number":51,"context_line":"            swift.make_path(\u0027.expiring_objects\u0027, str(int(timestamp))),"},{"line_number":52,"context_line":"            headers\u003d{\u0027X-Backend-Allow-Method\u0027: \u0027UPDATE\u0027,"},{"line_number":53,"context_line":"                     \u0027X-Backend-Storage-Policy-Index\u0027: \u00270\u0027,"},{"line_number":54,"context_line":"                     \u0027X-Timestamp\u0027: timestamp.internal},"}],"source_content_type":"text/x-python","patch_set":1,"id":"dfbec78f_20a4a5a5","line":51,"range":{"start_line":51,"start_character":30,"end_line":51,"end_character":46},"updated":"2019-05-14 21:27:26.000000000","message":"This should be looking for auto_create_account_prefix and expiring_objects_account_name config values from... some section?","commit_id":"0ff4fe3d248627b987f589584608139365944dce"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"297e2c5320209880baa7d361541d8cf62326a474","unresolved":false,"context_lines":[{"line_number":63,"context_line":"    parser.add_argument("},{"line_number":64,"context_line":"        \u0027--internal-client\u0027, dest\u003d\u0027conf\u0027,"},{"line_number":65,"context_line":"        default\u003d\u0027/etc/swift/internal-client.conf\u0027)"},{"line_number":66,"context_line":"    parser.add_argument(\u0027--request-tries\u0027, type\u003dint, default\u003d3)"},{"line_number":67,"context_line":"    parser.add_argument(\u0027account\u0027, help\u003d\u0027account from which to delete\u0027)"},{"line_number":68,"context_line":"    parser.add_argument(\u0027container\u0027, help\u003d\u0027container from which to delete\u0027)"},{"line_number":69,"context_line":"    parser.add_argument("}],"source_content_type":"text/x-python","patch_set":1,"id":"dfbec78f_00a721a9","line":66,"updated":"2019-05-14 21:27:26.000000000","message":"Better to have this coming from config, though I\u0027m not sure where.","commit_id":"0ff4fe3d248627b987f589584608139365944dce"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"297e2c5320209880baa7d361541d8cf62326a474","unresolved":false,"context_lines":[{"line_number":64,"context_line":"        \u0027--internal-client\u0027, dest\u003d\u0027conf\u0027,"},{"line_number":65,"context_line":"        default\u003d\u0027/etc/swift/internal-client.conf\u0027)"},{"line_number":66,"context_line":"    parser.add_argument(\u0027--request-tries\u0027, type\u003dint, default\u003d3)"},{"line_number":67,"context_line":"    parser.add_argument(\u0027account\u0027, help\u003d\u0027account from which to delete\u0027)"},{"line_number":68,"context_line":"    parser.add_argument(\u0027container\u0027, help\u003d\u0027container from which to delete\u0027)"},{"line_number":69,"context_line":"    parser.add_argument("},{"line_number":70,"context_line":"        \u0027--prefix\u0027, default\u003d\u0027\u0027,"},{"line_number":71,"context_line":"        help\u003d\u0027delete all objects with this prefix\u0027)"}],"source_content_type":"text/x-python","patch_set":1,"id":"dfbec78f_c00ca9a6","line":68,"range":{"start_line":67,"start_character":4,"end_line":68,"end_character":75},"updated":"2019-05-14 21:27:26.000000000","message":"Maybe have an alternate form that just takes a local DB file? But then it won\u0027t play nice with sharding...","commit_id":"0ff4fe3d248627b987f589584608139365944dce"},{"author":{"_account_id":25251,"name":"Alexandre Lécuyer","email":"alexandre.lecuyer@corp.ovh.com","username":"alecuyer"},"change_message_id":"5dc925fe9540cca47e1cd5b7d9663842063a3389","unresolved":false,"context_lines":[{"line_number":9,"context_line":"# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the"},{"line_number":10,"context_line":"# License for the specific language governing permissions and limitations"},{"line_number":11,"context_line":"# under the License."},{"line_number":12,"context_line":""},{"line_number":13,"context_line":"import argparse"},{"line_number":14,"context_line":"import io"},{"line_number":15,"context_line":"import itertools"}],"source_content_type":"text/x-python","patch_set":3,"id":"bfb3d3c7_5eee0a89","line":12,"updated":"2019-05-17 13:07:06.000000000","message":"Should we have a short docstring here for users? or in the config parser help.\n\nMaybe just the first line from your commit: \"tool to async-delete some or all objects in a container\" and a note about it relying on the object-expirer to effectively remove objects","commit_id":"aaa0e60541ff0b42097d8b2922ac666df61cb21f"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6597aab4173564b3a723cbfc0096871994d046a7","unresolved":false,"context_lines":[{"line_number":9,"context_line":"# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the"},{"line_number":10,"context_line":"# License for the specific language governing permissions and limitations"},{"line_number":11,"context_line":"# under the License."},{"line_number":12,"context_line":""},{"line_number":13,"context_line":"import argparse"},{"line_number":14,"context_line":"import io"},{"line_number":15,"context_line":"import itertools"}],"source_content_type":"text/x-python","patch_set":3,"id":"bfb3d3c7_7a7197e0","line":12,"in_reply_to":"bfb3d3c7_5eee0a89","updated":"2019-05-17 16:27:49.000000000","message":"Definitely; probably both.","commit_id":"aaa0e60541ff0b42097d8b2922ac666df61cb21f"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6597aab4173564b3a723cbfc0096871994d046a7","unresolved":false,"context_lines":[{"line_number":69,"context_line":"    parser.add_argument(\u0027container\u0027, help\u003d\u0027container from which to delete\u0027)"},{"line_number":70,"context_line":"    parser.add_argument("},{"line_number":71,"context_line":"        \u0027--prefix\u0027, default\u003d\u0027\u0027,"},{"line_number":72,"context_line":"        help\u003d\u0027delete all objects with this prefix\u0027)"},{"line_number":73,"context_line":"    parser.add_argument("},{"line_number":74,"context_line":"        \u0027--timestamp\u0027, type\u003dTimestamp, default\u003dTimestamp.now(),"},{"line_number":75,"context_line":"        help\u003d\u0027delete all objects as of this time\u0027)"}],"source_content_type":"text/x-python","patch_set":3,"id":"bfb3d3c7_7d3f51d6","line":72,"updated":"2019-05-17 16:27:49.000000000","message":"I wonder if this should also take marker/end_marker...\n\nHere\u0027s my thinking: If I want to go clear out a (presumably sharded) container with a billion objects in it, we\u0027re looking at hundreds of thousands of requests; this presents two challenges:\n\n- It\u0027s going to take a while. We ought to periodically emit a status update to stdout.\n- There are going to be failures. As it is, make_request() will raise UnexpectedResponse and we just stop where we are. So how do you resume? Starting again from the beginning sucks, but so does waiting for the expirer to clear out what\u0027s been marked so far.\n\nSo, if we accept a marker option and emit status like\n\n Marked {deleted} objects, through {delete_jobs[-1][\u0027name\u0027]:r}\n\nusers can see progress, and when there\u0027s a failure they can pick up (roughly) where they left off.","commit_id":"aaa0e60541ff0b42097d8b2922ac666df61cb21f"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"94f1537cdf4eca55b0be63a25358f0f87d5e9ca7","unresolved":false,"context_lines":[{"line_number":40,"context_line":"def make_delete_jobs(account, container, objects, timestamp):"},{"line_number":41,"context_line":"    return ["},{"line_number":42,"context_line":"        {"},{"line_number":43,"context_line":"            \u0027name\u0027: build_task_obj(timestamp, account, container, obj[\u0027name\u0027]),"},{"line_number":44,"context_line":"            \u0027deleted\u0027: 0,"},{"line_number":45,"context_line":"            \u0027created_at\u0027: timestamp.internal,"},{"line_number":46,"context_line":"            \u0027etag\u0027: MD5_OF_EMPTY_STRING,"}],"source_content_type":"text/x-python","patch_set":5,"id":"bfb3d3c7_16eff684","line":43,"range":{"start_line":43,"start_character":66,"end_line":43,"end_character":77},"updated":"2019-05-20 19:07:11.000000000","message":"Still not sure if this should take a list of dicts, or put it on the caller to extract names...\n\nFollow-up suggests maybe this should just be a list of names?","commit_id":"84b63da021dfb8c719e1c4616ea0aa426844cba4"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"dfa6b8a7511e30fe388fd46f9a02daac4fdb8c1d","unresolved":false,"context_lines":[{"line_number":40,"context_line":"def make_delete_jobs(account, container, objects, timestamp):"},{"line_number":41,"context_line":"    return ["},{"line_number":42,"context_line":"        {"},{"line_number":43,"context_line":"            \u0027name\u0027: build_task_obj(timestamp, account, container, obj[\u0027name\u0027]),"},{"line_number":44,"context_line":"            \u0027deleted\u0027: 0,"},{"line_number":45,"context_line":"            \u0027created_at\u0027: timestamp.internal,"},{"line_number":46,"context_line":"            \u0027etag\u0027: MD5_OF_EMPTY_STRING,"}],"source_content_type":"text/x-python","patch_set":5,"id":"bfb3d3c7_1abb820e","line":43,"range":{"start_line":43,"start_character":66,"end_line":43,"end_character":77},"in_reply_to":"bfb3d3c7_16eff684","updated":"2019-05-21 20:21:36.000000000","message":"Done","commit_id":"84b63da021dfb8c719e1c4616ea0aa426844cba4"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"d367e1cb6a08adab3b50266a3f5d512955358269","unresolved":false,"context_lines":[{"line_number":54,"context_line":"                      prefix, timestamp\u003dNone, yield_time\u003d10):"},{"line_number":55,"context_line":"    def enqueue_deletes():"},{"line_number":56,"context_line":"        if timestamp is None:"},{"line_number":57,"context_line":"            timestamp \u003d Timestamp.now()"},{"line_number":58,"context_line":""},{"line_number":59,"context_line":"        deleted \u003d 0"},{"line_number":60,"context_line":"        obj_iter \u003d swift.iter_objects("}],"source_content_type":"text/x-python","patch_set":5,"id":"bfb3d3c7_b6dcea74","line":57,"updated":"2019-05-20 18:42:41.000000000","message":"If you move this out of the closure you won\u0027t get the Unbound(Non)LocalError thing\n\nThis method needs some tests","commit_id":"84b63da021dfb8c719e1c4616ea0aa426844cba4"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"dfa6b8a7511e30fe388fd46f9a02daac4fdb8c1d","unresolved":false,"context_lines":[{"line_number":54,"context_line":"                      prefix, timestamp\u003dNone, yield_time\u003d10):"},{"line_number":55,"context_line":"    def enqueue_deletes():"},{"line_number":56,"context_line":"        if timestamp is None:"},{"line_number":57,"context_line":"            timestamp \u003d Timestamp.now()"},{"line_number":58,"context_line":""},{"line_number":59,"context_line":"        deleted \u003d 0"},{"line_number":60,"context_line":"        obj_iter \u003d swift.iter_objects("}],"source_content_type":"text/x-python","patch_set":5,"id":"bfb3d3c7_ba37d6b0","line":57,"in_reply_to":"bfb3d3c7_16d616c7","updated":"2019-05-21 20:21:36.000000000","message":"Done","commit_id":"84b63da021dfb8c719e1c4616ea0aa426844cba4"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"94f1537cdf4eca55b0be63a25358f0f87d5e9ca7","unresolved":false,"context_lines":[{"line_number":54,"context_line":"                      prefix, timestamp\u003dNone, yield_time\u003d10):"},{"line_number":55,"context_line":"    def enqueue_deletes():"},{"line_number":56,"context_line":"        if timestamp is None:"},{"line_number":57,"context_line":"            timestamp \u003d Timestamp.now()"},{"line_number":58,"context_line":""},{"line_number":59,"context_line":"        deleted \u003d 0"},{"line_number":60,"context_line":"        obj_iter \u003d swift.iter_objects("}],"source_content_type":"text/x-python","patch_set":5,"id":"bfb3d3c7_16d616c7","line":57,"in_reply_to":"bfb3d3c7_b6dcea74","updated":"2019-05-20 19:07:11.000000000","message":"Bah! Good catch.","commit_id":"84b63da021dfb8c719e1c4616ea0aa426844cba4"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"87bf07abc8ee1ca104138e7c9db7c72e05deb71a","unresolved":false,"context_lines":[{"line_number":67,"context_line":"            \u0027size\u0027: 0,"},{"line_number":68,"context_line":"            \u0027storage_policy_index\u0027: 0,"},{"line_number":69,"context_line":"            \u0027content_type\u0027: ASYNC_DELETE_TYPE,"},{"line_number":70,"context_line":"        } for obj in objects]"},{"line_number":71,"context_line":""},{"line_number":72,"context_line":""},{"line_number":73,"context_line":"def mark_for_deletion(swift, account, container, marker, end_marker,"}],"source_content_type":"text/x-python","patch_set":6,"id":"bfb3d3c7_0f124db2","line":70,"updated":"2019-05-22 20:26:53.000000000","message":"looking down the road this might be a candidate for something in a common module kind of place\n\nbut since this is the only consumer for now this is probably the correct place","commit_id":"2da6313f83b1b98e039844046bf8b3ff4c28c465"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a5c32db96b241f43e97cf1e43fcced8da55f9a5a","unresolved":false,"context_lines":[{"line_number":67,"context_line":"            \u0027size\u0027: 0,"},{"line_number":68,"context_line":"            \u0027storage_policy_index\u0027: 0,"},{"line_number":69,"context_line":"            \u0027content_type\u0027: ASYNC_DELETE_TYPE,"},{"line_number":70,"context_line":"        } for obj in objects]"},{"line_number":71,"context_line":""},{"line_number":72,"context_line":""},{"line_number":73,"context_line":"def mark_for_deletion(swift, account, container, marker, end_marker,"}],"source_content_type":"text/x-python","patch_set":6,"id":"bfb3d3c7_b50eee38","line":70,"in_reply_to":"bfb3d3c7_0f124db2","updated":"2019-05-22 22:23:59.000000000","message":"Yeah... in the s3api follow-up, I just imported from here -- but it definitely felt weird importing from swift.cli in swift.common.middleware.s3api","commit_id":"2da6313f83b1b98e039844046bf8b3ff4c28c465"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"87bf07abc8ee1ca104138e7c9db7c72e05deb71a","unresolved":false,"context_lines":[{"line_number":155,"context_line":"        help\u003d\u0027only delete objects before this end-marker (default: none)\u0027)"},{"line_number":156,"context_line":"    parser.add_argument("},{"line_number":157,"context_line":"        \u0027--timestamp\u0027, type\u003dTimestamp, default\u003dTimestamp.now(),"},{"line_number":158,"context_line":"        help\u003d\u0027delete all objects as of this time (default: now)\u0027)"},{"line_number":159,"context_line":"    args \u003d parser.parse_args()"},{"line_number":160,"context_line":""},{"line_number":161,"context_line":"    swift \u003d InternalClient("}],"source_content_type":"text/x-python","patch_set":6,"id":"bfb3d3c7_6fdbc91a","line":158,"updated":"2019-05-22 20:26:53.000000000","message":"we make no effort on the operator\u0027s behalf to \"bucketize\" the container like the x-delete-at updates do\n\nthis is correct, and flexible - but not very sophisticated\n\nI LIKE IT!","commit_id":"2da6313f83b1b98e039844046bf8b3ff4c28c465"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"f03f8f3b9ed2533ed6eeaeeac44e458fa02a4c3f","unresolved":false,"context_lines":[{"line_number":167,"context_line":"            print(\u0027Finished. Marked %d objects for deletion.\u0027 % deleted)"},{"line_number":168,"context_line":"        else:"},{"line_number":169,"context_line":"            print(\u0027Marked %d objects for deletion, through %r\u0027 % ("},{"line_number":170,"context_line":"                deleted, marker))"},{"line_number":171,"context_line":""},{"line_number":172,"context_line":""},{"line_number":173,"context_line":"if __name__ \u003d\u003d \u0027__main__\u0027:"}],"source_content_type":"text/x-python","patch_set":7,"id":"bfb3d3c7_b574ae69","line":170,"updated":"2019-05-22 21:23:31.000000000","message":"unittests don\u0027t cover main","commit_id":"83d01619910c34f2e3fafeda20bc4f7e7e8f482f"}],"swift/container/server.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"297e2c5320209880baa7d361541d8cf62326a474","unresolved":false,"context_lines":[{"line_number":765,"context_line":"            return HTTPInsufficientStorage(drive\u003ddrive, request\u003dreq)"},{"line_number":766,"context_line":"        if not self.check_free_space(drive):"},{"line_number":767,"context_line":"            return HTTPInsufficientStorage(drive\u003ddrive, request\u003dreq)"},{"line_number":768,"context_line":"        self.logger.info(dict(req.headers))"},{"line_number":769,"context_line":"        requested_policy_index \u003d self.get_and_validate_policy_index(req)"},{"line_number":770,"context_line":"        broker \u003d self._get_container_broker(drive, part, account, container)"},{"line_number":771,"context_line":"        self._maybe_autocreate(broker, req_timestamp, account,"}],"source_content_type":"text/x-python","patch_set":1,"id":"dfbec78f_80d311d5","line":768,"range":{"start_line":768,"start_character":8,"end_line":768,"end_character":43},"updated":"2019-05-14 21:27:26.000000000","message":"nix this; left over from debugging.","commit_id":"0ff4fe3d248627b987f589584608139365944dce"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"06ebc89d8abc5edfe8eac9026c0e271c8d6bd7f1","unresolved":false,"context_lines":[{"line_number":765,"context_line":"            return HTTPInsufficientStorage(drive\u003ddrive, request\u003dreq)"},{"line_number":766,"context_line":"        if not self.check_free_space(drive):"},{"line_number":767,"context_line":"            return HTTPInsufficientStorage(drive\u003ddrive, request\u003dreq)"},{"line_number":768,"context_line":"        self.logger.info(dict(req.headers))"},{"line_number":769,"context_line":"        requested_policy_index \u003d self.get_and_validate_policy_index(req)"},{"line_number":770,"context_line":"        broker \u003d self._get_container_broker(drive, part, account, container)"},{"line_number":771,"context_line":"        self._maybe_autocreate(broker, req_timestamp, account,"}],"source_content_type":"text/x-python","patch_set":1,"id":"dfbec78f_5b0bcc7b","line":768,"range":{"start_line":768,"start_character":8,"end_line":768,"end_character":43},"in_reply_to":"dfbec78f_80d311d5","updated":"2019-05-14 23:49:01.000000000","message":"Done","commit_id":"0ff4fe3d248627b987f589584608139365944dce"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"297e2c5320209880baa7d361541d8cf62326a474","unresolved":false,"context_lines":[{"line_number":774,"context_line":"            args \u003d json.load(req.environ[\u0027wsgi.input\u0027])"},{"line_number":775,"context_line":"        except ValueError as err:"},{"line_number":776,"context_line":"            return HTTPBadRequest(body\u003dstr(err), content_type\u003d\u0027text/plain\u0027)"},{"line_number":777,"context_line":"        self.replicator_rpc.merge_items(broker, (args, None))"},{"line_number":778,"context_line":"        return HTTPAccepted(request\u003dreq)"},{"line_number":779,"context_line":""},{"line_number":780,"context_line":"    @public"}],"source_content_type":"text/x-python","patch_set":1,"id":"dfbec78f_408759b6","line":777,"range":{"start_line":777,"start_character":55,"end_line":777,"end_character":59},"updated":"2019-05-14 21:27:26.000000000","message":"Maybe clarify that this None is in reference to the source? Or maybe I should just say\n\n broker.merge_items(args)\n\nIs the replicator_rpc buying me much?","commit_id":"0ff4fe3d248627b987f589584608139365944dce"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"06ebc89d8abc5edfe8eac9026c0e271c8d6bd7f1","unresolved":false,"context_lines":[{"line_number":774,"context_line":"            args \u003d json.load(req.environ[\u0027wsgi.input\u0027])"},{"line_number":775,"context_line":"        except ValueError as err:"},{"line_number":776,"context_line":"            return HTTPBadRequest(body\u003dstr(err), content_type\u003d\u0027text/plain\u0027)"},{"line_number":777,"context_line":"        self.replicator_rpc.merge_items(broker, (args, None))"},{"line_number":778,"context_line":"        return HTTPAccepted(request\u003dreq)"},{"line_number":779,"context_line":""},{"line_number":780,"context_line":"    @public"}],"source_content_type":"text/x-python","patch_set":1,"id":"dfbec78f_7b1010a9","line":777,"range":{"start_line":777,"start_character":55,"end_line":777,"end_character":59},"in_reply_to":"dfbec78f_408759b6","updated":"2019-05-14 23:49:01.000000000","message":"Done","commit_id":"0ff4fe3d248627b987f589584608139365944dce"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"87bf07abc8ee1ca104138e7c9db7c72e05deb71a","unresolved":false,"context_lines":[{"line_number":756,"context_line":"    def UPDATE(self, req):"},{"line_number":757,"context_line":"        \"\"\""},{"line_number":758,"context_line":"        Handle HTTP UPDATE request (merge_items RPCs coming from the proxy.)"},{"line_number":759,"context_line":"        \"\"\""},{"line_number":760,"context_line":"        drive, part, account, container \u003d split_and_validate_path(req, 4)"},{"line_number":761,"context_line":"        req_timestamp \u003d valid_timestamp(req)"},{"line_number":762,"context_line":"        try:"}],"source_content_type":"text/x-python","patch_set":6,"id":"bfb3d3c7_af6d4133","line":759,"updated":"2019-05-22 20:26:53.000000000","message":"neato!\n\n\tvagrant@saio:~/s3compat$ curl -XOPTIONS http://localhost:6011 -I\n\tHTTP/1.1 200 OK\n\tContent-Type: text/html; charset\u003dUTF-8\n\tAllow: DELETE, GET, HEAD, OPTIONS, POST, PUT, REPLICATE, UPDATE\n\tServer: container-server/2.22.0.dev102\n\tContent-Length: 0\n\tDate: Wed, 22 May 2019 20:06:58 GMT","commit_id":"2da6313f83b1b98e039844046bf8b3ff4c28c465"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a5c32db96b241f43e97cf1e43fcced8da55f9a5a","unresolved":false,"context_lines":[{"line_number":756,"context_line":"    def UPDATE(self, req):"},{"line_number":757,"context_line":"        \"\"\""},{"line_number":758,"context_line":"        Handle HTTP UPDATE request (merge_items RPCs coming from the proxy.)"},{"line_number":759,"context_line":"        \"\"\""},{"line_number":760,"context_line":"        drive, part, account, container \u003d split_and_validate_path(req, 4)"},{"line_number":761,"context_line":"        req_timestamp \u003d valid_timestamp(req)"},{"line_number":762,"context_line":"        try:"}],"source_content_type":"text/x-python","patch_set":6,"id":"bfb3d3c7_6003d60e","line":759,"in_reply_to":"bfb3d3c7_af6d4133","updated":"2019-05-22 22:23:59.000000000","message":"Yeah, I figured the container-server can \u0026 should advertise it... but the proxy (as a general rule) shouldn\u0027t.","commit_id":"2da6313f83b1b98e039844046bf8b3ff4c28c465"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"87bf07abc8ee1ca104138e7c9db7c72e05deb71a","unresolved":false,"context_lines":[{"line_number":774,"context_line":"            objs \u003d json.load(req.environ[\u0027wsgi.input\u0027])"},{"line_number":775,"context_line":"        except ValueError as err:"},{"line_number":776,"context_line":"            return HTTPBadRequest(body\u003dstr(err), content_type\u003d\u0027text/plain\u0027)"},{"line_number":777,"context_line":"        broker.merge_items(objs)"},{"line_number":778,"context_line":"        return HTTPAccepted(request\u003dreq)"},{"line_number":779,"context_line":""},{"line_number":780,"context_line":"    @public"}],"source_content_type":"text/x-python","patch_set":6,"id":"bfb3d3c7_2ff1516c","line":777,"updated":"2019-05-22 20:26:53.000000000","message":"this is pushed into a threadpool in _really_really_merge_items :heart:\n\nit\u0027s kinda weird that we don\u0027t put a timeout on anything in REPLICATE on the server side... but w/o some prior art I can\u0027t recommend a default here, or even advocate that it\u0027s needed...","commit_id":"2da6313f83b1b98e039844046bf8b3ff4c28c465"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a5c32db96b241f43e97cf1e43fcced8da55f9a5a","unresolved":false,"context_lines":[{"line_number":774,"context_line":"            objs \u003d json.load(req.environ[\u0027wsgi.input\u0027])"},{"line_number":775,"context_line":"        except ValueError as err:"},{"line_number":776,"context_line":"            return HTTPBadRequest(body\u003dstr(err), content_type\u003d\u0027text/plain\u0027)"},{"line_number":777,"context_line":"        broker.merge_items(objs)"},{"line_number":778,"context_line":"        return HTTPAccepted(request\u003dreq)"},{"line_number":779,"context_line":""},{"line_number":780,"context_line":"    @public"}],"source_content_type":"text/x-python","patch_set":6,"id":"bfb3d3c7_e00ee630","line":777,"in_reply_to":"bfb3d3c7_2ff1516c","updated":"2019-05-22 22:23:59.000000000","message":"Best of all, I got it for free :-)\n\nAgreed that a timeout might be good... maybe the replicator client timeout would be a good point of reference?","commit_id":"2da6313f83b1b98e039844046bf8b3ff4c28c465"}],"swift/obj/expirer.py":[{"author":{"_account_id":25251,"name":"Alexandre Lécuyer","email":"alexandre.lecuyer@corp.ovh.com","username":"alecuyer"},"change_message_id":"5dc925fe9540cca47e1cd5b7d9663842063a3389","unresolved":false,"context_lines":[{"line_number":150,"context_line":"            self.report_last_time \u003d time()"},{"line_number":151,"context_line":""},{"line_number":152,"context_line":"    def parse_task_obj(self, task_obj):"},{"line_number":153,"context_line":"        return parse_task_obj(task_obj)"},{"line_number":154,"context_line":""},{"line_number":155,"context_line":"    def round_robin_order(self, task_iter):"},{"line_number":156,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":3,"id":"bfb3d3c7_e8ce2882","line":153,"updated":"2019-05-17 13:07:06.000000000","message":"This is in the class for testing? (I the expirer otherwise calls the function directly)","commit_id":"aaa0e60541ff0b42097d8b2922ac666df61cb21f"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6597aab4173564b3a723cbfc0096871994d046a7","unresolved":false,"context_lines":[{"line_number":150,"context_line":"            self.report_last_time \u003d time()"},{"line_number":151,"context_line":""},{"line_number":152,"context_line":"    def parse_task_obj(self, task_obj):"},{"line_number":153,"context_line":"        return parse_task_obj(task_obj)"},{"line_number":154,"context_line":""},{"line_number":155,"context_line":"    def round_robin_order(self, task_iter):"},{"line_number":156,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":3,"id":"bfb3d3c7_1a5f3b42","line":153,"in_reply_to":"bfb3d3c7_e8ce2882","updated":"2019-05-17 16:27:49.000000000","message":"Yeah, some tests were expecting parse_task_obj to hang off the expirer instance -- I debated about just fixing the tests, but then I remembered how https://review.opendev.org/#/c/517389/ is planning to change parse_task_obj() and figured this shim would be preferable.","commit_id":"aaa0e60541ff0b42097d8b2922ac666df61cb21f"},{"author":{"_account_id":25251,"name":"Alexandre Lécuyer","email":"alexandre.lecuyer@corp.ovh.com","username":"alecuyer"},"change_message_id":"5dc925fe9540cca47e1cd5b7d9663842063a3389","unresolved":false,"context_lines":[{"line_number":446,"context_line":"        direct_delete_container_entry(self.swift.container_ring, task_account,"},{"line_number":447,"context_line":"                                      task_container, task_object)"},{"line_number":448,"context_line":""},{"line_number":449,"context_line":"    def delete_actual_object(self, actual_obj, timestamp, is_bulk_delete):"},{"line_number":450,"context_line":"        \"\"\""},{"line_number":451,"context_line":"        Deletes the end-user object indicated by the actual object name given"},{"line_number":452,"context_line":"        \u0027\u003caccount\u003e/\u003ccontainer\u003e/\u003cobject\u003e\u0027 if and only if the X-Delete-At value"}],"source_content_type":"text/x-python","patch_set":3,"id":"bfb3d3c7_3e34764a","line":449,"updated":"2019-05-17 13:07:06.000000000","message":"Maybe update the docstring with regards to \"is_bulk_delete\" ?","commit_id":"aaa0e60541ff0b42097d8b2922ac666df61cb21f"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6597aab4173564b3a723cbfc0096871994d046a7","unresolved":false,"context_lines":[{"line_number":446,"context_line":"        direct_delete_container_entry(self.swift.container_ring, task_account,"},{"line_number":447,"context_line":"                                      task_container, task_object)"},{"line_number":448,"context_line":""},{"line_number":449,"context_line":"    def delete_actual_object(self, actual_obj, timestamp, is_bulk_delete):"},{"line_number":450,"context_line":"        \"\"\""},{"line_number":451,"context_line":"        Deletes the end-user object indicated by the actual object name given"},{"line_number":452,"context_line":"        \u0027\u003caccount\u003e/\u003ccontainer\u003e/\u003cobject\u003e\u0027 if and only if the X-Delete-At value"}],"source_content_type":"text/x-python","patch_set":3,"id":"bfb3d3c7_da58c357","line":449,"in_reply_to":"bfb3d3c7_3e34764a","updated":"2019-05-17 16:27:49.000000000","message":"#willfix","commit_id":"aaa0e60541ff0b42097d8b2922ac666df61cb21f"},{"author":{"_account_id":25251,"name":"Alexandre Lécuyer","email":"alexandre.lecuyer@corp.ovh.com","username":"alecuyer"},"change_message_id":"5c1d62fd4ad1da79d315ce5bee2535cfd5e2ce55","unresolved":false,"context_lines":[{"line_number":264,"context_line":"                                 divisor) !\u003d my_index:"},{"line_number":265,"context_line":"                    continue"},{"line_number":266,"context_line":""},{"line_number":267,"context_line":"                is_async \u003d o.get(\u0027content_type\u0027) \u003d\u003d ASYNC_DELETE_TYPE"},{"line_number":268,"context_line":"                yield {\u0027task_account\u0027: task_account,"},{"line_number":269,"context_line":"                       \u0027task_container\u0027: task_container,"},{"line_number":270,"context_line":"                       \u0027task_object\u0027: task_object,"}],"source_content_type":"text/x-python","patch_set":4,"id":"bfb3d3c7_2d92f9f7","line":267,"updated":"2019-05-20 12:01:34.000000000","message":"Good idea to rename from \"bulk\" to \"async\" I think, the link is more obvious now with the rest of the patch","commit_id":"0ead11bd38012a47608eddbeec5e73c4831af764"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a5340f7af6e16ec749f55320105a4f566f26476a","unresolved":false,"context_lines":[{"line_number":264,"context_line":"                                 divisor) !\u003d my_index:"},{"line_number":265,"context_line":"                    continue"},{"line_number":266,"context_line":""},{"line_number":267,"context_line":"                is_async \u003d o.get(\u0027content_type\u0027) \u003d\u003d ASYNC_DELETE_TYPE"},{"line_number":268,"context_line":"                yield {\u0027task_account\u0027: task_account,"},{"line_number":269,"context_line":"                       \u0027task_container\u0027: task_container,"},{"line_number":270,"context_line":"                       \u0027task_object\u0027: task_object,"}],"source_content_type":"text/x-python","patch_set":4,"id":"bfb3d3c7_1a215efc","line":267,"in_reply_to":"bfb3d3c7_2d92f9f7","updated":"2019-05-20 17:05:53.000000000","message":"Yeah, that was a hold-over from the s3api bulk patch.","commit_id":"0ead11bd38012a47608eddbeec5e73c4831af764"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"d367e1cb6a08adab3b50266a3f5d512955358269","unresolved":false,"context_lines":[{"line_number":62,"context_line":"    timestamp \u003d Timestamp(timestamp)"},{"line_number":63,"context_line":"    target_account, target_container, target_obj \u003d \\"},{"line_number":64,"context_line":"        split_path(\u0027/\u0027 + target_path, 3, 3, True)"},{"line_number":65,"context_line":"    return timestamp, target_account, target_container, target_obj"},{"line_number":66,"context_line":""},{"line_number":67,"context_line":""},{"line_number":68,"context_line":"class ObjectExpirer(Daemon):"}],"source_content_type":"text/x-python","patch_set":5,"id":"bfb3d3c7_16d13655","line":65,"updated":"2019-05-20 18:42:41.000000000","message":"nice!\n\nI\u0027d love to see one of those cute tests that show:\n\n    input \u003d\u003d parse(build(input))","commit_id":"84b63da021dfb8c719e1c4616ea0aa426844cba4"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"d367e1cb6a08adab3b50266a3f5d512955358269","unresolved":false,"context_lines":[{"line_number":465,"context_line":"        \"\"\""},{"line_number":466,"context_line":"        path \u003d \u0027/v1/\u0027 + urllib.parse.quote(actual_obj.lstrip(\u0027/\u0027))"},{"line_number":467,"context_line":"        headers \u003d {\u0027X-Timestamp\u0027: timestamp.normal}"},{"line_number":468,"context_line":"        acceptable_statuses \u003d (2, HTTP_CONFLICT, HTTP_NOT_FOUND)"},{"line_number":469,"context_line":"        if not is_async_delete:"},{"line_number":470,"context_line":"            headers.update({"},{"line_number":471,"context_line":"                \u0027X-If-Delete-At\u0027: timestamp.normal,"}],"source_content_type":"text/x-python","patch_set":5,"id":"bfb3d3c7_76f0b2ca","line":468,"updated":"2019-05-20 18:42:41.000000000","message":"I guess the 404 here is desirable, since the object-server will get the request (the container can\u0027t 404 if it has objects) and w/o the if-delete-at it WILL write a tombstone - always.","commit_id":"84b63da021dfb8c719e1c4616ea0aa426844cba4"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"d367e1cb6a08adab3b50266a3f5d512955358269","unresolved":false,"context_lines":[{"line_number":470,"context_line":"            headers.update({"},{"line_number":471,"context_line":"                \u0027X-If-Delete-At\u0027: timestamp.normal,"},{"line_number":472,"context_line":"                \u0027X-Backend-Clean-Expiring-Object-Queue\u0027: \u0027no\u0027})"},{"line_number":473,"context_line":"            acceptable_statuses \u003d (2, HTTP_CONFLICT)"},{"line_number":474,"context_line":"        self.swift.make_request(\u0027DELETE\u0027, path, headers, acceptable_statuses)"}],"source_content_type":"text/x-python","patch_set":5,"id":"bfb3d3c7_16b056f4","line":473,"updated":"2019-05-20 18:42:41.000000000","message":"something about the order/overwriting here smells to me...\n\nI\u0027m think as long as make_request doesn\u0027t *require* that acceptable_statuses be an immutable/tuple we could reword this - but I don\u0027t know if it\u0027d be much better","commit_id":"84b63da021dfb8c719e1c4616ea0aa426844cba4"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"94f1537cdf4eca55b0be63a25358f0f87d5e9ca7","unresolved":false,"context_lines":[{"line_number":470,"context_line":"            headers.update({"},{"line_number":471,"context_line":"                \u0027X-If-Delete-At\u0027: timestamp.normal,"},{"line_number":472,"context_line":"                \u0027X-Backend-Clean-Expiring-Object-Queue\u0027: \u0027no\u0027})"},{"line_number":473,"context_line":"            acceptable_statuses \u003d (2, HTTP_CONFLICT)"},{"line_number":474,"context_line":"        self.swift.make_request(\u0027DELETE\u0027, path, headers, acceptable_statuses)"}],"source_content_type":"text/x-python","patch_set":5,"id":"bfb3d3c7_f6b7c285","line":473,"in_reply_to":"bfb3d3c7_16b056f4","updated":"2019-05-20 19:07:11.000000000","message":"Yeah, either way it gets kinda weird. Maybe the best bet would be to be explicit with something like\n\n if is_async_delete:\n     headers \u003d ...\n     acceptable_statuses \u003d ...\n else:\n     headers \u003d ...\n     acceptable_statuses \u003d ...","commit_id":"84b63da021dfb8c719e1c4616ea0aa426844cba4"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"dfa6b8a7511e30fe388fd46f9a02daac4fdb8c1d","unresolved":false,"context_lines":[{"line_number":470,"context_line":"            headers.update({"},{"line_number":471,"context_line":"                \u0027X-If-Delete-At\u0027: timestamp.normal,"},{"line_number":472,"context_line":"                \u0027X-Backend-Clean-Expiring-Object-Queue\u0027: \u0027no\u0027})"},{"line_number":473,"context_line":"            acceptable_statuses \u003d (2, HTTP_CONFLICT)"},{"line_number":474,"context_line":"        self.swift.make_request(\u0027DELETE\u0027, path, headers, acceptable_statuses)"}],"source_content_type":"text/x-python","patch_set":5,"id":"bfb3d3c7_5a46da23","line":473,"in_reply_to":"bfb3d3c7_f6b7c285","updated":"2019-05-21 20:21:36.000000000","message":"Done","commit_id":"84b63da021dfb8c719e1c4616ea0aa426844cba4"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"f03f8f3b9ed2533ed6eeaeeac44e458fa02a4c3f","unresolved":false,"context_lines":[{"line_number":48,"context_line":"    timestamp \u003d Timestamp(timestamp)"},{"line_number":49,"context_line":"    return \u0027%s-%s/%s/%s\u0027 % ("},{"line_number":50,"context_line":"        normalize_delete_at_timestamp(timestamp),"},{"line_number":51,"context_line":"        target_account, target_container, target_obj)"},{"line_number":52,"context_line":""},{"line_number":53,"context_line":""},{"line_number":54,"context_line":"def parse_task_obj(task_obj):"}],"source_content_type":"text/x-python","patch_set":7,"id":"bfb3d3c7_f2e79ce3","line":51,"updated":"2019-05-22 21:23:31.000000000","message":"name error in this helper doesn\u0027t blow up when running unittests for this module\n\n\t@@ -931,6 +985,15 @@ class TestObjectExpirer(TestCase):\n\t\t     self.assertEqual(container, \u0027c\u0027)\n\t\t     self.assertEqual(obj, \u0027o\u0027)\n\t \n\t+    def test_build_task_obj_round_trip(self):\n\t+        ts \u003d next(self.ts)\n\t+        a \u003d \u0027a1\u0027\n\t+        c \u003d \u0027c2\u0027\n\t+        o \u003d \u0027obj1\u0027\n\t+        args \u003d (ts, a, c, o)\n\t+        self.assertEqual(args, expirer.parse_task_obj(\n\t+            expirer.build_task_obj(ts, a, c, o)))\n\t+\n\t \n\t if __name__ \u003d\u003d \u0027__main__\u0027:\n\t     main()","commit_id":"83d01619910c34f2e3fafeda20bc4f7e7e8f482f"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"f03f8f3b9ed2533ed6eeaeeac44e458fa02a4c3f","unresolved":false,"context_lines":[{"line_number":477,"context_line":"            headers \u003d {\u0027X-Timestamp\u0027: timestamp.normal,"},{"line_number":478,"context_line":"                       \u0027X-If-Delete-At\u0027: timestamp.normal,"},{"line_number":479,"context_line":"                       \u0027X-Backend-Clean-Expiring-Object-Queue\u0027: \u0027no\u0027}"},{"line_number":480,"context_line":"            acceptable_statuses \u003d (2, HTTP_CONFLICT)"},{"line_number":481,"context_line":"        self.swift.make_request(\u0027DELETE\u0027, path, headers, acceptable_statuses)"}],"source_content_type":"text/x-python","patch_set":7,"id":"bfb3d3c7_d2ccd862","line":480,"updated":"2019-05-22 21:23:31.000000000","message":"looks great!","commit_id":"83d01619910c34f2e3fafeda20bc4f7e7e8f482f"}],"swift/proxy/server.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"297e2c5320209880baa7d361541d8cf62326a474","unresolved":false,"context_lines":[{"line_number":507,"context_line":"            controller.trans_id \u003d req.environ[\u0027swift.trans_id\u0027]"},{"line_number":508,"context_line":"            self.logger.client_ip \u003d get_remote_client(req)"},{"line_number":509,"context_line":""},{"line_number":510,"context_line":"            allowed_methods \u003d controller.allowed_methods"},{"line_number":511,"context_line":"            if \u0027X-Backend-Allow-Method\u0027 in req.headers:"},{"line_number":512,"context_line":"                allowed_methods.add(req.headers[\u0027X-Backend-Allow-Method\u0027])"},{"line_number":513,"context_line":"            if req.method not in allowed_methods:"}],"source_content_type":"text/x-python","patch_set":1,"id":"dfbec78f_806f91fa","line":510,"range":{"start_line":510,"start_character":30,"end_line":510,"end_character":56},"updated":"2019-05-14 21:27:26.000000000","message":"We should wrap this in a set(...), to ensure we don\u0027t modify the controller\u0027s cached set.","commit_id":"0ff4fe3d248627b987f589584608139365944dce"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"06ebc89d8abc5edfe8eac9026c0e271c8d6bd7f1","unresolved":false,"context_lines":[{"line_number":507,"context_line":"            controller.trans_id \u003d req.environ[\u0027swift.trans_id\u0027]"},{"line_number":508,"context_line":"            self.logger.client_ip \u003d get_remote_client(req)"},{"line_number":509,"context_line":""},{"line_number":510,"context_line":"            allowed_methods \u003d controller.allowed_methods"},{"line_number":511,"context_line":"            if \u0027X-Backend-Allow-Method\u0027 in req.headers:"},{"line_number":512,"context_line":"                allowed_methods.add(req.headers[\u0027X-Backend-Allow-Method\u0027])"},{"line_number":513,"context_line":"            if req.method not in allowed_methods:"}],"source_content_type":"text/x-python","patch_set":1,"id":"dfbec78f_1b29d4e4","line":510,"range":{"start_line":510,"start_character":30,"end_line":510,"end_character":56},"in_reply_to":"dfbec78f_806f91fa","updated":"2019-05-14 23:49:01.000000000","message":"Done","commit_id":"0ff4fe3d248627b987f589584608139365944dce"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"d367e1cb6a08adab3b50266a3f5d512955358269","unresolved":false,"context_lines":[{"line_number":509,"context_line":""},{"line_number":510,"context_line":"            allowed_methods \u003d set(controller.allowed_methods)"},{"line_number":511,"context_line":"            if \u0027X-Backend-Allow-Method\u0027 in req.headers:"},{"line_number":512,"context_line":"                allowed_methods.add(req.headers[\u0027X-Backend-Allow-Method\u0027])"},{"line_number":513,"context_line":"            if req.method not in allowed_methods:"},{"line_number":514,"context_line":"                return HTTPMethodNotAllowed(request\u003dreq, headers\u003d{"},{"line_number":515,"context_line":"                    # Only advertise the *controller\u0027s* allowed_methods"}],"source_content_type":"text/x-python","patch_set":5,"id":"bfb3d3c7_560a4ed2","line":512,"updated":"2019-05-20 18:42:41.000000000","message":"this is an interesting new interface...","commit_id":"84b63da021dfb8c719e1c4616ea0aa426844cba4"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"94f1537cdf4eca55b0be63a25358f0f87d5e9ca7","unresolved":false,"context_lines":[{"line_number":509,"context_line":""},{"line_number":510,"context_line":"            allowed_methods \u003d set(controller.allowed_methods)"},{"line_number":511,"context_line":"            if \u0027X-Backend-Allow-Method\u0027 in req.headers:"},{"line_number":512,"context_line":"                allowed_methods.add(req.headers[\u0027X-Backend-Allow-Method\u0027])"},{"line_number":513,"context_line":"            if req.method not in allowed_methods:"},{"line_number":514,"context_line":"                return HTTPMethodNotAllowed(request\u003dreq, headers\u003d{"},{"line_number":515,"context_line":"                    # Only advertise the *controller\u0027s* allowed_methods"}],"source_content_type":"text/x-python","patch_set":5,"id":"bfb3d3c7_36955ae5","line":512,"in_reply_to":"bfb3d3c7_560a4ed2","updated":"2019-05-20 19:07:11.000000000","message":"The question is, is it a good one? Or at any rate, *good enough*?","commit_id":"84b63da021dfb8c719e1c4616ea0aa426844cba4"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"f03f8f3b9ed2533ed6eeaeeac44e458fa02a4c3f","unresolved":false,"context_lines":[{"line_number":509,"context_line":""},{"line_number":510,"context_line":"            allowed_methods \u003d set(controller.allowed_methods)"},{"line_number":511,"context_line":"            if \u0027X-Backend-Allow-Method\u0027 in req.headers:"},{"line_number":512,"context_line":"                allowed_methods.add(req.headers[\u0027X-Backend-Allow-Method\u0027])"},{"line_number":513,"context_line":"            if req.method not in allowed_methods:"},{"line_number":514,"context_line":"                return HTTPMethodNotAllowed(request\u003dreq, headers\u003d{"},{"line_number":515,"context_line":"                    # Only advertise the *controller\u0027s* allowed_methods"}],"source_content_type":"text/x-python","patch_set":7,"id":"bfb3d3c7_527d48e5","line":512,"updated":"2019-05-22 21:23:31.000000000","message":"So this is like a double safety, you have to use an unallowed method and also set the unallowed method header.\n\nIt\u0027s not obvious to me that it should be a per-request option vs. per-context (let instances of this internal-client call UPDATE) but it seems really tricky to get internal client to plumb an update through the proxy.server app \u003d\u003e the controller \u003d\u003e the allowed_methods property\n\nI think it should just be a boolean?\n\n\tdiff --git a/swift/proxy/server.py b/swift/proxy/server.py\n\tindex 082a6d2f7..cf3b8df18 100644\n\t--- a/swift/proxy/server.py\n\t+++ b/swift/proxy/server.py\n\t@@ -510,7 +510,8 @@ class Application(object):\n\t\t     allowed_methods \u003d set(controller.allowed_methods)\n\t\t     if \u0027X-Backend-Allow-Method\u0027 in req.headers:\n\t\t\t allowed_methods.add(req.headers[\u0027X-Backend-Allow-Method\u0027])\n\t-            if req.method not in allowed_methods:\n\t+            if req.method not in allowed_methods and not config_true_value(\n\t+                    req.headers.get(\u0027X-Backend-Allow-Method\u0027, False)):\n\t\t\t return HTTPMethodNotAllowed(request\u003dreq, headers\u003d{\n\t\t\t     # Only advertise the *controller\u0027s* allowed_methods\n\t\t\t     \u0027Allow\u0027: \u0027, \u0027.join(controller.allowed_methods)})","commit_id":"83d01619910c34f2e3fafeda20bc4f7e7e8f482f"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a5c32db96b241f43e97cf1e43fcced8da55f9a5a","unresolved":false,"context_lines":[{"line_number":509,"context_line":""},{"line_number":510,"context_line":"            allowed_methods \u003d set(controller.allowed_methods)"},{"line_number":511,"context_line":"            if \u0027X-Backend-Allow-Method\u0027 in req.headers:"},{"line_number":512,"context_line":"                allowed_methods.add(req.headers[\u0027X-Backend-Allow-Method\u0027])"},{"line_number":513,"context_line":"            if req.method not in allowed_methods:"},{"line_number":514,"context_line":"                return HTTPMethodNotAllowed(request\u003dreq, headers\u003d{"},{"line_number":515,"context_line":"                    # Only advertise the *controller\u0027s* allowed_methods"}],"source_content_type":"text/x-python","patch_set":7,"id":"bfb3d3c7_804b6a57","line":512,"in_reply_to":"bfb3d3c7_527d48e5","updated":"2019-05-22 22:23:59.000000000","message":"Maybe it\u0027d be better with a @private decorator for methods like UPDATE, and you need to set X-Backend-Allow-Private-Methods: True to get to them? Then you get the benefits of a whitelist, including\n\n* being able to reason more easily about which methods to advertise when and\n* preventing someone from doing something screwy like\n\n     ic.make_request(method\u003d\u0027_backend_requests\u0027, headers\u003d{\n         \u0027X-Backend-Allow-Method\u0027: \u0027_backend_requests\u0027})\n\nThis certainly seemed more expedient/easier to implement, though.\n\nI think I agree that having it set for the whole client would be better than per-request, though I\u0027m not sure how best to accomplish that.\n\nIf we stuck with something like this, maybe it\u0027d be interesting to specify a default set of headers for an internal-client? Like, \"I know I\u0027m going to be doing a bunch of container creates; they should all have this storage policy.\"","commit_id":"83d01619910c34f2e3fafeda20bc4f7e7e8f482f"}],"test/unit/cli/test_container_deleter.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"f03f8f3b9ed2533ed6eeaeeac44e458fa02a4c3f","unresolved":false,"context_lines":[{"line_number":53,"context_line":"            raise Exception(\u0027Unused responses: %r\u0027 % unused_responses)"},{"line_number":54,"context_line":""},{"line_number":55,"context_line":""},{"line_number":56,"context_line":"class TestContainerDeleter(unittest.TestCase):"},{"line_number":57,"context_line":"    def setUp(self):"},{"line_number":58,"context_line":"        patcher \u003d mock.patch.object(container_deleter.time, \u0027time\u0027,"},{"line_number":59,"context_line":"                                    side_effect\u003ditertools.count())"}],"source_content_type":"text/x-python","patch_set":7,"id":"bfb3d3c7_f55a26f1","line":56,"updated":"2019-05-22 21:23:31.000000000","message":"these tests are great!","commit_id":"83d01619910c34f2e3fafeda20bc4f7e7e8f482f"}],"test/unit/obj/test_expirer.py":[{"author":{"_account_id":25251,"name":"Alexandre Lécuyer","email":"alexandre.lecuyer@corp.ovh.com","username":"alecuyer"},"change_message_id":"5c1d62fd4ad1da79d315ce5bee2535cfd5e2ce55","unresolved":false,"context_lines":[{"line_number":252,"context_line":""},{"line_number":253,"context_line":"            def delete_object(self, target_path, delete_timestamp,"},{"line_number":254,"context_line":"                              task_account, task_container, task_object,"},{"line_number":255,"context_line":"                              is_bulk_delete):"},{"line_number":256,"context_line":"                if task_container not in self.deleted_objects:"},{"line_number":257,"context_line":"                    self.deleted_objects[task_container] \u003d set()"},{"line_number":258,"context_line":"                self.deleted_objects[task_container].add(task_object)"}],"source_content_type":"text/x-python","patch_set":4,"id":"bfb3d3c7_33ac968b","line":255,"updated":"2019-05-20 12:01:34.000000000","message":"Should be is_async_delete to match your latest changes (similar case below)","commit_id":"0ead11bd38012a47608eddbeec5e73c4831af764"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a5340f7af6e16ec749f55320105a4f566f26476a","unresolved":false,"context_lines":[{"line_number":252,"context_line":""},{"line_number":253,"context_line":"            def delete_object(self, target_path, delete_timestamp,"},{"line_number":254,"context_line":"                              task_account, task_container, task_object,"},{"line_number":255,"context_line":"                              is_bulk_delete):"},{"line_number":256,"context_line":"                if task_container not in self.deleted_objects:"},{"line_number":257,"context_line":"                    self.deleted_objects[task_container] \u003d set()"},{"line_number":258,"context_line":"                self.deleted_objects[task_container].add(task_object)"}],"source_content_type":"text/x-python","patch_set":4,"id":"bfb3d3c7_ba87f2d7","line":255,"in_reply_to":"bfb3d3c7_33ac968b","updated":"2019-05-20 17:05:53.000000000","message":"That\u0027s what I get for not running unit tests before submitting!\n\nOh, right -- we need https://review.opendev.org/#/c/659189/ to be able to run them on py3... that\u0027s probably why I didn\u0027t bother :-/","commit_id":"0ead11bd38012a47608eddbeec5e73c4831af764"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"87bf07abc8ee1ca104138e7c9db7c72e05deb71a","unresolved":false,"context_lines":[{"line_number":589,"context_line":"        self.assertEqual("},{"line_number":590,"context_line":"            list(x.iter_task_to_expire("},{"line_number":591,"context_line":"                task_account_container_list, my_index, divisor)),"},{"line_number":592,"context_line":"            expected)"},{"line_number":593,"context_line":""},{"line_number":594,"context_line":"    def test_run_once_unicode_problem(self):"},{"line_number":595,"context_line":"        requests \u003d []"}],"source_content_type":"text/x-python","patch_set":6,"id":"bfb3d3c7_5251c80d","line":592,"updated":"2019-05-22 20:26:53.000000000","message":"Current we\u0027re don\u0027t have any tests blowing up with this diff:\n\n\t(vagrant-swift-all-in-one-Z1qCc7cJ) clayg@banana:~/Workspace/vagrant-swift-all-in-one/swift$ git diff swift/obj/expirer.py\n\tdiff --git a/swift/obj/expirer.py b/swift/obj/expirer.py\n\tindex 07974b9d6..28c77c987 100644\n\t--- a/swift/obj/expirer.py\n\t+++ b/swift/obj/expirer.py\n\t@@ -264,7 +264,7 @@ class ObjectExpirer(Daemon):\n\t\t\t\t\t  divisor) !\u003d my_index:\n\t\t\t     continue\n\t \n\t-                is_async \u003d o.get(\u0027content_type\u0027) \u003d\u003d ASYNC_DELETE_TYPE\n\t+                is_async \u003d False\n\t\t\t yield {\u0027task_account\u0027: task_account,\n\t\t\t\t\u0027task_container\u0027: task_container,\n\t\t\t\t\u0027task_object\u0027: task_object,\n\n\n\nI think we could expand this test to catch that tho...","commit_id":"2da6313f83b1b98e039844046bf8b3ff4c28c465"}],"test/unit/proxy/test_server.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"d367e1cb6a08adab3b50266a3f5d512955358269","unresolved":false,"context_lines":[{"line_number":684,"context_line":"        # But with appropriate (internal-only) overrides, you can still use it"},{"line_number":685,"context_line":"        resp \u003d baseapp.handle_request("},{"line_number":686,"context_line":"            Request.blank(\u0027/v1/a/c\u0027, environ\u003d{\u0027REQUEST_METHOD\u0027: \u0027UPDATE\u0027},"},{"line_number":687,"context_line":"                          headers\u003d{\u0027X-Backend-Allow-Method\u0027: \u0027UPDATE\u0027,"},{"line_number":688,"context_line":"                                   \u0027X-Backend-Storage-Policy-Index\u0027: \u00270\u0027}))"},{"line_number":689,"context_line":"        # Now we actually make the requests, but there aren\u0027t any nodes"},{"line_number":690,"context_line":"        self.assertEqual(resp.status, \u0027503 Service Unavailable\u0027)"}],"source_content_type":"text/x-python","patch_set":5,"id":"bfb3d3c7_7602f2b6","line":687,"updated":"2019-05-20 18:42:41.000000000","message":"ok, great example of how non-gatekeeper\u0027d requests can use the new \"private method\" interface!","commit_id":"84b63da021dfb8c719e1c4616ea0aa426844cba4"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"f03f8f3b9ed2533ed6eeaeeac44e458fa02a4c3f","unresolved":false,"context_lines":[{"line_number":687,"context_line":"                          headers\u003d{\u0027X-Backend-Allow-Method\u0027: \u0027UPDATE\u0027,"},{"line_number":688,"context_line":"                                   \u0027X-Backend-Storage-Policy-Index\u0027: \u00270\u0027}))"},{"line_number":689,"context_line":"        # Now we actually make the requests, but there aren\u0027t any nodes"},{"line_number":690,"context_line":"        self.assertEqual(resp.status, \u0027503 Service Unavailable\u0027)"},{"line_number":691,"context_line":""},{"line_number":692,"context_line":"    def test_calls_authorize_allow(self):"},{"line_number":693,"context_line":"        called \u003d [False]"}],"source_content_type":"text/x-python","patch_set":7,"id":"bfb3d3c7_f5cc66bd","line":690,"updated":"2019-05-22 21:23:31.000000000","message":"oic, so this fails because the methods we advertise aren\u0027t what we actually allow :\\\n\n\tdiff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py\n\tindex 6a4cf86c1..db344d7d9 100644\n\t--- a/test/unit/proxy/test_server.py\n\t+++ b/test/unit/proxy/test_server.py\n\t@@ -689,6 +689,14 @@ class TestProxyServer(unittest.TestCase):\n\t\t # Now we actually make the requests, but there aren\u0027t any nodes\n\t\t self.assertEqual(resp.status, \u0027503 Service Unavailable\u0027)\n\t \n\t+        # this is weird\n\t+        resp \u003d baseapp.handle_request(\n\t+            Request.blank(\u0027/v1/a/c\u0027, environ\u003d{\u0027REQUEST_METHOD\u0027: \u0027BOGUS\u0027},\n\t+                          headers\u003d{\u0027X-Backend-Allow-Method\u0027: \u0027MOARBOGUS\u0027}))\n\t+        self.assertEqual(resp.status, \u0027405 Method Not Allowed\u0027)\n\t+        self.assertEqual(sorted(resp.headers[\u0027Allow\u0027].split(\u0027, \u0027)), [\n\t+            \u0027DELETE\u0027, \u0027GET\u0027, \u0027HEAD\u0027, \u0027OPTIONS\u0027, \u0027POST\u0027, \u0027PUT\u0027, \u0027MOARBOGUS\u0027])\n\t+\n\t     def test_calls_authorize_allow(self):\n\t\t called \u003d [False]","commit_id":"83d01619910c34f2e3fafeda20bc4f7e7e8f482f"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a5c32db96b241f43e97cf1e43fcced8da55f9a5a","unresolved":false,"context_lines":[{"line_number":687,"context_line":"                          headers\u003d{\u0027X-Backend-Allow-Method\u0027: \u0027UPDATE\u0027,"},{"line_number":688,"context_line":"                                   \u0027X-Backend-Storage-Policy-Index\u0027: \u00270\u0027}))"},{"line_number":689,"context_line":"        # Now we actually make the requests, but there aren\u0027t any nodes"},{"line_number":690,"context_line":"        self.assertEqual(resp.status, \u0027503 Service Unavailable\u0027)"},{"line_number":691,"context_line":""},{"line_number":692,"context_line":"    def test_calls_authorize_allow(self):"},{"line_number":693,"context_line":"        called \u003d [False]"}],"source_content_type":"text/x-python","patch_set":7,"id":"bfb3d3c7_952aca8d","line":690,"in_reply_to":"bfb3d3c7_f5cc66bd","updated":"2019-05-22 22:23:59.000000000","message":"Yeah -- I figure if you\u0027re able to poke at this with X-Backend-* headers, you can read code to suss out what you can actually do ;-)","commit_id":"83d01619910c34f2e3fafeda20bc4f7e7e8f482f"}]}