)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"47333916d4c15d4e7fb1f79a305956b7eaeed590","unresolved":false,"context_lines":[{"line_number":100,"context_line":"https://gist.github.com/gilv/64d43be25d0b4cb8ef29"},{"line_number":101,"context_line":""},{"line_number":102,"context_line":"Change-Id: I3c82f8c0e7eafa3fcfc4385c9a240b14bc766ead"},{"line_number":103,"context_line":"Implements: blueprint on-demand-data-migration-for-swift"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":14,"id":"baada198_be2916f4","line":103,"updated":"2014-08-01 05:54:23.000000000","message":"Because you are adding the sample configuration for users to the proxy-server.conf.sample file, DocImpact should be added to the commit message.\n\nPlus you\u0027ve done a great job documenting all the files, I feel it should be documented for more users to use :)","commit_id":"cd8a0d5ed1e0685111396beaf3d693e136bc5997"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"363fa98d6c1b66cf6a6cc4ba77c57e4db93c9b62","unresolved":false,"context_lines":[{"line_number":79,"context_line":"1. Migration with usage of Apache LibCloud. Current code was"},{"line_number":80,"context_line":"tested to migrate data from Amazon S3 only. For this reason,"},{"line_number":81,"context_line":"currently no other clouds supported, but anyone welcome to"},{"line_number":82,"context_line":"check data migration from other storage clouds."},{"line_number":83,"context_line":""},{"line_number":84,"context_line":"2. Due to code complexity, unified view ( described in blue"},{"line_number":85,"context_line":"print ) is not part of this commit. Will be committed separately."}],"source_content_type":"text/x-gerrit-commit-message","patch_set":16,"id":"baada198_4c3c3193","line":82,"updated":"2014-08-13 15:09:11.000000000","message":"point 1 no longer applies","commit_id":"1e5ddde5fc3bc514d59e66a5c43592cbb502143c"},{"author":{"_account_id":9205,"name":"Gil Vernik","email":"gilv@il.ibm.com","username":"gilv"},"change_message_id":"8f1bbdcf5a4962e0cdf319f9941ca972103cf051","unresolved":false,"context_lines":[{"line_number":79,"context_line":"1. Migration with usage of Apache LibCloud. Current code was"},{"line_number":80,"context_line":"tested to migrate data from Amazon S3 only. For this reason,"},{"line_number":81,"context_line":"currently no other clouds supported, but anyone welcome to"},{"line_number":82,"context_line":"check data migration from other storage clouds."},{"line_number":83,"context_line":""},{"line_number":84,"context_line":"2. Due to code complexity, unified view ( described in blue"},{"line_number":85,"context_line":"print ) is not part of this commit. Will be committed separately."}],"source_content_type":"text/x-gerrit-commit-message","patch_set":16,"id":"1abeadc6_de757efa","line":82,"in_reply_to":"baada198_4c3c3193","updated":"2014-08-24 08:10:02.000000000","message":"Done","commit_id":"1e5ddde5fc3bc514d59e66a5c43592cbb502143c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"363fa98d6c1b66cf6a6cc4ba77c57e4db93c9b62","unresolved":false,"context_lines":[{"line_number":98,"context_line":"An example code how to use this feature to migrate data"},{"line_number":99,"context_line":"from file system:"},{"line_number":100,"context_line":"https://gist.github.com/gilv/64d43be25d0b4cb8ef29"},{"line_number":101,"context_line":""},{"line_number":102,"context_line":"Change-Id: I3c82f8c0e7eafa3fcfc4385c9a240b14bc766ead"},{"line_number":103,"context_line":"Implements: blueprint on-demand-data-migration-for-swift"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":16,"id":"baada198_0c279967","line":101,"updated":"2014-08-13 15:09:11.000000000","message":"Please insert line \u0027DocImpact\u0027","commit_id":"1e5ddde5fc3bc514d59e66a5c43592cbb502143c"},{"author":{"_account_id":9205,"name":"Gil Vernik","email":"gilv@il.ibm.com","username":"gilv"},"change_message_id":"8f1bbdcf5a4962e0cdf319f9941ca972103cf051","unresolved":false,"context_lines":[{"line_number":98,"context_line":"An example code how to use this feature to migrate data"},{"line_number":99,"context_line":"from file system:"},{"line_number":100,"context_line":"https://gist.github.com/gilv/64d43be25d0b4cb8ef29"},{"line_number":101,"context_line":""},{"line_number":102,"context_line":"Change-Id: I3c82f8c0e7eafa3fcfc4385c9a240b14bc766ead"},{"line_number":103,"context_line":"Implements: blueprint on-demand-data-migration-for-swift"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":16,"id":"1abeadc6_fe707a09","line":101,"in_reply_to":"baada198_0c279967","updated":"2014-08-24 08:10:02.000000000","message":"Done","commit_id":"1e5ddde5fc3bc514d59e66a5c43592cbb502143c"}],"etc/proxy-server.conf-sample":[{"author":{"_account_id":330,"name":"John Dickinson","email":"me@not.mn","username":"notmyname"},"change_message_id":"85a08e4e966678eb7d9f8e3a1fde04de1d3ca3a3","unresolved":false,"context_lines":[{"line_number":516,"context_line":"# List of all supported drivers. This is a dynamic list. "},{"line_number":517,"context_line":"supported_drivers \u003d fsystem,libcloud_s3,swift"},{"line_number":518,"context_line":"# For each driver XXX we define"},{"line_number":519,"context_line":"# driver_XXX_keys ( list of required input parameters )"},{"line_number":520,"context_line":"# driver_XXX_module ( the class that implements driver\u0027s logic )"},{"line_number":521,"context_line":""},{"line_number":522,"context_line":"# File System driver"}],"source_content_type":"application/octet-stream","patch_set":2,"id":"AAAAT3%2F%2F1DM%3D","line":519,"updated":"2013-12-30 15:40:19.000000000","message":"It wasn\u0027t clear to me at first that the \"keys\" are the required headers to be set on a container rather than the actual value of something here.\n\nIn general, please be explicit about what\u0027s the default config and what\u0027s the possible config. Based on your current version of this conf file, all 3 drivers are values that are required to be explicitly set, with no default values on them.","commit_id":"3d6b0eaf7d09738f13d44ba1384c2cdb31839027"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"07d88a7d47e7b9cf9097bfb0d8fca63c75e0bbd6","unresolved":false,"context_lines":[{"line_number":574,"context_line":"# A dynamic list of all supported drivers. "},{"line_number":575,"context_line":"supported_drivers \u003d fsystem,libcloud_s3,swift"},{"line_number":576,"context_line":""},{"line_number":577,"context_line":"# For each driver XXX should be define"},{"line_number":578,"context_line":"# driver_XXX_keys ( input parameters for the driver. This is the list of"},{"line_number":579,"context_line":"# required metadata headers to be set on container during data migration"},{"line_number":580,"context_line":"# setup process )"}],"source_content_type":"application/octet-stream","patch_set":8,"id":"AAAAUX%2F%2FP0k%3D","line":577,"updated":"2014-02-14 17:11:03.000000000","message":"s/define/defined","commit_id":"31b4aaeaebcc8bcec9eed67a0bb65352f55f9467"},{"author":{"_account_id":9205,"name":"Gil Vernik","email":"gilv@il.ibm.com","username":"gilv"},"change_message_id":"0fb520e153242eccdfa7133da32b7b0c542a5a7b","unresolved":false,"context_lines":[{"line_number":574,"context_line":"# A dynamic list of all supported drivers. "},{"line_number":575,"context_line":"supported_drivers \u003d fsystem,libcloud_s3,swift"},{"line_number":576,"context_line":""},{"line_number":577,"context_line":"# For each driver XXX should be define"},{"line_number":578,"context_line":"# driver_XXX_keys ( input parameters for the driver. This is the list of"},{"line_number":579,"context_line":"# required metadata headers to be set on container during data migration"},{"line_number":580,"context_line":"# setup process )"}],"source_content_type":"application/octet-stream","patch_set":8,"id":"AAAAUX%2F%2FJD0%3D","line":577,"in_reply_to":"AAAAUX%2F%2FP0k%3D","updated":"2014-02-16 08:59:15.000000000","message":"Done","commit_id":"31b4aaeaebcc8bcec9eed67a0bb65352f55f9467"},{"author":{"_account_id":9625,"name":"Thiago da Silva","email":"thiagodasilva@gmail.com","username":"thiago"},"change_message_id":"5c95f10bedc68a8b90ba312663cd3ce690d94891","unresolved":false,"context_lines":[{"line_number":618,"context_line":"use \u003d egg:swift#data_migration"},{"line_number":619,"context_line":""},{"line_number":620,"context_line":"# A dynamic list of all supported drivers."},{"line_number":621,"context_line":"supported_drivers \u003d fsystem,swift"},{"line_number":622,"context_line":""},{"line_number":623,"context_line":"# For each driver XXX should be defined"},{"line_number":624,"context_line":"# driver_XXX_keys ( input parameters for the driver. This is the list of"}],"source_content_type":"application/octet-stream","patch_set":17,"id":"1abeadc6_a4089992","line":621,"updated":"2014-08-26 17:16:47.000000000","message":"wondering if this is still necessary. The list of supported drivers could be made up of of the listed driver_XXX_modules.","commit_id":"610f4ddaace7ac8c85eb6ac427b10ae7730c9fa9"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"e387ba656abe913c3625351bde0b0b80d871365f","unresolved":false,"context_lines":[{"line_number":639,"context_line":"# This is mandatory parameter. Should contain root directory such that all source folders"},{"line_number":640,"context_line":"# will be below this root directory. Value should be set if file system driver "},{"line_number":641,"context_line":"# is activated for some container."},{"line_number":642,"context_line":"driver_fsystem_parent_path \u003d "},{"line_number":643,"context_line":""},{"line_number":644,"context_line":"# Swift driver based on python-swiftclient (optional)."},{"line_number":645,"context_line":"# driver_swift_module \u003d swift.common.data_migrator_drivers:SwiftAccessDriver"}],"source_content_type":"application/octet-stream","patch_set":23,"id":"da9df570_e48d127d","line":642,"updated":"2014-09-23 05:49:21.000000000","message":"Red boxes of doom (white spaces at the end of the lines which need to be removed)","commit_id":"4a4004287583df78997ec351b17896fc611d9fca"},{"author":{"_account_id":9205,"name":"Gil Vernik","email":"gilv@il.ibm.com","username":"gilv"},"change_message_id":"ac94e5e0f9df518bba58a726d3f3b737f3e68ee0","unresolved":false,"context_lines":[{"line_number":639,"context_line":"# This is mandatory parameter. Should contain root directory such that all source folders"},{"line_number":640,"context_line":"# will be below this root directory. Value should be set if file system driver "},{"line_number":641,"context_line":"# is activated for some container."},{"line_number":642,"context_line":"driver_fsystem_parent_path \u003d "},{"line_number":643,"context_line":""},{"line_number":644,"context_line":"# Swift driver based on python-swiftclient (optional)."},{"line_number":645,"context_line":"# driver_swift_module \u003d swift.common.data_migrator_drivers:SwiftAccessDriver"}],"source_content_type":"application/octet-stream","patch_set":23,"id":"da9df570_b020a4f8","line":642,"in_reply_to":"da9df570_e48d127d","updated":"2014-09-23 07:36:07.000000000","message":"Right. I wonder how it pass tox with this white spaces...","commit_id":"4a4004287583df78997ec351b17896fc611d9fca"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e28725b00b894bc701a270431eb72bd53f8b2342","unresolved":false,"context_lines":[{"line_number":628,"context_line":"use \u003d egg:swift#data_migration"},{"line_number":629,"context_line":""},{"line_number":630,"context_line":"# A dynamic list of all supported drivers."},{"line_number":631,"context_line":"supported_drivers \u003d fsystem,swift"},{"line_number":632,"context_line":""},{"line_number":633,"context_line":"# For each driver XXX should be defined"},{"line_number":634,"context_line":"# driver_XXX_module ( the class that implements driver\u0027s logic. This class"}],"source_content_type":"application/octet-stream","patch_set":24,"id":"da9df570_aaf9e8c9","line":631,"updated":"2014-09-24 08:45:32.000000000","message":"As it is, if this sample config is actually deployed then the proxy will fail to start because by default the required driver_fsystem_parent_path is not set. We do NOT want to set a default value for that, so we need to remove fsystem from the default sample config (but leave it here commented out).","commit_id":"f8af425febd8a059f0da2262e4446f3be0ece9b3"},{"author":{"_account_id":9625,"name":"Thiago da Silva","email":"thiagodasilva@gmail.com","username":"thiago"},"change_message_id":"22bbcb285c33aed1a40f6ecc8e4a141477619650","unresolved":false,"context_lines":[{"line_number":639,"context_line":"# This is mandatory parameter. Should contain root directory such that all source folders"},{"line_number":640,"context_line":"# will be below this root directory. Value should be set if file system driver"},{"line_number":641,"context_line":"# is activated for some container."},{"line_number":642,"context_line":"driver_fsystem_parent_path \u003d"},{"line_number":643,"context_line":""},{"line_number":644,"context_line":"# Swift driver based on python-swiftclient (optional)."},{"line_number":645,"context_line":"# driver_swift_module \u003d swift.common.data_migrator_drivers:SwiftAccessDriver"}],"source_content_type":"application/octet-stream","patch_set":24,"id":"da9df570_2359be5a","line":642,"updated":"2014-09-23 18:24:05.000000000","message":"would it make sense to change this line to something like this?\n  driver_fsystem_parent_path \u003d \u003croot directory of source\u003e","commit_id":"f8af425febd8a059f0da2262e4446f3be0ece9b3"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e28725b00b894bc701a270431eb72bd53f8b2342","unresolved":false,"context_lines":[{"line_number":639,"context_line":"# This is mandatory parameter. Should contain root directory such that all source folders"},{"line_number":640,"context_line":"# will be below this root directory. Value should be set if file system driver"},{"line_number":641,"context_line":"# is activated for some container."},{"line_number":642,"context_line":"driver_fsystem_parent_path \u003d"},{"line_number":643,"context_line":""},{"line_number":644,"context_line":"# Swift driver based on python-swiftclient (optional)."},{"line_number":645,"context_line":"# driver_swift_module \u003d swift.common.data_migrator_drivers:SwiftAccessDriver"}],"source_content_type":"application/octet-stream","patch_set":24,"id":"da9df570_6a59c0fd","line":642,"in_reply_to":"da9df570_2359be5a","updated":"2014-09-24 08:45:32.000000000","message":"Agree. And comment it (see my comment above)","commit_id":"f8af425febd8a059f0da2262e4446f3be0ece9b3"},{"author":{"_account_id":9625,"name":"Thiago da Silva","email":"thiagodasilva@gmail.com","username":"thiago"},"change_message_id":"8933e72b69e150a6b9947f849e4d3fad56eca123","unresolved":false,"context_lines":[{"line_number":636,"context_line":""},{"line_number":637,"context_line":"# File System driver."},{"line_number":638,"context_line":"# This driver may be activated by adding \u0027fsystem\u0027 to the list of"},{"line_number":639,"context_line":"# supported_drivers, e.g. supported_drivers \u003d swift,fsystem "},{"line_number":640,"context_line":"driver_fsystem_module \u003d swift.common.data_migrator_drivers:FileSystemAccessDriver"},{"line_number":641,"context_line":""},{"line_number":642,"context_line":"# This is mandatory parameter for the File System driver. Should contain a root"}],"source_content_type":"application/octet-stream","patch_set":26,"id":"da9df570_142bf682","line":639,"updated":"2014-09-24 13:06:14.000000000","message":"oops, whitespace :)","commit_id":"111223d0ac9553bccf1be6529870cb622894f545"},{"author":{"_account_id":9625,"name":"Thiago da Silva","email":"thiagodasilva@gmail.com","username":"thiago"},"change_message_id":"5f98e767b3306e79041acfe6b9bc8c822833309e","unresolved":false,"context_lines":[{"line_number":644,"context_line":"# driver_fsystem_parent_path \u003d \u003croot directory of source\u003e"},{"line_number":645,"context_line":""},{"line_number":646,"context_line":"# Swift driver based on python-swiftclient (optional)."},{"line_number":647,"context_line":"driver_swift_module \u003d swift.common.data_migrator_drivers:SwiftAccessDriver"}],"source_content_type":"application/octet-stream","patch_set":26,"id":"da9df570_148db631","line":647,"updated":"2014-09-24 13:13:14.000000000","message":"should there be a comment here saying that to use this driver the swiftclient must be installed on the proxy nodes?","commit_id":"111223d0ac9553bccf1be6529870cb622894f545"}],"swift/common/data_migration_common.py":[{"author":{"_account_id":9625,"name":"Thiago da Silva","email":"thiagodasilva@gmail.com","username":"thiago"},"change_message_id":"22bbcb285c33aed1a40f6ecc8e4a141477619650","unresolved":false,"context_lines":[{"line_number":113,"context_line":"        :param timestamp: timestamp of an object in the old storage"},{"line_number":114,"context_line":"        :returns HTTP code result of an upload operation"},{"line_number":115,"context_line":"        \"\"\""},{"line_number":116,"context_line":"        new_env \u003d dict(env)"},{"line_number":117,"context_line":"        new_env[\u0027REQUEST_METHOD\u0027] \u003d \u0027PUT\u0027"},{"line_number":118,"context_line":"        new_env[\u0027wsgi.input\u0027] \u003d data"},{"line_number":119,"context_line":"        new_env[\u0027CONTENT_LENGTH\u0027] \u003d length"}],"source_content_type":"text/x-python","patch_set":24,"id":"da9df570_d2027d6e","line":116,"updated":"2014-09-23 18:24:05.000000000","message":"i\u0027m still a bit concerned about the many copies of env that are being made. In data_migration.py there\u0027s a shallow copy and a deep copy and here another deep copy. It seems we would need only one deep copy. thoughts?","commit_id":"f8af425febd8a059f0da2262e4446f3be0ece9b3"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"479b462c83b3a6a2870763509bd9c6b5c861fc41","unresolved":false,"context_lines":[{"line_number":113,"context_line":"        :param timestamp: timestamp of an object in the old storage"},{"line_number":114,"context_line":"        :returns HTTP code result of an upload operation"},{"line_number":115,"context_line":"        \"\"\""},{"line_number":116,"context_line":"        new_env \u003d dict(env)"},{"line_number":117,"context_line":"        new_env[\u0027REQUEST_METHOD\u0027] \u003d \u0027PUT\u0027"},{"line_number":118,"context_line":"        new_env[\u0027wsgi.input\u0027] \u003d data"},{"line_number":119,"context_line":"        new_env[\u0027CONTENT_LENGTH\u0027] \u003d length"}],"source_content_type":"text/x-python","patch_set":24,"id":"da9df570_782a957f","line":116,"updated":"2014-09-23 17:40:18.000000000","message":"nit: I\u0027m not sure this new dict() is needed. env is passed from migrate_object original_env which is a copy of the req.environ made at line 199 in data_migration.py. That original_env dict does not appear to be used elsewhere so should be safe to use here without copying? But if there is doubt leave it there I guess.","commit_id":"f8af425febd8a059f0da2262e4446f3be0ece9b3"}],"swift/common/data_migrator_drivers.py":[{"author":{"_account_id":6889,"name":"Fabien Boucher","email":"fboucher@redhat.com","username":"fabien-boucher"},"change_message_id":"6989cb8bfbcafb02f9c9a44fa05b2f1bed71f27d","unresolved":false,"context_lines":[{"line_number":169,"context_line":""},{"line_number":170,"context_line":"    def get_object(self, object_name):"},{"line_number":171,"context_line":"        \"\"\""},{"line_number":172,"context_line":"        Read an object from filesystem"},{"line_number":173,"context_line":""},{"line_number":174,"context_line":"        :params object_name: name of the object"},{"line_number":175,"context_line":"        :returns (metadata, object size, data stream, content type, timestamp)"}],"source_content_type":"text/x-python","patch_set":5,"id":"AAAAUH%2F%2F8HA%3D","line":172,"updated":"2014-01-06 21:57:30.000000000","message":"Not from the FS","commit_id":"94be0c854c189f22f9f1debf0f4a022c09fc3f40"},{"author":{"_account_id":9205,"name":"Gil Vernik","email":"gilv@il.ibm.com","username":"gilv"},"change_message_id":"5c8750ba8b7931c1aa887dbab94bb58ebf67c7d3","unresolved":false,"context_lines":[{"line_number":169,"context_line":""},{"line_number":170,"context_line":"    def get_object(self, object_name):"},{"line_number":171,"context_line":"        \"\"\""},{"line_number":172,"context_line":"        Read an object from filesystem"},{"line_number":173,"context_line":""},{"line_number":174,"context_line":"        :params object_name: name of the object"},{"line_number":175,"context_line":"        :returns (metadata, object size, data stream, content type, timestamp)"}],"source_content_type":"text/x-python","patch_set":5,"id":"AAAAUH%2F%2F4%2Fw%3D","line":172,"in_reply_to":"AAAAUH%2F%2F8HA%3D","updated":"2014-01-07 07:34:33.000000000","message":"Done","commit_id":"94be0c854c189f22f9f1debf0f4a022c09fc3f40"},{"author":{"_account_id":6889,"name":"Fabien Boucher","email":"fboucher@redhat.com","username":"fabien-boucher"},"change_message_id":"6989cb8bfbcafb02f9c9a44fa05b2f1bed71f27d","unresolved":false,"context_lines":[{"line_number":227,"context_line":""},{"line_number":228,"context_line":"    def get_object(self, object_name):"},{"line_number":229,"context_line":"        \"\"\""},{"line_number":230,"context_line":"        Read an object from filesystem"},{"line_number":231,"context_line":""},{"line_number":232,"context_line":"        :params object_name: name of the object"},{"line_number":233,"context_line":"        :returns (metadata, object size, data stream, content type, timestamp)"}],"source_content_type":"text/x-python","patch_set":5,"id":"AAAAUH%2F%2F8F8%3D","line":230,"updated":"2014-01-06 21:57:30.000000000","message":"Ditto","commit_id":"94be0c854c189f22f9f1debf0f4a022c09fc3f40"},{"author":{"_account_id":9205,"name":"Gil Vernik","email":"gilv@il.ibm.com","username":"gilv"},"change_message_id":"5c8750ba8b7931c1aa887dbab94bb58ebf67c7d3","unresolved":false,"context_lines":[{"line_number":227,"context_line":""},{"line_number":228,"context_line":"    def get_object(self, object_name):"},{"line_number":229,"context_line":"        \"\"\""},{"line_number":230,"context_line":"        Read an object from filesystem"},{"line_number":231,"context_line":""},{"line_number":232,"context_line":"        :params object_name: name of the object"},{"line_number":233,"context_line":"        :returns (metadata, object size, data stream, content type, timestamp)"}],"source_content_type":"text/x-python","patch_set":5,"id":"AAAAUH%2F%2F4%2Fg%3D","line":230,"in_reply_to":"AAAAUH%2F%2F8F8%3D","updated":"2014-01-07 07:34:33.000000000","message":"Done","commit_id":"94be0c854c189f22f9f1debf0f4a022c09fc3f40"},{"author":{"_account_id":6889,"name":"Fabien Boucher","email":"fboucher@redhat.com","username":"fabien-boucher"},"change_message_id":"6989cb8bfbcafb02f9c9a44fa05b2f1bed71f27d","unresolved":false,"context_lines":[{"line_number":233,"context_line":"        :returns (metadata, object size, data stream, content type, timestamp)"},{"line_number":234,"context_line":"        \"\"\""},{"line_number":235,"context_line":"        if SWIFTCLIENT_IMPORTED:"},{"line_number":236,"context_line":"            storage_url, token \u003d client.get_auth(self.token_url,"},{"line_number":237,"context_line":"                                                 self.user, self.key)"},{"line_number":238,"context_line":"            resp_headers, data \u003d client.get_object(storage_url,"},{"line_number":239,"context_line":"                                                   token,"}],"source_content_type":"text/x-python","patch_set":5,"id":"AAAAUH%2F%2F8EU%3D","line":236,"updated":"2014-01-06 21:57:30.000000000","message":"What about having storage_url and token cached in memcache during a small amount of time ? If a bunch of object is retrieved from a the remote storage it can be long to authenticate each time as token will not change so quickly. Authenticate only if the use of the token form the cache fails.","commit_id":"94be0c854c189f22f9f1debf0f4a022c09fc3f40"},{"author":{"_account_id":9205,"name":"Gil Vernik","email":"gilv@il.ibm.com","username":"gilv"},"change_message_id":"5c8750ba8b7931c1aa887dbab94bb58ebf67c7d3","unresolved":false,"context_lines":[{"line_number":233,"context_line":"        :returns (metadata, object size, data stream, content type, timestamp)"},{"line_number":234,"context_line":"        \"\"\""},{"line_number":235,"context_line":"        if SWIFTCLIENT_IMPORTED:"},{"line_number":236,"context_line":"            storage_url, token \u003d client.get_auth(self.token_url,"},{"line_number":237,"context_line":"                                                 self.user, self.key)"},{"line_number":238,"context_line":"            resp_headers, data \u003d client.get_object(storage_url,"},{"line_number":239,"context_line":"                                                   token,"}],"source_content_type":"text/x-python","patch_set":5,"id":"AAAAUH%2F%2F45o%3D","line":236,"in_reply_to":"AAAAUH%2F%2F8EU%3D","updated":"2014-01-07 07:34:33.000000000","message":"Migration is per object and a single GET /HEAD trigger migration. To store this in memcache will means that it should be carefully protected between processes of different users, otherwise one user may get something from cache that is related to other user. And we deal here with credentials. I agree with you that it should be somehow cached here ( i thought Swift already make cache for the token for the same credentials, so subsequent calls are minor ). I think it\u0027s better to leave caching improvements to bit later stage in the context of data migration, since it\u0027s a door to dozens of issues. Please let me know if you think differently on this point.","commit_id":"94be0c854c189f22f9f1debf0f4a022c09fc3f40"},{"author":{"_account_id":6889,"name":"Fabien Boucher","email":"fboucher@redhat.com","username":"fabien-boucher"},"change_message_id":"6989cb8bfbcafb02f9c9a44fa05b2f1bed71f27d","unresolved":false,"context_lines":[{"line_number":240,"context_line":"                                                   self.data_source,"},{"line_number":241,"context_line":"                                                   object_name)"},{"line_number":242,"context_line":"            metadata \u003d dict()"},{"line_number":243,"context_line":"            for key, val in resp_headers.items():"},{"line_number":244,"context_line":"                if key.startswith(\u0027x-object-meta\u0027):"},{"line_number":245,"context_line":"                    metadata[key] \u003d val"},{"line_number":246,"context_line":"            return metadata, resp_headers.get(\u0027content-length\u0027),\\"}],"source_content_type":"text/x-python","patch_set":5,"id":"AAAAUH%2F%2F8C8%3D","line":243,"updated":"2014-01-06 21:57:30.000000000","message":"iteritems better ?","commit_id":"94be0c854c189f22f9f1debf0f4a022c09fc3f40"},{"author":{"_account_id":9205,"name":"Gil Vernik","email":"gilv@il.ibm.com","username":"gilv"},"change_message_id":"5c8750ba8b7931c1aa887dbab94bb58ebf67c7d3","unresolved":false,"context_lines":[{"line_number":240,"context_line":"                                                   self.data_source,"},{"line_number":241,"context_line":"                                                   object_name)"},{"line_number":242,"context_line":"            metadata \u003d dict()"},{"line_number":243,"context_line":"            for key, val in resp_headers.items():"},{"line_number":244,"context_line":"                if key.startswith(\u0027x-object-meta\u0027):"},{"line_number":245,"context_line":"                    metadata[key] \u003d val"},{"line_number":246,"context_line":"            return metadata, resp_headers.get(\u0027content-length\u0027),\\"}],"source_content_type":"text/x-python","patch_set":5,"id":"AAAAUH%2F%2F44U%3D","line":243,"in_reply_to":"AAAAUH%2F%2F8C8%3D","updated":"2014-01-07 07:34:33.000000000","message":"Done","commit_id":"94be0c854c189f22f9f1debf0f4a022c09fc3f40"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"ebc43cc9e5b5c1aa102c8e70c43858a9871b0930","unresolved":false,"context_lines":[{"line_number":234,"context_line":"                h, self.data \u003d client.get_object(self.storage_url, self.token,"},{"line_number":235,"context_line":"                                                 self.data_source,"},{"line_number":236,"context_line":"                                                 self.object_name, None, size)"},{"line_number":237,"context_line":"            return self.data.next()"},{"line_number":238,"context_line":""},{"line_number":239,"context_line":"    def __init__(self, migration_provider, data_source, params):"},{"line_number":240,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":12,"id":"AAAAVn%2F%2FMWE%3D","line":237,"updated":"2014-03-07 19:09:45.000000000","message":"This is a bit odd, but I saw 499 client disconnect in my logs for the upload PUT. They went away when I did this:\n\ntry:\n    chunk \u003d self.data.next()\nexcept StopIteration:\n    chunk \u003d \u0027\u0027\nreturn chunk\n\nI\u0027m not sure I can explain why but it may be to do with a piece of code in proxy_logging middleware that constructs a response iterator. \n\nI\u0027m still seeing a 499 for a GET logged right after the upload PUT but I haven\u0027t tracked the cause of that down.","commit_id":"219e29e5c9dac4e12faf4c15d204b5da4a5d1b09"},{"author":{"_account_id":9205,"name":"Gil Vernik","email":"gilv@il.ibm.com","username":"gilv"},"change_message_id":"14e3bdc725365f47838c070e48a1aa94f7d73270","unresolved":false,"context_lines":[{"line_number":234,"context_line":"                h, self.data \u003d client.get_object(self.storage_url, self.token,"},{"line_number":235,"context_line":"                                                 self.data_source,"},{"line_number":236,"context_line":"                                                 self.object_name, None, size)"},{"line_number":237,"context_line":"            return self.data.next()"},{"line_number":238,"context_line":""},{"line_number":239,"context_line":"    def __init__(self, migration_provider, data_source, params):"},{"line_number":240,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":12,"id":"AAAAWH%2F%2BXVs%3D","line":237,"in_reply_to":"AAAAVn%2F%2FMWE%3D","updated":"2014-03-27 11:24:40.000000000","message":"So this 499 is printed in logs only if proxy pipe contains proxy-log after the migration middleware. If proxy-log before migration mw - than there are no errors. I need to further  understand why it happens. Any ideas?","commit_id":"219e29e5c9dac4e12faf4c15d204b5da4a5d1b09"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"ebc43cc9e5b5c1aa102c8e70c43858a9871b0930","unresolved":false,"context_lines":[{"line_number":260,"context_line":"                                                 self.user, self.key)"},{"line_number":261,"context_line":"            resp_headers \u003d client.head_object(storage_url, token,"},{"line_number":262,"context_line":"                                              self.data_source, object_name)"},{"line_number":263,"context_line":"            data \u003d self.ReadWrapper(storage_url, token, self.data_source,"},{"line_number":264,"context_line":"                                    object_name)"},{"line_number":265,"context_line":"            metadata \u003d dict()"},{"line_number":266,"context_line":"            for key, val in resp_headers.iteritems():"}],"source_content_type":"text/x-python","patch_set":12,"id":"AAAAVn%2F%2FMaA%3D","line":263,"updated":"2014-03-07 19:09:45.000000000","message":"I still think there is an issue here: you are making two separate requests (HEAD then GET) which isn\u0027t atomic so there is a risk that the source object or its metadata changes between the HEAD and the GET. e.g. the content-length returned by the HEAD may not be the content length of the subsequent GET.\n\nI think the only solution is to implement some kind of buffer that matches the consuming read size to the pre-chosen get_object chunk size.","commit_id":"219e29e5c9dac4e12faf4c15d204b5da4a5d1b09"},{"author":{"_account_id":9205,"name":"Gil Vernik","email":"gilv@il.ibm.com","username":"gilv"},"change_message_id":"14e3bdc725365f47838c070e48a1aa94f7d73270","unresolved":false,"context_lines":[{"line_number":260,"context_line":"                                                 self.user, self.key)"},{"line_number":261,"context_line":"            resp_headers \u003d client.head_object(storage_url, token,"},{"line_number":262,"context_line":"                                              self.data_source, object_name)"},{"line_number":263,"context_line":"            data \u003d self.ReadWrapper(storage_url, token, self.data_source,"},{"line_number":264,"context_line":"                                    object_name)"},{"line_number":265,"context_line":"            metadata \u003d dict()"},{"line_number":266,"context_line":"            for key, val in resp_headers.iteritems():"}],"source_content_type":"text/x-python","patch_set":12,"id":"AAAAWH%2F%2BXWk%3D","line":263,"in_reply_to":"AAAAVn%2F%2FMaA%3D","updated":"2014-03-27 11:24:40.000000000","message":"It\u0027s better to resolve this in Swiftlient itself. All what i need is an file like reader from Swift client and not the iterator. This will resolve the need to make two requests.  For know it can be known limitation, i prefer this limitation than previous code where i had to kept the whole object in memory.","commit_id":"219e29e5c9dac4e12faf4c15d204b5da4a5d1b09"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"322cdf9d121df319ad381a11fe3244dc20c14a52","unresolved":false,"context_lines":[{"line_number":231,"context_line":""},{"line_number":232,"context_line":"        def read(self, size):"},{"line_number":233,"context_line":"            if (self.data is None):"},{"line_number":234,"context_line":"                h, self.data \u003d client.get_object(self.storage_url, self.token,"},{"line_number":235,"context_line":"                                                 self.data_source,"},{"line_number":236,"context_line":"                                                 self.object_name, None, size)"},{"line_number":237,"context_line":"            return self.data.next()"}],"source_content_type":"text/x-python","patch_set":13,"id":"baada198_51fe36b0","line":234,"updated":"2014-07-25 15:16:20.000000000","message":"It doesn\u0027t seem \u0027safe\u0027 to make a second request - the object may have changed on remote server since the head request was made so the headers  returned by the earlier HEAD may not match the data content from the GET.\n\nI think a single GET should be used with some kind of buffer to match the read size to the chunk size. I think that buffer does not need to be the size of the object, just the size of max(read size, get chunk size) I think.","commit_id":"38bf06b8de46421318c3d119faa56b03689d0d70"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"363fa98d6c1b66cf6a6cc4ba77c57e4db93c9b62","unresolved":false,"context_lines":[{"line_number":44,"context_line":"    object_to_migrate \u003d None"},{"line_number":45,"context_line":"    driver_loaded \u003d True"},{"line_number":46,"context_line":""},{"line_number":47,"context_line":"    def __init__(self, migration_provider, data_source, container_md):"},{"line_number":48,"context_line":"        \"\"\""},{"line_number":49,"context_line":"        :params migration_provider: name of the old provider."},{"line_number":50,"context_line":"        :params data_source:  a folder that contains objects."}],"source_content_type":"text/x-python","patch_set":16,"id":"baada198_de7d4ee8","line":47,"updated":"2014-08-13 15:09:11.000000000","message":"migration_provider is unused","commit_id":"1e5ddde5fc3bc514d59e66a5c43592cbb502143c"},{"author":{"_account_id":9205,"name":"Gil Vernik","email":"gilv@il.ibm.com","username":"gilv"},"change_message_id":"8f1bbdcf5a4962e0cdf319f9941ca972103cf051","unresolved":false,"context_lines":[{"line_number":44,"context_line":"    object_to_migrate \u003d None"},{"line_number":45,"context_line":"    driver_loaded \u003d True"},{"line_number":46,"context_line":""},{"line_number":47,"context_line":"    def __init__(self, migration_provider, data_source, container_md):"},{"line_number":48,"context_line":"        \"\"\""},{"line_number":49,"context_line":"        :params migration_provider: name of the old provider."},{"line_number":50,"context_line":"        :params data_source:  a folder that contains objects."}],"source_content_type":"text/x-python","patch_set":16,"id":"7ac7a964_22c5635b","line":47,"in_reply_to":"baada198_de7d4ee8","updated":"2014-08-24 08:10:02.000000000","message":"Done","commit_id":"1e5ddde5fc3bc514d59e66a5c43592cbb502143c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"363fa98d6c1b66cf6a6cc4ba77c57e4db93c9b62","unresolved":false,"context_lines":[{"line_number":103,"context_line":"    X-Container-Migration-Provider: swift"},{"line_number":104,"context_line":"    X-Container-Migration-Token-Url: token URL,"},{"line_number":105,"context_line":"    e.g. http://127.0.0.1:8080/auth/v1.0"},{"line_number":106,"context_line":"    X-Container-Migration-Source: container that contain\u0027s objects"},{"line_number":107,"context_line":"    X-Container-Migration-User: credentials to access"},{"line_number":108,"context_line":"    source container, e.g. test:tester"},{"line_number":109,"context_line":"    X-Container-Migration-Key: key, e.g testing"}],"source_content_type":"text/x-python","patch_set":16,"id":"baada198_18a2af96","line":106,"updated":"2014-08-13 15:09:11.000000000","message":"nit s/contain\u0027s/contains/","commit_id":"1e5ddde5fc3bc514d59e66a5c43592cbb502143c"},{"author":{"_account_id":9205,"name":"Gil Vernik","email":"gilv@il.ibm.com","username":"gilv"},"change_message_id":"8f1bbdcf5a4962e0cdf319f9941ca972103cf051","unresolved":false,"context_lines":[{"line_number":103,"context_line":"    X-Container-Migration-Provider: swift"},{"line_number":104,"context_line":"    X-Container-Migration-Token-Url: token URL,"},{"line_number":105,"context_line":"    e.g. http://127.0.0.1:8080/auth/v1.0"},{"line_number":106,"context_line":"    X-Container-Migration-Source: container that contain\u0027s objects"},{"line_number":107,"context_line":"    X-Container-Migration-User: credentials to access"},{"line_number":108,"context_line":"    source container, e.g. test:tester"},{"line_number":109,"context_line":"    X-Container-Migration-Key: key, e.g testing"}],"source_content_type":"text/x-python","patch_set":16,"id":"7ac7a964_82263767","line":106,"in_reply_to":"baada198_18a2af96","updated":"2014-08-24 08:10:02.000000000","message":"Done","commit_id":"1e5ddde5fc3bc514d59e66a5c43592cbb502143c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"363fa98d6c1b66cf6a6cc4ba77c57e4db93c9b62","unresolved":false,"context_lines":[{"line_number":108,"context_line":"    source container, e.g. test:tester"},{"line_number":109,"context_line":"    X-Container-Migration-Key: key, e.g testing"},{"line_number":110,"context_line":""},{"line_number":111,"context_line":"    X-Container-Migration-Auth-Type currently supports auth1 only."},{"line_number":112,"context_line":"    \"\"\""},{"line_number":113,"context_line":"    driver_loaded \u003d SWIFTCLIENT_IMPORTED"},{"line_number":114,"context_line":""}],"source_content_type":"text/x-python","patch_set":16,"id":"baada198_73c4ac93","line":111,"updated":"2014-08-13 15:09:11.000000000","message":"can this be made an optional rather required header since its not being used, and default to auth1.","commit_id":"1e5ddde5fc3bc514d59e66a5c43592cbb502143c"},{"author":{"_account_id":9205,"name":"Gil Vernik","email":"gilv@il.ibm.com","username":"gilv"},"change_message_id":"8f1bbdcf5a4962e0cdf319f9941ca972103cf051","unresolved":false,"context_lines":[{"line_number":108,"context_line":"    source container, e.g. test:tester"},{"line_number":109,"context_line":"    X-Container-Migration-Key: key, e.g testing"},{"line_number":110,"context_line":""},{"line_number":111,"context_line":"    X-Container-Migration-Auth-Type currently supports auth1 only."},{"line_number":112,"context_line":"    \"\"\""},{"line_number":113,"context_line":"    driver_loaded \u003d SWIFTCLIENT_IMPORTED"},{"line_number":114,"context_line":""}],"source_content_type":"text/x-python","patch_set":16,"id":"7ac7a964_b349c48d","line":111,"in_reply_to":"baada198_73c4ac93","updated":"2014-08-24 08:10:02.000000000","message":"Done","commit_id":"1e5ddde5fc3bc514d59e66a5c43592cbb502143c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"363fa98d6c1b66cf6a6cc4ba77c57e4db93c9b62","unresolved":false,"context_lines":[{"line_number":112,"context_line":"    \"\"\""},{"line_number":113,"context_line":"    driver_loaded \u003d SWIFTCLIENT_IMPORTED"},{"line_number":114,"context_line":""},{"line_number":115,"context_line":"    class ReadWrapper():"},{"line_number":116,"context_line":"        \u0027\u0027\u0027"},{"line_number":117,"context_line":"        This will allow proxy server object implementation"},{"line_number":118,"context_line":"        to read object\u0027s data in chunks."}],"source_content_type":"text/x-python","patch_set":16,"id":"baada198_ba126d62","line":115,"updated":"2014-08-13 15:09:11.000000000","message":"this class is no longer used","commit_id":"1e5ddde5fc3bc514d59e66a5c43592cbb502143c"},{"author":{"_account_id":9205,"name":"Gil Vernik","email":"gilv@il.ibm.com","username":"gilv"},"change_message_id":"8f1bbdcf5a4962e0cdf319f9941ca972103cf051","unresolved":false,"context_lines":[{"line_number":112,"context_line":"    \"\"\""},{"line_number":113,"context_line":"    driver_loaded \u003d SWIFTCLIENT_IMPORTED"},{"line_number":114,"context_line":""},{"line_number":115,"context_line":"    class ReadWrapper():"},{"line_number":116,"context_line":"        \u0027\u0027\u0027"},{"line_number":117,"context_line":"        This will allow proxy server object implementation"},{"line_number":118,"context_line":"        to read object\u0027s data in chunks."}],"source_content_type":"text/x-python","patch_set":16,"id":"7ac7a964_020307b9","line":115,"in_reply_to":"baada198_ba126d62","updated":"2014-08-24 08:10:02.000000000","message":"Done","commit_id":"1e5ddde5fc3bc514d59e66a5c43592cbb502143c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"363fa98d6c1b66cf6a6cc4ba77c57e4db93c9b62","unresolved":false,"context_lines":[{"line_number":131,"context_line":"                                                 self.object_name, None, size)"},{"line_number":132,"context_line":"            return self.data.next()"},{"line_number":133,"context_line":""},{"line_number":134,"context_line":"    def __init__(self, migration_provider, data_source, params):"},{"line_number":135,"context_line":"        \"\"\""},{"line_number":136,"context_line":"        :params migration_provider: name of the old provider."},{"line_number":137,"context_line":"        :params data_source:  a folder that contains objects."}],"source_content_type":"text/x-python","patch_set":16,"id":"baada198_be5fba73","line":134,"updated":"2014-08-13 15:09:11.000000000","message":"migration_provider is unused","commit_id":"1e5ddde5fc3bc514d59e66a5c43592cbb502143c"},{"author":{"_account_id":9205,"name":"Gil Vernik","email":"gilv@il.ibm.com","username":"gilv"},"change_message_id":"8f1bbdcf5a4962e0cdf319f9941ca972103cf051","unresolved":false,"context_lines":[{"line_number":131,"context_line":"                                                 self.object_name, None, size)"},{"line_number":132,"context_line":"            return self.data.next()"},{"line_number":133,"context_line":""},{"line_number":134,"context_line":"    def __init__(self, migration_provider, data_source, params):"},{"line_number":135,"context_line":"        \"\"\""},{"line_number":136,"context_line":"        :params migration_provider: name of the old provider."},{"line_number":137,"context_line":"        :params data_source:  a folder that contains objects."}],"source_content_type":"text/x-python","patch_set":16,"id":"7ac7a964_82ab9788","line":134,"in_reply_to":"baada198_be5fba73","updated":"2014-08-24 08:10:02.000000000","message":"Done","commit_id":"1e5ddde5fc3bc514d59e66a5c43592cbb502143c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"363fa98d6c1b66cf6a6cc4ba77c57e4db93c9b62","unresolved":false,"context_lines":[{"line_number":135,"context_line":"        \"\"\""},{"line_number":136,"context_line":"        :params migration_provider: name of the old provider."},{"line_number":137,"context_line":"        :params data_source:  a folder that contains objects."},{"line_number":138,"context_line":"        :params container_md: metadata of the Swift container"},{"line_number":139,"context_line":"        \"\"\""},{"line_number":140,"context_line":"        self.data_source \u003d data_source"},{"line_number":141,"context_line":"        self.token_url \u003d params.get(\u0027token-url\u0027, \u0027\u0027)"}],"source_content_type":"text/x-python","patch_set":16,"id":"baada198_6373d04f","line":138,"updated":"2014-08-13 15:09:11.000000000","message":"container_md is not a param","commit_id":"1e5ddde5fc3bc514d59e66a5c43592cbb502143c"},{"author":{"_account_id":9205,"name":"Gil Vernik","email":"gilv@il.ibm.com","username":"gilv"},"change_message_id":"8f1bbdcf5a4962e0cdf319f9941ca972103cf051","unresolved":false,"context_lines":[{"line_number":135,"context_line":"        \"\"\""},{"line_number":136,"context_line":"        :params migration_provider: name of the old provider."},{"line_number":137,"context_line":"        :params data_source:  a folder that contains objects."},{"line_number":138,"context_line":"        :params container_md: metadata of the Swift container"},{"line_number":139,"context_line":"        \"\"\""},{"line_number":140,"context_line":"        self.data_source \u003d data_source"},{"line_number":141,"context_line":"        self.token_url \u003d params.get(\u0027token-url\u0027, \u0027\u0027)"}],"source_content_type":"text/x-python","patch_set":16,"id":"7ac7a964_82dd5745","line":138,"in_reply_to":"baada198_6373d04f","updated":"2014-08-24 08:10:02.000000000","message":"Done","commit_id":"1e5ddde5fc3bc514d59e66a5c43592cbb502143c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"363fa98d6c1b66cf6a6cc4ba77c57e4db93c9b62","unresolved":false,"context_lines":[{"line_number":157,"context_line":"            path \u003d \u0027%s/%s/%s\u0027 % (parsed.path, quote(self.data_source),"},{"line_number":158,"context_line":"                                 quote(object_name))"},{"line_number":159,"context_line":"            conn.request(\u0027GET\u0027, path, \u0027\u0027, {\u0027X-Auth-Token\u0027: token})"},{"line_number":160,"context_line":"            resp \u003d conn.getresponse()"},{"line_number":161,"context_line":"            if resp.status \u003c 200 or resp.status \u003e\u003d 300:"},{"line_number":162,"context_line":"                body \u003d resp.read()"},{"line_number":163,"context_line":"                raise Exception(\u0027Object GET failed: %s\u0027 % body)"}],"source_content_type":"text/x-python","patch_set":16,"id":"baada198_28ca97a0","line":160,"updated":"2014-08-13 15:09:11.000000000","message":"If I setup a bad storage_url then I see response header \u0027x-migration-status\u0027, \"HTTPConnectionPool(host\u003d\u0027BROKEN.localdomain\u0027 ......\n\nSee http://paste.openstack.org/show/94291/\n\nI think wrapping line 159-160 in try/except to catch requests library connection errors would be good. Then raise an Exception with a more friendly/simpler message like \"Connection failed to %s\" % storage_url","commit_id":"1e5ddde5fc3bc514d59e66a5c43592cbb502143c"},{"author":{"_account_id":9205,"name":"Gil Vernik","email":"gilv@il.ibm.com","username":"gilv"},"change_message_id":"8f1bbdcf5a4962e0cdf319f9941ca972103cf051","unresolved":false,"context_lines":[{"line_number":157,"context_line":"            path \u003d \u0027%s/%s/%s\u0027 % (parsed.path, quote(self.data_source),"},{"line_number":158,"context_line":"                                 quote(object_name))"},{"line_number":159,"context_line":"            conn.request(\u0027GET\u0027, path, \u0027\u0027, {\u0027X-Auth-Token\u0027: token})"},{"line_number":160,"context_line":"            resp \u003d conn.getresponse()"},{"line_number":161,"context_line":"            if resp.status \u003c 200 or resp.status \u003e\u003d 300:"},{"line_number":162,"context_line":"                body \u003d resp.read()"},{"line_number":163,"context_line":"                raise Exception(\u0027Object GET failed: %s\u0027 % body)"}],"source_content_type":"text/x-python","patch_set":16,"id":"7ac7a964_53c79012","line":160,"in_reply_to":"baada198_28ca97a0","updated":"2014-08-24 08:10:02.000000000","message":"Done","commit_id":"1e5ddde5fc3bc514d59e66a5c43592cbb502143c"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"b7b25fb0c336b17f7b5f051c9a7fe5d976ac63f4","unresolved":false,"context_lines":[{"line_number":151,"context_line":"        return {}, None, None, None, None"},{"line_number":152,"context_line":""},{"line_number":153,"context_line":"    def finalize(self):"},{"line_number":154,"context_line":"        pass"}],"source_content_type":"text/x-python","patch_set":17,"id":"1abeadc6_283b339d","line":154,"updated":"2014-08-27 04:39:24.000000000","message":"To aid in other devs to extend and add more drivers, I wonder if we should make a base class that all migration drivers can inherit from rather then just object. As I can see there needs to be get_object and finalize methods and a driver_loaded property that they all share. \n\n  class DataMigrationDriver(object):\n      driver_loaded \u003d False\n\n      @property\n      def driver_loaded(self):\n          return self.driver_loaded\n\n      def get_object(self, object_name):\n          # You must override this method in children classes\n          pass\n\n      def finalize(self):\n          pass\n\nThen the SwiftAccessDriver wont need to specify the finalize method anymore.\n\nThe other benefit of this is if we ever need more features we could potentially add them to them all at once.","commit_id":"610f4ddaace7ac8c85eb6ac427b10ae7730c9fa9"},{"author":{"_account_id":9625,"name":"Thiago da Silva","email":"thiagodasilva@gmail.com","username":"thiago"},"change_message_id":"d852dd936c6479a740e8a9312088b4b2696f90e4","unresolved":false,"context_lines":[{"line_number":50,"context_line":""},{"line_number":51,"context_line":"    X-Container-Migration-Active: True"},{"line_number":52,"context_line":"    X-Container-Migration-Provider: fsystem"},{"line_number":53,"context_line":"    X-Container-Migration-Path: parent folders of the Migration-Source"},{"line_number":54,"context_line":"    X-Container-Migration-Source: folder that contains objects"},{"line_number":55,"context_line":""},{"line_number":56,"context_line":"    e.g. to migrate objects from folder /home/vacation/images"}],"source_content_type":"text/x-python","patch_set":18,"id":"1abeadc6_fa82cafe","line":53,"updated":"2014-08-27 15:01:18.000000000","message":"update doc, X-Container-Migration-Path was removed in a prior patchset, right?","commit_id":"97b2e67ebf70f20b8fd5f4883840872a3cc0d8f1"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"bc14f3b3b1132d94c7079f61e8fee122b68defcc","unresolved":false,"context_lines":[{"line_number":54,"context_line":""},{"line_number":55,"context_line":"    e.g. to migrate objects from folder /home/vacation/images"},{"line_number":56,"context_line":""},{"line_number":57,"context_line":"    X-Container-Migration-Path: /home/vacation"},{"line_number":58,"context_line":"    X-Container-Migration-Source: images"},{"line_number":59,"context_line":"    \"\"\""},{"line_number":60,"context_line":"    object_to_migrate \u003d None"}],"source_content_type":"text/x-python","patch_set":20,"id":"fa98f980_806662eb","line":57,"updated":"2014-09-04 15:07:44.000000000","message":"X-Container-Migration-Source: /home/vacation/images","commit_id":"d20cd4f09c40fd195da3e94aad079e0baa9fb2e4"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"bc14f3b3b1132d94c7079f61e8fee122b68defcc","unresolved":false,"context_lines":[{"line_number":123,"context_line":""},{"line_number":124,"context_line":"    def __init__(self, data_source, params):"},{"line_number":125,"context_line":"        \"\"\""},{"line_number":126,"context_line":"        :param data_source:  a folder that contains objects."},{"line_number":127,"context_line":"        \"\"\""},{"line_number":128,"context_line":"        self.data_source \u003d data_source"},{"line_number":129,"context_line":"        self.token_url \u003d params.get(\u0027token-url\u0027, \u0027\u0027)"}],"source_content_type":"text/x-python","patch_set":20,"id":"fa98f980_c045ca43","line":126,"updated":"2014-09-04 15:07:44.000000000","message":"nit: param params (i can see now that the container_md param that got deleted from version 16 was just misnamed.)","commit_id":"d20cd4f09c40fd195da3e94aad079e0baa9fb2e4"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"98ad43e485f854f6bbca1f5ffd0d3ea88e31fb1e","unresolved":false,"context_lines":[{"line_number":153,"context_line":"        :returns (metadata, object size, data stream, content type, timestamp)"},{"line_number":154,"context_line":"        \"\"\""},{"line_number":155,"context_line":"        if SWIFTCLIENT_IMPORTED:"},{"line_number":156,"context_line":"            storage_url, token \u003d client.get_auth(self.token_url,"},{"line_number":157,"context_line":"                                                 self.user, self.key)"},{"line_number":158,"context_line":"            parsed, conn \u003d http_connection(storage_url)"},{"line_number":159,"context_line":"            path \u003d \u0027%s/%s/%s\u0027 % (parsed.path, quote(self.data_source),"}],"source_content_type":"text/x-python","patch_set":21,"id":"fa98f980_cca18e47","line":156,"updated":"2014-09-12 16:58:58.000000000","message":"If the swift credentials are bad get_auth will raise ClientException - this eventually gets caught back in the middleware __call__ method and is mapped to a 400 Bad Request. I *think* maybe the get_auth call should be made inside the try/except below, so auth failure will result in DataMigrationDriverError which then causes a 404 to be returned.","commit_id":"f54f7ff4b834e9e893d701a9f7a9217096bad8ae"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"6d587660cb81286b69ba5fc6d1d73204db77974c","unresolved":false,"context_lines":[{"line_number":60,"context_line":"        if not self.is_valid_path(self.root_path):"},{"line_number":61,"context_line":"            raise DataMigrationDriverError(\u0027driver_fsystem_parent_path: \u0027 +"},{"line_number":62,"context_line":"                                           \u0027{0} \u0027.format((self.root_path)) +"},{"line_number":63,"context_line":"                                           \u0027 is invalid\u0027)"},{"line_number":64,"context_line":""},{"line_number":65,"context_line":"        if not self.is_valid_path(data_source):"},{"line_number":66,"context_line":"            raise DataMigrationDriverError(\u0027Migration source \u0027 +"}],"source_content_type":"text/x-python","patch_set":22,"id":"fa98f980_cb9dc90b","line":63,"updated":"2014-09-15 16:57:16.000000000","message":"I would prefer this to be checked as the driver is loaded during proxy start up and have an exception raised there","commit_id":"63a19a6152b5eae20641d541aa9fa1c7b9977208"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"b8b53fdd5d648e8f50fce62e7a022aaac4e1c1f5","unresolved":false,"context_lines":[{"line_number":60,"context_line":"        if not self.is_valid_path(self.root_path):"},{"line_number":61,"context_line":"            raise DataMigrationDriverError(\u0027driver_fsystem_parent_path: \u0027 +"},{"line_number":62,"context_line":"                                           \u0027{0} \u0027.format((self.root_path)) +"},{"line_number":63,"context_line":"                                           \u0027 is invalid\u0027)"},{"line_number":64,"context_line":""},{"line_number":65,"context_line":"        if not self.is_valid_path(data_source):"},{"line_number":66,"context_line":"            raise DataMigrationDriverError(\u0027Migration source \u0027 +"}],"source_content_type":"text/x-python","patch_set":22,"id":"fa98f980_6158608b","line":63,"in_reply_to":"fa98f980_a297bf38","updated":"2014-09-16 13:29:37.000000000","message":"Yes, I agree - but I think the checks can be encapsulated in each driver AND called earlier - see https://review.openstack.org/#/c/121831/","commit_id":"63a19a6152b5eae20641d541aa9fa1c7b9977208"},{"author":{"_account_id":9205,"name":"Gil Vernik","email":"gilv@il.ibm.com","username":"gilv"},"change_message_id":"0f84dbfb78d2a768bbf2eaef57566b312499a1ed","unresolved":false,"context_lines":[{"line_number":60,"context_line":"        if not self.is_valid_path(self.root_path):"},{"line_number":61,"context_line":"            raise DataMigrationDriverError(\u0027driver_fsystem_parent_path: \u0027 +"},{"line_number":62,"context_line":"                                           \u0027{0} \u0027.format((self.root_path)) +"},{"line_number":63,"context_line":"                                           \u0027 is invalid\u0027)"},{"line_number":64,"context_line":""},{"line_number":65,"context_line":"        if not self.is_valid_path(data_source):"},{"line_number":66,"context_line":"            raise DataMigrationDriverError(\u0027Migration source \u0027 +"}],"source_content_type":"text/x-python","patch_set":22,"id":"fa98f980_a297bf38","line":63,"in_reply_to":"fa98f980_cb9dc90b","updated":"2014-09-15 19:01:29.000000000","message":"My original idea was that code during proxy start up knows nothing about specific drivers and parameters. It only check that the defined driver which was defined can be imported. Each driver should check it\u0027s required parameters, file system driver for it\u0027s driver_system_path and some other arbitrary driver A for it\u0027s mandatory parameters X, Y,... Otherwise logic of the driver will spread to different components and it will be hard to introduce a new driver.\nDon\u0027t you agree with it?","commit_id":"63a19a6152b5eae20641d541aa9fa1c7b9977208"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"6d587660cb81286b69ba5fc6d1d73204db77974c","unresolved":false,"context_lines":[{"line_number":65,"context_line":"        if not self.is_valid_path(data_source):"},{"line_number":66,"context_line":"            raise DataMigrationDriverError(\u0027Migration source \u0027 +"},{"line_number":67,"context_line":"                                           \u0027{0} \u0027.format((data_source)) +"},{"line_number":68,"context_line":"                                           \u0027is invalid\u0027)"},{"line_number":69,"context_line":"        if data_source.startswith(os.path.sep):"},{"line_number":70,"context_line":"            data_source \u003d data_source[1:]"},{"line_number":71,"context_line":""}],"source_content_type":"text/x-python","patch_set":22,"id":"fa98f980_5df0c066","line":68,"updated":"2014-09-15 16:57:16.000000000","message":"I would prefer this to be checked when the container metadata is set, perhaps by calling a static method on the driver class. That method is passed the X-Container-Migration headers and raises an exception if any have bad values.","commit_id":"63a19a6152b5eae20641d541aa9fa1c7b9977208"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"6d587660cb81286b69ba5fc6d1d73204db77974c","unresolved":false,"context_lines":[{"line_number":88,"context_line":"        :param object_name: name of the object"},{"line_number":89,"context_line":"        :returns (metadata, object size, data stream, content type, timestamp)"},{"line_number":90,"context_line":"        \"\"\""},{"line_number":91,"context_line":"        file_path \u003d os.path.join(os.path.sep, self.data_source, object_name)"},{"line_number":92,"context_line":"        try:"},{"line_number":93,"context_line":"            statinfo \u003d os.stat(file_path)"},{"line_number":94,"context_line":"        except Exception:"}],"source_content_type":"text/x-python","patch_set":22,"id":"fa98f980_56289d85","line":91,"updated":"2014-09-15 16:57:16.000000000","message":"need to check valid_path again here in case object_name has a \u0027../\u0027 in it","commit_id":"63a19a6152b5eae20641d541aa9fa1c7b9977208"},{"author":{"_account_id":9625,"name":"Thiago da Silva","email":"thiagodasilva@gmail.com","username":"thiago"},"change_message_id":"a2a5457425f0ff8074b00d7f9652038db33707d9","unresolved":false,"context_lines":[{"line_number":98,"context_line":"            md \u003d dict()"},{"line_number":99,"context_line":"            md[\u0027uid\u0027] \u003d statinfo.st_uid"},{"line_number":100,"context_line":"            md[\u0027gid\u0027] \u003d statinfo.st_gid"},{"line_number":101,"context_line":"            last_modified \u003d statinfo.st_mtime"},{"line_number":102,"context_line":"            content_type, encoding \u003d mimetypes.guess_type(file_path)"},{"line_number":103,"context_line":"            self.object_to_migrate \u003d open(file_path, \"r\")"},{"line_number":104,"context_line":"            try:"}],"source_content_type":"text/x-python","patch_set":22,"id":"fa98f980_8ba6500d","line":101,"updated":"2014-09-16 01:51:51.000000000","message":"I think this should be st_ctime","commit_id":"63a19a6152b5eae20641d541aa9fa1c7b9977208"},{"author":{"_account_id":9625,"name":"Thiago da Silva","email":"thiagodasilva@gmail.com","username":"thiago"},"change_message_id":"22bbcb285c33aed1a40f6ecc8e4a141477619650","unresolved":false,"context_lines":[{"line_number":32,"context_line":"    \"\"\""},{"line_number":33,"context_line":"    A driver to migrate objects from disk."},{"line_number":34,"context_line":"    It is mandatory to set `driver_fsystem_parent_path` with non"},{"line_number":35,"context_line":"    empty value in the proxy-server configration prior usage of this driver."},{"line_number":36,"context_line":"    This will be used a root path for all migration process."},{"line_number":37,"context_line":""},{"line_number":38,"context_line":"    To use this driver the following container metadata required:"}],"source_content_type":"text/x-python","patch_set":24,"id":"da9df570_123b2539","line":35,"updated":"2014-09-23 18:24:05.000000000","message":"*configuration","commit_id":"f8af425febd8a059f0da2262e4446f3be0ece9b3"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"54baaf7d3eee021bebc6c045801f7d4257b569c3","unresolved":false,"context_lines":[{"line_number":44,"context_line":""},{"line_number":45,"context_line":"    e.g. to migrate objects from folder /home/vacation/images"},{"line_number":46,"context_line":"    driver_fsystem_parent_path \u003d /home/"},{"line_number":47,"context_line":"    X-Container-Migration-Source: vacation/images"},{"line_number":48,"context_line":"    \"\"\""},{"line_number":49,"context_line":"    object_to_migrate \u003d None"},{"line_number":50,"context_line":"    driver_loaded \u003d True"}],"source_content_type":"text/x-python","patch_set":27,"id":"9aa7fdbe_db387fd5","line":47,"updated":"2014-10-24 09:42:41.000000000","message":"what\u0027s the security model here?  Anyone can set up a migration from the shared root?","commit_id":"52eeff33144174bbd51593163c5391ce48e08473"},{"author":{"_account_id":9205,"name":"Gil Vernik","email":"gilv@il.ibm.com","username":"gilv"},"change_message_id":"9f05a155ac4d16a8c2b6652f81a409a5c67e71bc","unresolved":false,"context_lines":[{"line_number":44,"context_line":""},{"line_number":45,"context_line":"    e.g. to migrate objects from folder /home/vacation/images"},{"line_number":46,"context_line":"    driver_fsystem_parent_path \u003d /home/"},{"line_number":47,"context_line":"    X-Container-Migration-Source: vacation/images"},{"line_number":48,"context_line":"    \"\"\""},{"line_number":49,"context_line":"    object_to_migrate \u003d None"},{"line_number":50,"context_line":"    driver_loaded \u003d True"}],"source_content_type":"text/x-python","patch_set":27,"id":"7a8c0949_e7950be5","line":47,"in_reply_to":"9aa7fdbe_db387fd5","updated":"2014-11-02 02:27:13.000000000","message":"Yes, at this point, with respect to file system driver, if the migration is configured, than everybody can setup a migration from the root.","commit_id":"52eeff33144174bbd51593163c5391ce48e08473"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"54baaf7d3eee021bebc6c045801f7d4257b569c3","unresolved":false,"context_lines":[{"line_number":61,"context_line":"            data_source \u003d data_source[1:]"},{"line_number":62,"context_line":""},{"line_number":63,"context_line":"        self.data_source \u003d os.path.join(os.path.sep, self.root_path,"},{"line_number":64,"context_line":"                                        data_source)"},{"line_number":65,"context_line":"        self.logger \u003d logger"},{"line_number":66,"context_line":""},{"line_number":67,"context_line":"    @staticmethod"}],"source_content_type":"text/x-python","patch_set":27,"id":"9aa7fdbe_9b2357a6","line":64,"updated":"2014-10-24 09:42:41.000000000","message":"so on a set of load-balanced proxies we assume that the root_path is some sort of shared file system?","commit_id":"52eeff33144174bbd51593163c5391ce48e08473"},{"author":{"_account_id":9205,"name":"Gil Vernik","email":"gilv@il.ibm.com","username":"gilv"},"change_message_id":"9f05a155ac4d16a8c2b6652f81a409a5c67e71bc","unresolved":false,"context_lines":[{"line_number":61,"context_line":"            data_source \u003d data_source[1:]"},{"line_number":62,"context_line":""},{"line_number":63,"context_line":"        self.data_source \u003d os.path.join(os.path.sep, self.root_path,"},{"line_number":64,"context_line":"                                        data_source)"},{"line_number":65,"context_line":"        self.logger \u003d logger"},{"line_number":66,"context_line":""},{"line_number":67,"context_line":"    @staticmethod"}],"source_content_type":"text/x-python","patch_set":27,"id":"7a8c0949_079bf718","line":64,"in_reply_to":"9aa7fdbe_9b2357a6","updated":"2014-11-02 02:27:13.000000000","message":"Root path can be any folder in the local file system or a folder in NAS, etc..","commit_id":"52eeff33144174bbd51593163c5391ce48e08473"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"54baaf7d3eee021bebc6c045801f7d4257b569c3","unresolved":false,"context_lines":[{"line_number":236,"context_line":"                    metadata[key] \u003d value"},{"line_number":237,"context_line":"            return metadata, resp_headers.get(\u0027content-length\u0027),\\"},{"line_number":238,"context_line":"                resp, resp_headers.get(\u0027content-type\u0027), \\"},{"line_number":239,"context_line":"                resp_headers.get(\u0027x-timestamp\u0027)"},{"line_number":240,"context_line":"        return {}, None, None, None, None"}],"source_content_type":"text/x-python","patch_set":27,"id":"9aa7fdbe_7b1eb360","line":239,"updated":"2014-10-24 09:42:41.000000000","message":"much better to use a dictionary return value here instead of 5-tuple (6-tuple?)","commit_id":"52eeff33144174bbd51593163c5391ce48e08473"},{"author":{"_account_id":9205,"name":"Gil Vernik","email":"gilv@il.ibm.com","username":"gilv"},"change_message_id":"9f05a155ac4d16a8c2b6652f81a409a5c67e71bc","unresolved":false,"context_lines":[{"line_number":236,"context_line":"                    metadata[key] \u003d value"},{"line_number":237,"context_line":"            return metadata, resp_headers.get(\u0027content-length\u0027),\\"},{"line_number":238,"context_line":"                resp, resp_headers.get(\u0027content-type\u0027), \\"},{"line_number":239,"context_line":"                resp_headers.get(\u0027x-timestamp\u0027)"},{"line_number":240,"context_line":"        return {}, None, None, None, None"}],"source_content_type":"text/x-python","patch_set":27,"id":"7a8c0949_27a0f342","line":239,"in_reply_to":"9aa7fdbe_7b1eb360","updated":"2014-11-02 02:27:13.000000000","message":"I wanted to explicitly define what is the return value of the get_object function. Any new driver that will be implemented, suppose to return the same parameters. It\u0027s like interface. With dictionary, it\u0027s less explicit, people may return there whatever they want. But I do agree that dictionary is much nice, however afraid that it may mislead those will implement additional drivers.","commit_id":"52eeff33144174bbd51593163c5391ce48e08473"}],"swift/common/middleware/data_migration.py":[{"author":{"_account_id":330,"name":"John Dickinson","email":"me@not.mn","username":"notmyname"},"change_message_id":"85a08e4e966678eb7d9f8e3a1fde04de1d3ca3a3","unresolved":false,"context_lines":[{"line_number":180,"context_line":"            if migration_active is not None:"},{"line_number":181,"context_line":"                if migration_active not in [\u0027True\u0027, \u0027False\u0027]:"},{"line_number":182,"context_line":"                    return HTTPBadRequest(body\u003d\u0027Migration active key \u0027 +"},{"line_number":183,"context_line":"                                          \u0027should be True or False\u0027)"},{"line_number":184,"context_line":""},{"line_number":185,"context_line":"        original_resp \u003d req.get_response(self.app)"},{"line_number":186,"context_line":"        if (original_resp.status_int \u003d\u003d NOT_FOUND):"}],"source_content_type":"text/x-python","patch_set":2,"id":"AAAAT3%2F%2F1KE%3D","line":183,"updated":"2013-12-30 15:40:19.000000000","message":"I\u0027d suggest using utils.config_true_value() for consistency with other settings (and flexibility on what is allowed).","commit_id":"3d6b0eaf7d09738f13d44ba1384c2cdb31839027"},{"author":{"_account_id":330,"name":"John Dickinson","email":"me@not.mn","username":"notmyname"},"change_message_id":"85a08e4e966678eb7d9f8e3a1fde04de1d3ca3a3","unresolved":false,"context_lines":[{"line_number":182,"context_line":"                    return HTTPBadRequest(body\u003d\u0027Migration active key \u0027 +"},{"line_number":183,"context_line":"                                          \u0027should be True or False\u0027)"},{"line_number":184,"context_line":""},{"line_number":185,"context_line":"        original_resp \u003d req.get_response(self.app)"},{"line_number":186,"context_line":"        if (original_resp.status_int \u003d\u003d NOT_FOUND):"},{"line_number":187,"context_line":"            if obj and container and account and req.method in [\u0027GET\u0027, \u0027HEAD\u0027]:"},{"line_number":188,"context_line":"                container_info \u003d get_container_info(req.environ, self.app,"}],"source_content_type":"text/x-python","patch_set":2,"id":"AAAAT3%2F%2F1H8%3D","line":185,"updated":"2013-12-30 15:40:19.000000000","message":"note to self: I need to validate if this requires using WSGIContext()","commit_id":"3d6b0eaf7d09738f13d44ba1384c2cdb31839027"},{"author":{"_account_id":9205,"name":"Gil Vernik","email":"gilv@il.ibm.com","username":"gilv"},"change_message_id":"b98c6034c68c3fe6f040a71c3a5632df22805277","unresolved":false,"context_lines":[{"line_number":182,"context_line":"                    return HTTPBadRequest(body\u003d\u0027Migration active key \u0027 +"},{"line_number":183,"context_line":"                                          \u0027should be True or False\u0027)"},{"line_number":184,"context_line":""},{"line_number":185,"context_line":"        original_resp \u003d req.get_response(self.app)"},{"line_number":186,"context_line":"        if (original_resp.status_int \u003d\u003d NOT_FOUND):"},{"line_number":187,"context_line":"            if obj and container and account and req.method in [\u0027GET\u0027, \u0027HEAD\u0027]:"},{"line_number":188,"context_line":"                container_info \u003d get_container_info(req.environ, self.app,"}],"source_content_type":"text/x-python","patch_set":2,"id":"AAAAT3%2F%2Fy3o%3D","line":185,"in_reply_to":"AAAAT3%2F%2F0rI%3D","updated":"2013-12-31 12:30:04.000000000","message":"I keep thinking about your comment and I have a feeling that I miss something important here. Can you please explain me one more time, what is your concern here? As I see it, original_resp \u003d req.get_response(self.app) just allow to proceed original request and all what my middleware cares is to check whether the response is 404 or something else.  What does it means that there is no guarantee that I am getting correct response?  That some other middleware between this one and proxy controller may set another response code, like 400 Bad Request? or something else?  and so my code will miss object not found case?\nWhy WSGIConext() solves it? Because it uses iterator that iterate over all responses?\nIf so, i can replace my code with something like this...\n\nclass DataMigrationContext(WSGIContext):\n\n    def __init__(self, wsgi_app):\n        WSGIContext.__init__(self, wsgi_app)\n\n    def hadle_original_call(self, req, start_response):\n        app_resp \u003d self._app_call(req.environ)\n        start_response(self._response_status,\n                       self._response_headers,\n                       self._response_exc_info)\n\treq.environ[\u0027original_call\u0027] \u003d self._response_status\n        return app_resp\n\n..................\nctx \u003d DataMigrationContext(self.app)\nres \u003d ctx.hadle_original_call(req, start_response)\noriginal_status \u003d req.environ[\u0027original_status\u0027]\ndel req.environ[\u0027original_status\u0027]\nif (original_status \u003d\u003d NOT_FOUND ):\n     migration logic....","commit_id":"3d6b0eaf7d09738f13d44ba1384c2cdb31839027"},{"author":{"_account_id":330,"name":"John Dickinson","email":"me@not.mn","username":"notmyname"},"change_message_id":"3e948b030b54b16d15e513256b42c31d304da0c9","unresolved":false,"context_lines":[{"line_number":182,"context_line":"                    return HTTPBadRequest(body\u003d\u0027Migration active key \u0027 +"},{"line_number":183,"context_line":"                                          \u0027should be True or False\u0027)"},{"line_number":184,"context_line":""},{"line_number":185,"context_line":"        original_resp \u003d req.get_response(self.app)"},{"line_number":186,"context_line":"        if (original_resp.status_int \u003d\u003d NOT_FOUND):"},{"line_number":187,"context_line":"            if obj and container and account and req.method in [\u0027GET\u0027, \u0027HEAD\u0027]:"},{"line_number":188,"context_line":"                container_info \u003d get_container_info(req.environ, self.app,"}],"source_content_type":"text/x-python","patch_set":2,"id":"AAAAT3%2F%2F0rI%3D","line":185,"in_reply_to":"AAAAT3%2F%2F1H8%3D","updated":"2013-12-30 19:13:53.000000000","message":"Yes, this will need to be refactored to use the functionality provided by WSGIContext(). If you don\u0027t, then you don\u0027t have any guarantee that the response you get is actually correct right now. WSGIContext will ensure that all other middleware has had a chance to process and set the response code and headers.","commit_id":"3d6b0eaf7d09738f13d44ba1384c2cdb31839027"},{"author":{"_account_id":330,"name":"John Dickinson","email":"me@not.mn","username":"notmyname"},"change_message_id":"f5acdf7cda0cd669b8349f2526f463c266c048c5","unresolved":false,"context_lines":[{"line_number":182,"context_line":"                    return HTTPBadRequest(body\u003d\u0027Migration active key \u0027 +"},{"line_number":183,"context_line":"                                          \u0027should be True or False\u0027)"},{"line_number":184,"context_line":""},{"line_number":185,"context_line":"        original_resp \u003d req.get_response(self.app)"},{"line_number":186,"context_line":"        if (original_resp.status_int \u003d\u003d NOT_FOUND):"},{"line_number":187,"context_line":"            if obj and container and account and req.method in [\u0027GET\u0027, \u0027HEAD\u0027]:"},{"line_number":188,"context_line":"                container_info \u003d get_container_info(req.environ, self.app,"}],"source_content_type":"text/x-python","patch_set":2,"id":"AAAAT3%2F%2FuY8%3D","line":185,"in_reply_to":"AAAAT3%2F%2Fy3o%3D","updated":"2014-01-02 21:14:16.000000000","message":"The reason to use WSGIContext() is because the response is only actually a generator and therefore isn\u0027t fully evaluated until the wsgi server starts evaluating the generator. (You can see a small example of this at https://gist.github.com/notmyname/8225201.)\n\nWhat this means is that when you get the response (without using WSGIContext), you may not have access to the correct response code since it wouldn\u0027t be evaluated yet. WSGIContext solves this by keeping track of the response codes and headers from the application (ie everything after you in the pipeline) and guaranteeing that you have the right response code when you ask for it.\n\nIt looks like you have the right idea in your example, but take a look at catch_errors.py for a very simple example of how WSGIContext() should be used.","commit_id":"3d6b0eaf7d09738f13d44ba1384c2cdb31839027"},{"author":{"_account_id":330,"name":"John Dickinson","email":"me@not.mn","username":"notmyname"},"change_message_id":"85a08e4e966678eb7d9f8e3a1fde04de1d3ca3a3","unresolved":false,"context_lines":[{"line_number":297,"context_line":"        the_class \u003d getattr(module, cl)"},{"line_number":298,"context_line":"        migration_conf[dr] \u003d {\u0027keys\u0027: keys, \u0027the_class\u0027: the_class,"},{"line_number":299,"context_line":"                              \u0027driver_loaded\u0027: the_class.driver_loaded}"},{"line_number":300,"context_line":""},{"line_number":301,"context_line":"    def data_migration_filter(app):"},{"line_number":302,"context_line":"        return DataMigrationMiddleware(app, conf, migration_conf)"},{"line_number":303,"context_line":"    return data_migration_filter"}],"source_content_type":"text/x-python","patch_set":2,"id":"AAAAT3%2F%2F1Gs%3D","line":300,"updated":"2013-12-30 15:40:19.000000000","message":"please add a call to utils.register_swift_info() so this functionality is client-discoverable.\n\nI\u0027d like to see (at minimum) that the functionality is available and also what drivers are available.","commit_id":"3d6b0eaf7d09738f13d44ba1384c2cdb31839027"},{"author":{"_account_id":6889,"name":"Fabien Boucher","email":"fboucher@redhat.com","username":"fabien-boucher"},"change_message_id":"6989cb8bfbcafb02f9c9a44fa05b2f1bed71f27d","unresolved":false,"context_lines":[{"line_number":41,"context_line":"4. (Optional) User defined custom driver."},{"line_number":42,"context_line":""},{"line_number":43,"context_line":"Data migration middleware run on Proxy with the configuration as follows:"},{"line_number":44,"context_line":"*************************************************************************"},{"line_number":45,"context_line":"[filter:data-migration]"},{"line_number":46,"context_line":"use \u003d egg:swift#data_migration"},{"line_number":47,"context_line":""}],"source_content_type":"text/x-python","patch_set":5,"id":"AAAAUH%2F%2F7fA%3D","line":44,"updated":"2014-01-06 21:57:30.000000000","message":"Could be nice to also have an example of a pipeline line like here:\nhttps://github.com/openstack/swift/blob/master/swift/common/middleware/account_quotas.py#L31","commit_id":"94be0c854c189f22f9f1debf0f4a022c09fc3f40"},{"author":{"_account_id":9205,"name":"Gil Vernik","email":"gilv@il.ibm.com","username":"gilv"},"change_message_id":"5c8750ba8b7931c1aa887dbab94bb58ebf67c7d3","unresolved":false,"context_lines":[{"line_number":41,"context_line":"4. (Optional) User defined custom driver."},{"line_number":42,"context_line":""},{"line_number":43,"context_line":"Data migration middleware run on Proxy with the configuration as follows:"},{"line_number":44,"context_line":"*************************************************************************"},{"line_number":45,"context_line":"[filter:data-migration]"},{"line_number":46,"context_line":"use \u003d egg:swift#data_migration"},{"line_number":47,"context_line":""}],"source_content_type":"text/x-python","patch_set":5,"id":"AAAAUH%2F%2F494%3D","line":44,"in_reply_to":"AAAAUH%2F%2F7fA%3D","updated":"2014-01-07 07:34:33.000000000","message":"I will improve documentation","commit_id":"94be0c854c189f22f9f1debf0f4a022c09fc3f40"},{"author":{"_account_id":6889,"name":"Fabien Boucher","email":"fboucher@redhat.com","username":"fabien-boucher"},"change_message_id":"6989cb8bfbcafb02f9c9a44fa05b2f1bed71f27d","unresolved":false,"context_lines":[{"line_number":151,"context_line":"    def __init__(self, wsgi_app):"},{"line_number":152,"context_line":"        WSGIContext.__init__(self, wsgi_app)"},{"line_number":153,"context_line":""},{"line_number":154,"context_line":"    def hadle_original_call(self, env, start_response):"},{"line_number":155,"context_line":"        app_resp \u003d self._app_call(env)"},{"line_number":156,"context_line":"        if self._response_headers is None:"},{"line_number":157,"context_line":"            self._response_headers \u003d []"}],"source_content_type":"text/x-python","patch_set":5,"id":"AAAAUH%2F%2F7ek%3D","line":154,"updated":"2014-01-06 21:57:30.000000000","message":"is this a typo ? s/hadle/handle/ ?","commit_id":"94be0c854c189f22f9f1debf0f4a022c09fc3f40"},{"author":{"_account_id":9205,"name":"Gil Vernik","email":"gilv@il.ibm.com","username":"gilv"},"change_message_id":"5c8750ba8b7931c1aa887dbab94bb58ebf67c7d3","unresolved":false,"context_lines":[{"line_number":151,"context_line":"    def __init__(self, wsgi_app):"},{"line_number":152,"context_line":"        WSGIContext.__init__(self, wsgi_app)"},{"line_number":153,"context_line":""},{"line_number":154,"context_line":"    def hadle_original_call(self, env, start_response):"},{"line_number":155,"context_line":"        app_resp \u003d self._app_call(env)"},{"line_number":156,"context_line":"        if self._response_headers is None:"},{"line_number":157,"context_line":"            self._response_headers \u003d []"}],"source_content_type":"text/x-python","patch_set":5,"id":"AAAAUH%2F%2F4%2FQ%3D","line":154,"in_reply_to":"AAAAUH%2F%2F7ek%3D","updated":"2014-01-07 07:34:33.000000000","message":"Done","commit_id":"94be0c854c189f22f9f1debf0f4a022c09fc3f40"},{"author":{"_account_id":6889,"name":"Fabien Boucher","email":"fboucher@redhat.com","username":"fabien-boucher"},"change_message_id":"6989cb8bfbcafb02f9c9a44fa05b2f1bed71f27d","unresolved":false,"context_lines":[{"line_number":182,"context_line":"        response \u0027404 Not Found\u0027."},{"line_number":183,"context_line":"        \"\"\""},{"line_number":184,"context_line":"        req \u003d Request(env)"},{"line_number":185,"context_line":"        original_env \u003d req.environ.copy()"},{"line_number":186,"context_line":"        try:"},{"line_number":187,"context_line":"            (version, account, container, obj) \u003d req.split_path(3, 4, True)"},{"line_number":188,"context_line":"        except ValueError:"}],"source_content_type":"text/x-python","patch_set":5,"id":"AAAAUH%2F%2F7mM%3D","line":185,"updated":"2014-01-06 21:57:30.000000000","message":"No need to copy the env right now as the copy is not use if X-Container-Meta-Migration is not used. Can be instead done line 210 ?","commit_id":"94be0c854c189f22f9f1debf0f4a022c09fc3f40"},{"author":{"_account_id":9205,"name":"Gil Vernik","email":"gilv@il.ibm.com","username":"gilv"},"change_message_id":"5c8750ba8b7931c1aa887dbab94bb58ebf67c7d3","unresolved":false,"context_lines":[{"line_number":182,"context_line":"        response \u0027404 Not Found\u0027."},{"line_number":183,"context_line":"        \"\"\""},{"line_number":184,"context_line":"        req \u003d Request(env)"},{"line_number":185,"context_line":"        original_env \u003d req.environ.copy()"},{"line_number":186,"context_line":"        try:"},{"line_number":187,"context_line":"            (version, account, container, obj) \u003d req.split_path(3, 4, True)"},{"line_number":188,"context_line":"        except ValueError:"}],"source_content_type":"text/x-python","patch_set":5,"id":"AAAAUH%2F%2F4%2B4%3D","line":185,"in_reply_to":"AAAAUH%2F%2F7mM%3D","updated":"2014-01-07 07:34:33.000000000","message":"I will move it further in the code, but still copy need to be done prior original_resp \u003d ctx.handle_original_call(env, start_response). Otherwise it will be too late, since env will already be changed.","commit_id":"94be0c854c189f22f9f1debf0f4a022c09fc3f40"},{"author":{"_account_id":6889,"name":"Fabien Boucher","email":"fboucher@redhat.com","username":"fabien-boucher"},"change_message_id":"6989cb8bfbcafb02f9c9a44fa05b2f1bed71f27d","unresolved":false,"context_lines":[{"line_number":212,"context_line":"        original_resp_status \u003d env[\u0027swift.original_status\u0027]"},{"line_number":213,"context_line":"        del env[\u0027swift.original_status\u0027]"},{"line_number":214,"context_line":"        if (original_resp_status \u003d\u003d NOT_FOUND):"},{"line_number":215,"context_line":"            if obj and container and account and req.method in [\u0027GET\u0027, \u0027HEAD\u0027]:"},{"line_number":216,"context_line":"                container_info \u003d get_container_info(req.environ, self.app,"},{"line_number":217,"context_line":"                                                    swift_source\u003d\u0027DM\u0027)"},{"line_number":218,"context_line":"                container_md \u003d container_info.get(\u0027meta\u0027, {})"}],"source_content_type":"text/x-python","patch_set":5,"id":"AAAAUH%2F%2F7pI%3D","line":215,"updated":"2014-01-06 21:57:30.000000000","message":"In a second time you should handle the COPY verb too.","commit_id":"94be0c854c189f22f9f1debf0f4a022c09fc3f40"},{"author":{"_account_id":9205,"name":"Gil Vernik","email":"gilv@il.ibm.com","username":"gilv"},"change_message_id":"5c8750ba8b7931c1aa887dbab94bb58ebf67c7d3","unresolved":false,"context_lines":[{"line_number":212,"context_line":"        original_resp_status \u003d env[\u0027swift.original_status\u0027]"},{"line_number":213,"context_line":"        del env[\u0027swift.original_status\u0027]"},{"line_number":214,"context_line":"        if (original_resp_status \u003d\u003d NOT_FOUND):"},{"line_number":215,"context_line":"            if obj and container and account and req.method in [\u0027GET\u0027, \u0027HEAD\u0027]:"},{"line_number":216,"context_line":"                container_info \u003d get_container_info(req.environ, self.app,"},{"line_number":217,"context_line":"                                                    swift_source\u003d\u0027DM\u0027)"},{"line_number":218,"context_line":"                container_md \u003d container_info.get(\u0027meta\u0027, {})"}],"source_content_type":"text/x-python","patch_set":5,"id":"AAAAUH%2F%2F4%2BM%3D","line":215,"in_reply_to":"AAAAUH%2F%2F7pI%3D","updated":"2014-01-07 07:34:33.000000000","message":"I want to be sure i understand this comment correctly. You mean, I need to make also copy for the get_container_info, since it might change env also?","commit_id":"94be0c854c189f22f9f1debf0f4a022c09fc3f40"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"07d88a7d47e7b9cf9097bfb0d8fca63c75e0bbd6","unresolved":false,"context_lines":[{"line_number":49,"context_line":""},{"line_number":50,"context_line":"# List of supported drivers. This is a dynamic list."},{"line_number":51,"context_line":"supported_drivers \u003d fsystem,libcloud_s3,swift"},{"line_number":52,"context_line":"# For each driver XXX should be define"},{"line_number":53,"context_line":"# driver_XXX_keys ( input parameters for the driver. This is the list of"},{"line_number":54,"context_line":"# required metadata headers to be set on container during data migration"},{"line_number":55,"context_line":"# setup process )"}],"source_content_type":"text/x-python","patch_set":8,"id":"AAAAUX%2F%2FPyI%3D","line":52,"updated":"2014-02-14 17:11:03.000000000","message":"s/define/defined","commit_id":"31b4aaeaebcc8bcec9eed67a0bb65352f55f9467"},{"author":{"_account_id":9205,"name":"Gil Vernik","email":"gilv@il.ibm.com","username":"gilv"},"change_message_id":"0fb520e153242eccdfa7133da32b7b0c542a5a7b","unresolved":false,"context_lines":[{"line_number":49,"context_line":""},{"line_number":50,"context_line":"# List of supported drivers. This is a dynamic list."},{"line_number":51,"context_line":"supported_drivers \u003d fsystem,libcloud_s3,swift"},{"line_number":52,"context_line":"# For each driver XXX should be define"},{"line_number":53,"context_line":"# driver_XXX_keys ( input parameters for the driver. This is the list of"},{"line_number":54,"context_line":"# required metadata headers to be set on container during data migration"},{"line_number":55,"context_line":"# setup process )"}],"source_content_type":"text/x-python","patch_set":8,"id":"AAAAUX%2F%2FJTs%3D","line":52,"in_reply_to":"AAAAUX%2F%2FPyI%3D","updated":"2014-02-16 08:59:15.000000000","message":"Done","commit_id":"31b4aaeaebcc8bcec9eed67a0bb65352f55f9467"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"07d88a7d47e7b9cf9097bfb0d8fca63c75e0bbd6","unresolved":false,"context_lines":[{"line_number":205,"context_line":"            elif req.method in [\u0027PUT\u0027, \u0027POST\u0027]:"},{"line_number":206,"context_line":"                prov \u003d req.headers.get(\u0027X-Container-Migration-Provider\u0027)"},{"line_number":207,"context_line":"                if prov is not None:"},{"line_number":208,"context_line":"                    if prov.startswith(\u0027libcloud\u0027) and not prov.endswith(\u0027s3\u0027):"},{"line_number":209,"context_line":"                            return HTTPBadRequest(body\u003d\u0027Currently only S3 \u0027 +"},{"line_number":210,"context_line":"                                                  \u0027driver of \u0027 +"},{"line_number":211,"context_line":"                                                  \u0027LibCloud is \u0027 +"}],"source_content_type":"text/x-python","patch_set":8,"id":"AAAAUX%2F%2FPe0%3D","line":208,"updated":"2014-02-14 17:11:03.000000000","message":"why does libcloud get special treatment ? why not just treat libcloud-not-s3 the same as any other provider thats not found in conf and return \u0027Invalid migration provider\u0027?","commit_id":"31b4aaeaebcc8bcec9eed67a0bb65352f55f9467"},{"author":{"_account_id":9205,"name":"Gil Vernik","email":"gilv@il.ibm.com","username":"gilv"},"change_message_id":"0fb520e153242eccdfa7133da32b7b0c542a5a7b","unresolved":false,"context_lines":[{"line_number":205,"context_line":"            elif req.method in [\u0027PUT\u0027, \u0027POST\u0027]:"},{"line_number":206,"context_line":"                prov \u003d req.headers.get(\u0027X-Container-Migration-Provider\u0027)"},{"line_number":207,"context_line":"                if prov is not None:"},{"line_number":208,"context_line":"                    if prov.startswith(\u0027libcloud\u0027) and not prov.endswith(\u0027s3\u0027):"},{"line_number":209,"context_line":"                            return HTTPBadRequest(body\u003d\u0027Currently only S3 \u0027 +"},{"line_number":210,"context_line":"                                                  \u0027driver of \u0027 +"},{"line_number":211,"context_line":"                                                  \u0027LibCloud is \u0027 +"}],"source_content_type":"text/x-python","patch_set":8,"id":"AAAAUX%2F%2FJTQ%3D","line":208,"in_reply_to":"AAAAUX%2F%2FPe0%3D","updated":"2014-02-16 08:59:15.000000000","message":"LibCloud is actually a family of drivers that contains a lot of drivers for various clouds, it\u0027s different from other drivers in my code, since they are not family of drivers. My current code has support for S3 and the the only reason for that -  i have an account in S3 and can easily test it. However i want to emphasize the benefits of LibCloud, if some other member would like to test another driver of LibCloud - he can adapt the code to support it also. My framework does support LibCloud with any driver, but the driver itself need to be adapted.\nThis is just temporarily verification, i assume it will be removed or changed in separate patches that will expose more drivers of LibCloud.","commit_id":"31b4aaeaebcc8bcec9eed67a0bb65352f55f9467"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"07d88a7d47e7b9cf9097bfb0d8fca63c75e0bbd6","unresolved":false,"context_lines":[{"line_number":215,"context_line":"                        return HTTPBadRequest(body\u003d\u0027Invalid migration\u0027 +"},{"line_number":216,"context_line":"                                              \u0027provider\u0027)(env, start_response)"},{"line_number":217,"context_line":"                    if self.migration_conf[prov][\u0027driver_loaded\u0027] is False:"},{"line_number":218,"context_line":"                        return HTTPBadRequest(body\u003d\u0027Invalid access\u0027 +"},{"line_number":219,"context_line":"                                              \u0027 driver\u0027)(env, start_response)"},{"line_number":220,"context_line":""},{"line_number":221,"context_line":"                    for key in self.migration_conf[prov][\"keys\"]:"}],"source_content_type":"text/x-python","patch_set":8,"id":"AAAAUX%2F%2FPds%3D","line":218,"updated":"2014-02-14 17:11:03.000000000","message":"Is this a BadRequest (i.e. client error)? or should it be ServiceUnavailable? client has specified a valid provider but there\u0027s something wrong with the internal driver loading.","commit_id":"31b4aaeaebcc8bcec9eed67a0bb65352f55f9467"},{"author":{"_account_id":9205,"name":"Gil Vernik","email":"gilv@il.ibm.com","username":"gilv"},"change_message_id":"0fb520e153242eccdfa7133da32b7b0c542a5a7b","unresolved":false,"context_lines":[{"line_number":215,"context_line":"                        return HTTPBadRequest(body\u003d\u0027Invalid migration\u0027 +"},{"line_number":216,"context_line":"                                              \u0027provider\u0027)(env, start_response)"},{"line_number":217,"context_line":"                    if self.migration_conf[prov][\u0027driver_loaded\u0027] is False:"},{"line_number":218,"context_line":"                        return HTTPBadRequest(body\u003d\u0027Invalid access\u0027 +"},{"line_number":219,"context_line":"                                              \u0027 driver\u0027)(env, start_response)"},{"line_number":220,"context_line":""},{"line_number":221,"context_line":"                    for key in self.migration_conf[prov][\"keys\"]:"}],"source_content_type":"text/x-python","patch_set":8,"id":"AAAAUX%2F%2FJTE%3D","line":218,"in_reply_to":"AAAAUX%2F%2FPds%3D","updated":"2014-02-16 08:59:15.000000000","message":"Good point.","commit_id":"31b4aaeaebcc8bcec9eed67a0bb65352f55f9467"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"07d88a7d47e7b9cf9097bfb0d8fca63c75e0bbd6","unresolved":false,"context_lines":[{"line_number":222,"context_line":"                        if (req.headers.get(\u0027X-Container-Migration-\u0027 +"},{"line_number":223,"context_line":"                                            key.title()) is None):"},{"line_number":224,"context_line":"                            return HTTPBadRequest(body\u003d\u0027Missing required \u0027 +"},{"line_number":225,"context_line":"                                                  \u0027key\u0027)(env, start_response)"},{"line_number":226,"context_line":""},{"line_number":227,"context_line":"                for key, val in req.headers.iteritems():"},{"line_number":228,"context_line":"                    if key.startswith(\u0027X-Container-Migration-\u0027):"}],"source_content_type":"text/x-python","patch_set":8,"id":"AAAAUX%2F%2FPcQ%3D","line":225,"updated":"2014-02-14 17:11:03.000000000","message":"did you mean to quote \u0027key\u0027 here? it might be helpful to return the name of the missing key.","commit_id":"31b4aaeaebcc8bcec9eed67a0bb65352f55f9467"},{"author":{"_account_id":9205,"name":"Gil Vernik","email":"gilv@il.ibm.com","username":"gilv"},"change_message_id":"0fb520e153242eccdfa7133da32b7b0c542a5a7b","unresolved":false,"context_lines":[{"line_number":222,"context_line":"                        if (req.headers.get(\u0027X-Container-Migration-\u0027 +"},{"line_number":223,"context_line":"                                            key.title()) is None):"},{"line_number":224,"context_line":"                            return HTTPBadRequest(body\u003d\u0027Missing required \u0027 +"},{"line_number":225,"context_line":"                                                  \u0027key\u0027)(env, start_response)"},{"line_number":226,"context_line":""},{"line_number":227,"context_line":"                for key, val in req.headers.iteritems():"},{"line_number":228,"context_line":"                    if key.startswith(\u0027X-Container-Migration-\u0027):"}],"source_content_type":"text/x-python","patch_set":8,"id":"AAAAUX%2F%2FJMI%3D","line":225,"in_reply_to":"AAAAUX%2F%2FPcQ%3D","updated":"2014-02-16 08:59:15.000000000","message":"right, good idea","commit_id":"31b4aaeaebcc8bcec9eed67a0bb65352f55f9467"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"07d88a7d47e7b9cf9097bfb0d8fca63c75e0bbd6","unresolved":false,"context_lines":[{"line_number":224,"context_line":"                            return HTTPBadRequest(body\u003d\u0027Missing required \u0027 +"},{"line_number":225,"context_line":"                                                  \u0027key\u0027)(env, start_response)"},{"line_number":226,"context_line":""},{"line_number":227,"context_line":"                for key, val in req.headers.iteritems():"},{"line_number":228,"context_line":"                    if key.startswith(\u0027X-Container-Migration-\u0027):"},{"line_number":229,"context_line":"                        req.headers[key.replace(\u0027X-Container-\u0027,"},{"line_number":230,"context_line":"                                                \u0027X-Container-Sysmeta-\u0027)] \u003d val"}],"source_content_type":"text/x-python","patch_set":8,"id":"AAAAUX%2F%2FPZo%3D","line":227,"updated":"2014-02-14 17:11:03.000000000","message":"does it make sense to persist the metadata if prov is None? Isn\u0027t that maybe a BadRequest?\n\nAlso, see comment below - there should be a check somewhere for the generic mandatory headers x-container-migration-[active,source] - and return BadRequest if any of those is missing.","commit_id":"31b4aaeaebcc8bcec9eed67a0bb65352f55f9467"},{"author":{"_account_id":9205,"name":"Gil Vernik","email":"gilv@il.ibm.com","username":"gilv"},"change_message_id":"88fccb4496a53a0c1a4f17a8a26af3c191f9a7f5","unresolved":false,"context_lines":[{"line_number":224,"context_line":"                            return HTTPBadRequest(body\u003d\u0027Missing required \u0027 +"},{"line_number":225,"context_line":"                                                  \u0027key\u0027)(env, start_response)"},{"line_number":226,"context_line":""},{"line_number":227,"context_line":"                for key, val in req.headers.iteritems():"},{"line_number":228,"context_line":"                    if key.startswith(\u0027X-Container-Migration-\u0027):"},{"line_number":229,"context_line":"                        req.headers[key.replace(\u0027X-Container-\u0027,"},{"line_number":230,"context_line":"                                                \u0027X-Container-Sysmeta-\u0027)] \u003d val"}],"source_content_type":"text/x-python","patch_set":8,"id":"AAAAU3%2F%2F%2BXY%3D","line":227,"in_reply_to":"AAAAUX%2F%2BkHs%3D","updated":"2014-02-23 07:54:35.000000000","message":"Yes, I got convinced that you are correct. I will change the flow here, so if migration-enable \u003d  true, i will check the all required headers. I think it will cover 99.99% of cases actually, since people will most likely to setup migration in one request and may only need to disable  / enable it in later separate requests.","commit_id":"31b4aaeaebcc8bcec9eed67a0bb65352f55f9467"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a8818ee2ffc8b596310be5ba1926661c0f435205","unresolved":false,"context_lines":[{"line_number":224,"context_line":"                            return HTTPBadRequest(body\u003d\u0027Missing required \u0027 +"},{"line_number":225,"context_line":"                                                  \u0027key\u0027)(env, start_response)"},{"line_number":226,"context_line":""},{"line_number":227,"context_line":"                for key, val in req.headers.iteritems():"},{"line_number":228,"context_line":"                    if key.startswith(\u0027X-Container-Migration-\u0027):"},{"line_number":229,"context_line":"                        req.headers[key.replace(\u0027X-Container-\u0027,"},{"line_number":230,"context_line":"                                                \u0027X-Container-Sysmeta-\u0027)] \u003d val"}],"source_content_type":"text/x-python","patch_set":8,"id":"AAAAUX%2F%2BkHs%3D","line":227,"in_reply_to":"AAAAUX%2F%2FJLk%3D","updated":"2014-02-20 15:02:48.000000000","message":"I got confused by the doc describing some headers as mandatory and others as optional, whereas the code returns BadRequest if the optional headers are missing but does not insist on the mandatory headers. I understand your intent though. I don\u0027t know, perhaps better words to use in the doc would be \u0027generic\u0027 and \u0027driver-specific\u0027?\n\nI can understand migration-enable being independent of the other headers. but as it is, I am required to send  access-key, secret-key and region in the same request as provider, but I can send migration-source separately, which seems inconsistent. Would it be more consistent to either require all apart from migration-active to arrive together OR allow all to be set independently?\n\nThe bottom line is that all required headers must be set when migration-active \u003d enable - if that condition is not met then I guess an HTTPPreconditionFailed might be appropriate?","commit_id":"31b4aaeaebcc8bcec9eed67a0bb65352f55f9467"},{"author":{"_account_id":9205,"name":"Gil Vernik","email":"gilv@il.ibm.com","username":"gilv"},"change_message_id":"0fb520e153242eccdfa7133da32b7b0c542a5a7b","unresolved":false,"context_lines":[{"line_number":224,"context_line":"                            return HTTPBadRequest(body\u003d\u0027Missing required \u0027 +"},{"line_number":225,"context_line":"                                                  \u0027key\u0027)(env, start_response)"},{"line_number":226,"context_line":""},{"line_number":227,"context_line":"                for key, val in req.headers.iteritems():"},{"line_number":228,"context_line":"                    if key.startswith(\u0027X-Container-Migration-\u0027):"},{"line_number":229,"context_line":"                        req.headers[key.replace(\u0027X-Container-\u0027,"},{"line_number":230,"context_line":"                                                \u0027X-Container-Sysmeta-\u0027)] \u003d val"}],"source_content_type":"text/x-python","patch_set":8,"id":"AAAAUX%2F%2FJLk%3D","line":227,"in_reply_to":"AAAAUX%2F%2FPZo%3D","updated":"2014-02-16 08:59:15.000000000","message":"My code doesn\u0027t require customer to send all his metadata at once and customer allowed to setup migration not in one the same request. So a customer can send request with specifying provider + required keys, and another request with activation or source, etc.  Customer may also disable / enable activation via migration-active only.\n\nAbout sanity check for migration-active. I had it once, but then there was a remark by John to use \"config_true_value\" function. This function will return true in migration-active has any of \"true\" values list - otherwise false.\n\nI don\u0027t know what to check about migration-source. It may arrive in one request and another request will contain only migration-active, in this case i need to get system metadata and verify if migration-source exists there. Would you prefer this?","commit_id":"31b4aaeaebcc8bcec9eed67a0bb65352f55f9467"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"07d88a7d47e7b9cf9097bfb0d8fca63c75e0bbd6","unresolved":false,"context_lines":[{"line_number":248,"context_line":"                                                       container_md,"},{"line_number":249,"context_line":"                                                       start_response)"},{"line_number":250,"context_line":"                        except Exception as e:"},{"line_number":251,"context_line":"                            self.logger.error(e)"},{"line_number":252,"context_line":"        return original_resp"},{"line_number":253,"context_line":""},{"line_number":254,"context_line":"    def GETorHEAD_miss(self, original_env, account, container,"}],"source_content_type":"text/x-python","patch_set":8,"id":"AAAAUX%2F%2FPP4%3D","line":251,"updated":"2014-02-14 17:11:03.000000000","message":"If exception is raised during migration then I *think* that should result in a 5xx response code to the client - it is equivalent to backend requests failing in a normal GET. As it is it looks like this code will fall through to returning original_resp (404) which I *think* would be incorrect (the object may well exist, just failed to migrate on this attempt).","commit_id":"31b4aaeaebcc8bcec9eed67a0bb65352f55f9467"},{"author":{"_account_id":9205,"name":"Gil Vernik","email":"gilv@il.ibm.com","username":"gilv"},"change_message_id":"88fccb4496a53a0c1a4f17a8a26af3c191f9a7f5","unresolved":false,"context_lines":[{"line_number":248,"context_line":"                                                       container_md,"},{"line_number":249,"context_line":"                                                       start_response)"},{"line_number":250,"context_line":"                        except Exception as e:"},{"line_number":251,"context_line":"                            self.logger.error(e)"},{"line_number":252,"context_line":"        return original_resp"},{"line_number":253,"context_line":""},{"line_number":254,"context_line":"    def GETorHEAD_miss(self, original_env, account, container,"}],"source_content_type":"text/x-python","patch_set":8,"id":"AAAAU3%2F%2F%2BTQ%3D","line":251,"in_reply_to":"AAAAUX%2F%2BjoI%3D","updated":"2014-02-23 07:54:35.000000000","message":"I had one version of the code ( i decided not to commit it ) that verify credentials during setup phase and I can fail setup if credentials are wrong. \nI agree with your concerns, especially if credentials are wrong - 404 may be not a good response here.\nDo you think i should add this verification back the setup phase? \n\nMy major concern here that i didn\u0027t wanted to change any API of Swift, I think it\u0027s problematic for the community and users. If migration layer fails - return original Swift\u0027s response and report to log. Otherwise i can find myself, in translating migration related errors to HTTP responses to user ( which may be a hole without bottom, since there are dozens of possible errors here ) and it will modify Swift API which i really really wants to avoid. ( even the cost of migration failing consistently, for example old bucket was deleted).\nI think the end-user who setup migration on some container, need to have sort of responsibility to go and see logs why migration fails, e.g. he will get original 404... it clearly means to him that migration failed, since he is the one who original setup it.\n\nAnother point, consider some admin that setup Swift and decided to setup migration on some container. Now, there are dozens of users of that organization that access that container ( they might be not aware that this container has active migration ), so if they will start getting some non standard responses on GET ( like precondition failed ) - I think it will be not good and not clear to them what happened to Swift.\n\nWhat do you think?","commit_id":"31b4aaeaebcc8bcec9eed67a0bb65352f55f9467"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a8818ee2ffc8b596310be5ba1926661c0f435205","unresolved":false,"context_lines":[{"line_number":248,"context_line":"                                                       container_md,"},{"line_number":249,"context_line":"                                                       start_response)"},{"line_number":250,"context_line":"                        except Exception as e:"},{"line_number":251,"context_line":"                            self.logger.error(e)"},{"line_number":252,"context_line":"        return original_resp"},{"line_number":253,"context_line":""},{"line_number":254,"context_line":"    def GETorHEAD_miss(self, original_env, account, container,"}],"source_content_type":"text/x-python","patch_set":8,"id":"AAAAUX%2F%2BjoI%3D","line":251,"in_reply_to":"AAAAUX%2F%2FJSc%3D","updated":"2014-02-20 15:02:48.000000000","message":"My concern is that any kind of error that might occur during migration fetch from S3/Swift etc (e.g. auth failure, timeouts, service unavailable) is currently represented to the user as a 404\n\ne.g. if user has provided wrong credentials then (I think) GETorHEAD_miss will raise an exception due to swiftclient get_auth(), and I am thinking that perhaps this should be indicated to the client as HTTPUnauthorized rather than 404. Otherwise, when does the user get a hint that they will never be able to GET any objects from this container because their migration credentials are wrong? \n\nAnother example: GETorHEAD_miss fails to resolve the access provider (the config changed after the user set container migration provider) - at the moment this will cause a 404 response, but maybe PreconditionFailed would be  appropriate?","commit_id":"31b4aaeaebcc8bcec9eed67a0bb65352f55f9467"},{"author":{"_account_id":9205,"name":"Gil Vernik","email":"gilv@il.ibm.com","username":"gilv"},"change_message_id":"0fb520e153242eccdfa7133da32b7b0c542a5a7b","unresolved":false,"context_lines":[{"line_number":248,"context_line":"                                                       container_md,"},{"line_number":249,"context_line":"                                                       start_response)"},{"line_number":250,"context_line":"                        except Exception as e:"},{"line_number":251,"context_line":"                            self.logger.error(e)"},{"line_number":252,"context_line":"        return original_resp"},{"line_number":253,"context_line":""},{"line_number":254,"context_line":"    def GETorHEAD_miss(self, original_env, account, container,"}],"source_content_type":"text/x-python","patch_set":8,"id":"AAAAUX%2F%2FJSc%3D","line":251,"in_reply_to":"AAAAUX%2F%2FPP4%3D","updated":"2014-02-16 08:59:15.000000000","message":"I run this code many times, migrating from various sources. If any error exists during migration - it never failed to return original response 404 ( line 252 ). If you can identify a case where it will fail to return original response in case of some error - it would be great to know it and certainly need to be fixed.\n\nAbout 5xx - it\u0027s very crucial not to disrupt existing Swift flow / API / whatever. There might be various reasons for failures during migration, 99% of them are temporal. I  don\u0027t want migration layer to change something in Swift responses - if migration is failed  - return original Swift response. Error is printed to the log - so it can be easily read from there.","commit_id":"31b4aaeaebcc8bcec9eed67a0bb65352f55f9467"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"07d88a7d47e7b9cf9097bfb0d8fca63c75e0bbd6","unresolved":false,"context_lines":[{"line_number":255,"context_line":"                       obj, container_md, start_response):"},{"line_number":256,"context_line":"        \"\"\""},{"line_number":257,"context_line":"        Data migration logic for GET or HEAD requests. Identifies correct"},{"line_number":258,"context_line":"        access driver and then reads an object from another storage. Recieved"},{"line_number":259,"context_line":"        object will be written in local Swift and returned back to the user."},{"line_number":260,"context_line":""},{"line_number":261,"context_line":"        :params original_env: environ of the application"}],"source_content_type":"text/x-python","patch_set":8,"id":"AAAAUX%2F%2FPNk%3D","line":258,"updated":"2014-02-14 17:11:03.000000000","message":"s/Recieved/Received","commit_id":"31b4aaeaebcc8bcec9eed67a0bb65352f55f9467"},{"author":{"_account_id":9205,"name":"Gil Vernik","email":"gilv@il.ibm.com","username":"gilv"},"change_message_id":"0fb520e153242eccdfa7133da32b7b0c542a5a7b","unresolved":false,"context_lines":[{"line_number":255,"context_line":"                       obj, container_md, start_response):"},{"line_number":256,"context_line":"        \"\"\""},{"line_number":257,"context_line":"        Data migration logic for GET or HEAD requests. Identifies correct"},{"line_number":258,"context_line":"        access driver and then reads an object from another storage. Recieved"},{"line_number":259,"context_line":"        object will be written in local Swift and returned back to the user."},{"line_number":260,"context_line":""},{"line_number":261,"context_line":"        :params original_env: environ of the application"}],"source_content_type":"text/x-python","patch_set":8,"id":"AAAAUX%2F%2FJIE%3D","line":258,"in_reply_to":"AAAAUX%2F%2FPNk%3D","updated":"2014-02-16 08:59:15.000000000","message":"Done","commit_id":"31b4aaeaebcc8bcec9eed67a0bb65352f55f9467"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"07d88a7d47e7b9cf9097bfb0d8fca63c75e0bbd6","unresolved":false,"context_lines":[{"line_number":274,"context_line":"            content_type, timestamp \u003d access_resolver.get_object(obj)"},{"line_number":275,"context_line":"        if body_stream is None:"},{"line_number":276,"context_line":"            raise DataMigrationError(\u0027Failed to read an old object\u0027)"},{"line_number":277,"context_line":"        metadata[\u0027Migration-Import-Time\u0027] \u003d str(time.time())"},{"line_number":278,"context_line":"        metadata[\u0027Migration-Import-Owner\u0027] \u003d \u0027On-Demand\u0027"},{"line_number":279,"context_line":""},{"line_number":280,"context_line":"        status \u003d self.upload_object(original_env, body_stream, read_length,"}],"source_content_type":"text/x-python","patch_set":8,"id":"AAAAUX%2F%2FPKY%3D","line":277,"updated":"2014-02-14 17:11:03.000000000","message":"I would have the same complaint here as I did with using the x-container-meta namespace - ideally we would not be inserting \u0027system\u0027 metadata in the user\u0027s namespace. But since there is not yet support for object sysmeta I can\u0027t complain :)","commit_id":"31b4aaeaebcc8bcec9eed67a0bb65352f55f9467"},{"author":{"_account_id":9205,"name":"Gil Vernik","email":"gilv@il.ibm.com","username":"gilv"},"change_message_id":"0fb520e153242eccdfa7133da32b7b0c542a5a7b","unresolved":false,"context_lines":[{"line_number":274,"context_line":"            content_type, timestamp \u003d access_resolver.get_object(obj)"},{"line_number":275,"context_line":"        if body_stream is None:"},{"line_number":276,"context_line":"            raise DataMigrationError(\u0027Failed to read an old object\u0027)"},{"line_number":277,"context_line":"        metadata[\u0027Migration-Import-Time\u0027] \u003d str(time.time())"},{"line_number":278,"context_line":"        metadata[\u0027Migration-Import-Owner\u0027] \u003d \u0027On-Demand\u0027"},{"line_number":279,"context_line":""},{"line_number":280,"context_line":"        status \u003d self.upload_object(original_env, body_stream, read_length,"}],"source_content_type":"text/x-python","patch_set":8,"id":"AAAAUX%2F%2FJIA%3D","line":277,"in_reply_to":"AAAAUX%2F%2FPKY%3D","updated":"2014-02-16 08:59:15.000000000","message":"I expected this remark to come.. :)","commit_id":"31b4aaeaebcc8bcec9eed67a0bb65352f55f9467"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"07d88a7d47e7b9cf9097bfb0d8fca63c75e0bbd6","unresolved":false,"context_lines":[{"line_number":306,"context_line":"        new_env[\u0027CONTENT_TYPE\u0027] \u003d content_type"},{"line_number":307,"context_line":"        new_env[\u0027swift.source\u0027] \u003d \u0027DM\u0027"},{"line_number":308,"context_line":"        create_obj_req \u003d Request.blank(new_env[\u0027PATH_INFO\u0027], new_env)"},{"line_number":309,"context_line":"        create_obj_req.headers[\u0027X-Timestamp\u0027] \u003d timestamp"},{"line_number":310,"context_line":"        for key in metadata.keys():"},{"line_number":311,"context_line":"            create_obj_req.headers[\u0027X-Object-Meta-\u0027 + key] \u003d metadata[key]"},{"line_number":312,"context_line":""}],"source_content_type":"text/x-python","patch_set":8,"id":"AAAAUX%2F%2FO%2Fc%3D","line":309,"updated":"2014-02-14 17:11:03.000000000","message":"this timestamp should be sanity checked because it has come from outside of swift e.g. check that it is less than time.time()","commit_id":"31b4aaeaebcc8bcec9eed67a0bb65352f55f9467"},{"author":{"_account_id":9205,"name":"Gil Vernik","email":"gilv@il.ibm.com","username":"gilv"},"change_message_id":"88fccb4496a53a0c1a4f17a8a26af3c191f9a7f5","unresolved":false,"context_lines":[{"line_number":306,"context_line":"        new_env[\u0027CONTENT_TYPE\u0027] \u003d content_type"},{"line_number":307,"context_line":"        new_env[\u0027swift.source\u0027] \u003d \u0027DM\u0027"},{"line_number":308,"context_line":"        create_obj_req \u003d Request.blank(new_env[\u0027PATH_INFO\u0027], new_env)"},{"line_number":309,"context_line":"        create_obj_req.headers[\u0027X-Timestamp\u0027] \u003d timestamp"},{"line_number":310,"context_line":"        for key in metadata.keys():"},{"line_number":311,"context_line":"            create_obj_req.headers[\u0027X-Object-Meta-\u0027 + key] \u003d metadata[key]"},{"line_number":312,"context_line":""}],"source_content_type":"text/x-python","patch_set":8,"id":"AAAAU3%2F%2F%2BKc%3D","line":309,"in_reply_to":"AAAAUX%2F%2BjiQ%3D","updated":"2014-02-23 07:54:35.000000000","message":"I think your point is very good. I will add it to the code","commit_id":"31b4aaeaebcc8bcec9eed67a0bb65352f55f9467"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a8818ee2ffc8b596310be5ba1926661c0f435205","unresolved":false,"context_lines":[{"line_number":306,"context_line":"        new_env[\u0027CONTENT_TYPE\u0027] \u003d content_type"},{"line_number":307,"context_line":"        new_env[\u0027swift.source\u0027] \u003d \u0027DM\u0027"},{"line_number":308,"context_line":"        create_obj_req \u003d Request.blank(new_env[\u0027PATH_INFO\u0027], new_env)"},{"line_number":309,"context_line":"        create_obj_req.headers[\u0027X-Timestamp\u0027] \u003d timestamp"},{"line_number":310,"context_line":"        for key in metadata.keys():"},{"line_number":311,"context_line":"            create_obj_req.headers[\u0027X-Object-Meta-\u0027 + key] \u003d metadata[key]"},{"line_number":312,"context_line":""}],"source_content_type":"text/x-python","patch_set":8,"id":"AAAAUX%2F%2BjiQ%3D","line":309,"in_reply_to":"AAAAUX%2F%2FJRs%3D","updated":"2014-02-20 15:02:48.000000000","message":"The corner case I was thinking of was fetching in an object from another provider whose timestamps are ahead of local time. If that object is then PUT with the imported future timestamp then it will be hard to delete until future time is reached.\n\nHmmm, maybe that is just the user\u0027s fault. I\u0027m not sure now whether the timestamp should be min(time.time(), timestamp) or just left as it is.","commit_id":"31b4aaeaebcc8bcec9eed67a0bb65352f55f9467"},{"author":{"_account_id":9205,"name":"Gil Vernik","email":"gilv@il.ibm.com","username":"gilv"},"change_message_id":"0fb520e153242eccdfa7133da32b7b0c542a5a7b","unresolved":false,"context_lines":[{"line_number":306,"context_line":"        new_env[\u0027CONTENT_TYPE\u0027] \u003d content_type"},{"line_number":307,"context_line":"        new_env[\u0027swift.source\u0027] \u003d \u0027DM\u0027"},{"line_number":308,"context_line":"        create_obj_req \u003d Request.blank(new_env[\u0027PATH_INFO\u0027], new_env)"},{"line_number":309,"context_line":"        create_obj_req.headers[\u0027X-Timestamp\u0027] \u003d timestamp"},{"line_number":310,"context_line":"        for key in metadata.keys():"},{"line_number":311,"context_line":"            create_obj_req.headers[\u0027X-Object-Meta-\u0027 + key] \u003d metadata[key]"},{"line_number":312,"context_line":""}],"source_content_type":"text/x-python","patch_set":8,"id":"AAAAUX%2F%2FJRs%3D","line":309,"in_reply_to":"AAAAUX%2F%2FO%2Fc%3D","updated":"2014-02-16 08:59:15.000000000","message":"yes, good point. But I wonder what you think about the following: \n\nany driver need to provide response ( including timestamp ) that Swift can handle. It\u0027s already written like this ( you can see the code ). So I wonder what could happen? if timestamp is correct - all is good, if not - Swift will response with error, and then 404 will be returned to the customer. In any case my code has nothing to do with sanity check, if it fails - return original response - 404. \nI feel there is no bottom here, i can also verify names and so on.. all this came outside of Swift. \nWhat do you think about it? Should i still add sanity check here? Did i missed the point of the remark? \nThanks.","commit_id":"31b4aaeaebcc8bcec9eed67a0bb65352f55f9467"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"07d88a7d47e7b9cf9097bfb0d8fca63c75e0bbd6","unresolved":false,"context_lines":[{"line_number":343,"context_line":"    conf.update(local_conf)"},{"line_number":344,"context_line":"    migration_conf \u003d {}"},{"line_number":345,"context_line":"    swift_info \u003d {}"},{"line_number":346,"context_line":"    supported_drivers \u003d conf.get(\u0027supported_drivers\u0027, \u0027\u0027).split(\u0027,\u0027)"},{"line_number":347,"context_line":"    for dr in supported_drivers:"},{"line_number":348,"context_line":"        keys \u003d conf.get(\u0027driver_\u0027 + dr + \u0027_keys\u0027, \u0027\u0027).split(\u0027,\u0027)"},{"line_number":349,"context_line":"        module_name \u003d conf.get(\u0027driver_\u0027 + dr + \u0027_module\u0027, \u0027\u0027)"}],"source_content_type":"text/x-python","patch_set":8,"id":"AAAAUX%2F%2FPnc%3D","line":346,"updated":"2014-02-14 17:11:03.000000000","message":"other swift config tolerates whitespace in lists. I suggest something like:\n\nsupported_drivers \u003d [driver.strip()\n            for driver in conf.get(\u0027supported_drivers\u0027, \u0027\u0027).split(\u0027,\u0027)\n            if driver.strip()]","commit_id":"31b4aaeaebcc8bcec9eed67a0bb65352f55f9467"},{"author":{"_account_id":9205,"name":"Gil Vernik","email":"gilv@il.ibm.com","username":"gilv"},"change_message_id":"0fb520e153242eccdfa7133da32b7b0c542a5a7b","unresolved":false,"context_lines":[{"line_number":343,"context_line":"    conf.update(local_conf)"},{"line_number":344,"context_line":"    migration_conf \u003d {}"},{"line_number":345,"context_line":"    swift_info \u003d {}"},{"line_number":346,"context_line":"    supported_drivers \u003d conf.get(\u0027supported_drivers\u0027, \u0027\u0027).split(\u0027,\u0027)"},{"line_number":347,"context_line":"    for dr in supported_drivers:"},{"line_number":348,"context_line":"        keys \u003d conf.get(\u0027driver_\u0027 + dr + \u0027_keys\u0027, \u0027\u0027).split(\u0027,\u0027)"},{"line_number":349,"context_line":"        module_name \u003d conf.get(\u0027driver_\u0027 + dr + \u0027_module\u0027, \u0027\u0027)"}],"source_content_type":"text/x-python","patch_set":8,"id":"AAAAUX%2F%2FJFo%3D","line":346,"in_reply_to":"AAAAUX%2F%2FPnc%3D","updated":"2014-02-16 08:59:15.000000000","message":"Done","commit_id":"31b4aaeaebcc8bcec9eed67a0bb65352f55f9467"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"07d88a7d47e7b9cf9097bfb0d8fca63c75e0bbd6","unresolved":false,"context_lines":[{"line_number":345,"context_line":"    swift_info \u003d {}"},{"line_number":346,"context_line":"    supported_drivers \u003d conf.get(\u0027supported_drivers\u0027, \u0027\u0027).split(\u0027,\u0027)"},{"line_number":347,"context_line":"    for dr in supported_drivers:"},{"line_number":348,"context_line":"        keys \u003d conf.get(\u0027driver_\u0027 + dr + \u0027_keys\u0027, \u0027\u0027).split(\u0027,\u0027)"},{"line_number":349,"context_line":"        module_name \u003d conf.get(\u0027driver_\u0027 + dr + \u0027_module\u0027, \u0027\u0027)"},{"line_number":350,"context_line":"        mo \u003d module_name[:module_name.rfind(\u0027:\u0027)]"},{"line_number":351,"context_line":"        cl \u003d module_name[module_name.rfind(\u0027:\u0027) + 1:]"}],"source_content_type":"text/x-python","patch_set":8,"id":"AAAAUX%2F%2FPmg%3D","line":348,"updated":"2014-02-14 17:11:03.000000000","message":"ditto, make tolerant of white space in keys list\n\nAlso, perhaps the generic mandatory migration keys should be added to the list of driver specific keys so that their absence will be detected in __call__() method.","commit_id":"31b4aaeaebcc8bcec9eed67a0bb65352f55f9467"},{"author":{"_account_id":9205,"name":"Gil Vernik","email":"gilv@il.ibm.com","username":"gilv"},"change_message_id":"0fb520e153242eccdfa7133da32b7b0c542a5a7b","unresolved":false,"context_lines":[{"line_number":345,"context_line":"    swift_info \u003d {}"},{"line_number":346,"context_line":"    supported_drivers \u003d conf.get(\u0027supported_drivers\u0027, \u0027\u0027).split(\u0027,\u0027)"},{"line_number":347,"context_line":"    for dr in supported_drivers:"},{"line_number":348,"context_line":"        keys \u003d conf.get(\u0027driver_\u0027 + dr + \u0027_keys\u0027, \u0027\u0027).split(\u0027,\u0027)"},{"line_number":349,"context_line":"        module_name \u003d conf.get(\u0027driver_\u0027 + dr + \u0027_module\u0027, \u0027\u0027)"},{"line_number":350,"context_line":"        mo \u003d module_name[:module_name.rfind(\u0027:\u0027)]"},{"line_number":351,"context_line":"        cl \u003d module_name[module_name.rfind(\u0027:\u0027) + 1:]"}],"source_content_type":"text/x-python","patch_set":8,"id":"AAAAUX%2F%2FJHA%3D","line":348,"in_reply_to":"AAAAUX%2F%2FPmg%3D","updated":"2014-02-16 08:59:15.000000000","message":"About adding generic meatadata keys to the list - please see answer to your remark *line 227\".","commit_id":"31b4aaeaebcc8bcec9eed67a0bb65352f55f9467"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"07d88a7d47e7b9cf9097bfb0d8fca63c75e0bbd6","unresolved":false,"context_lines":[{"line_number":346,"context_line":"    supported_drivers \u003d conf.get(\u0027supported_drivers\u0027, \u0027\u0027).split(\u0027,\u0027)"},{"line_number":347,"context_line":"    for dr in supported_drivers:"},{"line_number":348,"context_line":"        keys \u003d conf.get(\u0027driver_\u0027 + dr + \u0027_keys\u0027, \u0027\u0027).split(\u0027,\u0027)"},{"line_number":349,"context_line":"        module_name \u003d conf.get(\u0027driver_\u0027 + dr + \u0027_module\u0027, \u0027\u0027)"},{"line_number":350,"context_line":"        mo \u003d module_name[:module_name.rfind(\u0027:\u0027)]"},{"line_number":351,"context_line":"        cl \u003d module_name[module_name.rfind(\u0027:\u0027) + 1:]"},{"line_number":352,"context_line":"        module \u003d __import__(mo, fromlist\u003d[cl])"}],"source_content_type":"text/x-python","patch_set":8,"id":"AAAAUX%2F%2FPh8%3D","line":349,"updated":"2014-02-14 17:11:03.000000000","message":"You might want to enclose the module loading code in a try/except so that a misconfiguration or single missing module does not cause *all* driver loading to fail and filter_factory to raise an exception. Drivers that fail to load could be noted as such in the swift_info.","commit_id":"31b4aaeaebcc8bcec9eed67a0bb65352f55f9467"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a8818ee2ffc8b596310be5ba1926661c0f435205","unresolved":false,"context_lines":[{"line_number":346,"context_line":"    supported_drivers \u003d conf.get(\u0027supported_drivers\u0027, \u0027\u0027).split(\u0027,\u0027)"},{"line_number":347,"context_line":"    for dr in supported_drivers:"},{"line_number":348,"context_line":"        keys \u003d conf.get(\u0027driver_\u0027 + dr + \u0027_keys\u0027, \u0027\u0027).split(\u0027,\u0027)"},{"line_number":349,"context_line":"        module_name \u003d conf.get(\u0027driver_\u0027 + dr + \u0027_module\u0027, \u0027\u0027)"},{"line_number":350,"context_line":"        mo \u003d module_name[:module_name.rfind(\u0027:\u0027)]"},{"line_number":351,"context_line":"        cl \u003d module_name[module_name.rfind(\u0027:\u0027) + 1:]"},{"line_number":352,"context_line":"        module \u003d __import__(mo, fromlist\u003d[cl])"}],"source_content_type":"text/x-python","patch_set":8,"id":"AAAAUX%2F%2Bj0o%3D","line":349,"in_reply_to":"AAAAUX%2F%2FJM0%3D","updated":"2014-02-20 15:02:48.000000000","message":"ok that is reasonable.","commit_id":"31b4aaeaebcc8bcec9eed67a0bb65352f55f9467"},{"author":{"_account_id":9205,"name":"Gil Vernik","email":"gilv@il.ibm.com","username":"gilv"},"change_message_id":"0fb520e153242eccdfa7133da32b7b0c542a5a7b","unresolved":false,"context_lines":[{"line_number":346,"context_line":"    supported_drivers \u003d conf.get(\u0027supported_drivers\u0027, \u0027\u0027).split(\u0027,\u0027)"},{"line_number":347,"context_line":"    for dr in supported_drivers:"},{"line_number":348,"context_line":"        keys \u003d conf.get(\u0027driver_\u0027 + dr + \u0027_keys\u0027, \u0027\u0027).split(\u0027,\u0027)"},{"line_number":349,"context_line":"        module_name \u003d conf.get(\u0027driver_\u0027 + dr + \u0027_module\u0027, \u0027\u0027)"},{"line_number":350,"context_line":"        mo \u003d module_name[:module_name.rfind(\u0027:\u0027)]"},{"line_number":351,"context_line":"        cl \u003d module_name[module_name.rfind(\u0027:\u0027) + 1:]"},{"line_number":352,"context_line":"        module \u003d __import__(mo, fromlist\u003d[cl])"}],"source_content_type":"text/x-python","patch_set":8,"id":"AAAAUX%2F%2FJM0%3D","line":349,"in_reply_to":"AAAAUX%2F%2FPh8%3D","updated":"2014-02-16 08:59:15.000000000","message":"Well...originally i wrote this code exactly as you suggested. Then i had many debate over this subject and got convinced to change it. The point here is like this :\nCustomer / admin / .. configured data migration to use certain driver for the customer\u0027s specific needs. It may be one driver, or dozens  - depends where customer has his current data. If any driver failed to load - \"go and scream\" and fail all. This will force ( say admin ) to check what is wrong. \nIt\u0027s bad to allow  continue if load driver failed and then migration will fail in run time and the way to know what is wrong - to see swift_info.\nIt\u0027s very crucial to have drivers exists prior anything starts and in case of failure it should stop everything.\nIt\u0027s like Swift that fails to load certain middleware, it halts on restart and not in run time.\nIn addition  there might be dozens of cases when driver fails to load, like small typo in the driver name. If it will allow to continue, even if driver failed  then all looks  good, but data migration will consistently fail and customer will say \"bad, bad data migration\" without knowing the real cause...","commit_id":"31b4aaeaebcc8bcec9eed67a0bb65352f55f9467"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"ebc43cc9e5b5c1aa102c8e70c43858a9871b0930","unresolved":false,"context_lines":[{"line_number":207,"context_line":"            elif req.method in [\u0027PUT\u0027, \u0027POST\u0027]:"},{"line_number":208,"context_line":"                prov \u003d req.headers.get(\u0027X-Container-Migration-Provider\u0027)"},{"line_number":209,"context_line":"                mig_source \u003d req.headers.get(\u0027X-Container-Migration-Source\u0027)"},{"line_number":210,"context_line":"                if ((not mig_source) and (prov is not None)):"},{"line_number":211,"context_line":"                    return HTTPPreconditionFailed(\u0027Migration source is \u0027 +"},{"line_number":212,"context_line":"                                                  \u0027missing\u0027)(env,"},{"line_number":213,"context_line":"                                                             start_response)"}],"source_content_type":"text/x-python","patch_set":12,"id":"AAAAVn%2F%2FMSc%3D","line":210,"updated":"2014-03-07 19:09:45.000000000","message":"I couldn\u0027t figure out how I could delete my migration headers, if I set all the migration headers to \u0027\u0027 then I get a 400 response. I think the \u0027xxx is not None\u0027 conditions need to be simply \u0027xxx\u0027 so that the condition evaluates as false when xxx is an empty string.","commit_id":"219e29e5c9dac4e12faf4c15d204b5da4a5d1b09"},{"author":{"_account_id":9205,"name":"Gil Vernik","email":"gilv@il.ibm.com","username":"gilv"},"change_message_id":"14e3bdc725365f47838c070e48a1aa94f7d73270","unresolved":false,"context_lines":[{"line_number":207,"context_line":"            elif req.method in [\u0027PUT\u0027, \u0027POST\u0027]:"},{"line_number":208,"context_line":"                prov \u003d req.headers.get(\u0027X-Container-Migration-Provider\u0027)"},{"line_number":209,"context_line":"                mig_source \u003d req.headers.get(\u0027X-Container-Migration-Source\u0027)"},{"line_number":210,"context_line":"                if ((not mig_source) and (prov is not None)):"},{"line_number":211,"context_line":"                    return HTTPPreconditionFailed(\u0027Migration source is \u0027 +"},{"line_number":212,"context_line":"                                                  \u0027missing\u0027)(env,"},{"line_number":213,"context_line":"                                                             start_response)"}],"source_content_type":"text/x-python","patch_set":12,"id":"AAAAWH%2F%2BXTo%3D","line":210,"in_reply_to":"AAAAVn%2F%2FMSc%3D","updated":"2014-03-27 11:24:40.000000000","message":"It\u0027s not allowed to delete migration related headers, but only to enable / disable it. Those headers should be kept for various monitoring issues.","commit_id":"219e29e5c9dac4e12faf4c15d204b5da4a5d1b09"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"ebc43cc9e5b5c1aa102c8e70c43858a9871b0930","unresolved":false,"context_lines":[{"line_number":211,"context_line":"                    return HTTPPreconditionFailed(\u0027Migration source is \u0027 +"},{"line_number":212,"context_line":"                                                  \u0027missing\u0027)(env,"},{"line_number":213,"context_line":"                                                             start_response)"},{"line_number":214,"context_line":"                elif ((not prov) and (mig_source is not None)):"},{"line_number":215,"context_line":"                    return HTTPPreconditionFailed(\u0027Migration provider is \u0027 +"},{"line_number":216,"context_line":"                                                  \u0027missing\u0027)(env,"},{"line_number":217,"context_line":"                                                             start_response)"}],"source_content_type":"text/x-python","patch_set":12,"id":"AAAAVn%2F%2FMSM%3D","line":214,"updated":"2014-03-07 19:09:45.000000000","message":"ditto","commit_id":"219e29e5c9dac4e12faf4c15d204b5da4a5d1b09"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"ebc43cc9e5b5c1aa102c8e70c43858a9871b0930","unresolved":false,"context_lines":[{"line_number":215,"context_line":"                    return HTTPPreconditionFailed(\u0027Migration provider is \u0027 +"},{"line_number":216,"context_line":"                                                  \u0027missing\u0027)(env,"},{"line_number":217,"context_line":"                                                             start_response)"},{"line_number":218,"context_line":"                elif (prov is not None) and (mig_source is not None):"},{"line_number":219,"context_line":"                    if prov.startswith(\u0027libcloud\u0027) and not prov.endswith(\u0027s3\u0027):"},{"line_number":220,"context_line":"                            return HTTPServiceUnavailable(body\u003d\u0027Only S3 \u0027 +"},{"line_number":221,"context_line":"                                                          \u0027driver of Lib\u0027 +"}],"source_content_type":"text/x-python","patch_set":12,"id":"AAAAVn%2F%2FMRw%3D","line":218,"updated":"2014-03-07 19:09:45.000000000","message":"ditto","commit_id":"219e29e5c9dac4e12faf4c15d204b5da4a5d1b09"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"322cdf9d121df319ad381a11fe3244dc20c14a52","unresolved":false,"context_lines":[{"line_number":291,"context_line":"        if body_stream is None:"},{"line_number":292,"context_line":"            raise DataMigrationError(\u0027Failed to read an old object\u0027)"},{"line_number":293,"context_line":"        metadata[\u0027Migration-Import-Time\u0027] \u003d str(time.time())"},{"line_number":294,"context_line":"        metadata[\u0027Migration-Import-Owner\u0027] \u003d \u0027On-Demand\u0027"},{"line_number":295,"context_line":""},{"line_number":296,"context_line":"        status \u003d self.upload_object(original_env, body_stream, read_length,"},{"line_number":297,"context_line":"                                    metadata, content_type, timestamp)"}],"source_content_type":"text/x-python","patch_set":13,"id":"baada198_918e0e26","line":294,"updated":"2014-07-25 15:16:20.000000000","message":"These metadata items will be ephemeral - they will be deleted is a user ever makes a POST to the object. Ideally they would be stored as object sysmeta when that becomes enabled.","commit_id":"38bf06b8de46421318c3d119faa56b03689d0d70"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"8b50f96d9660529af80365afa348886b73132423","unresolved":false,"context_lines":[{"line_number":37,"context_line":"Our model comes with variety of storage drivers that"},{"line_number":38,"context_line":"may be used to access remote storage to migrate data."},{"line_number":39,"context_line":"1. (Default) Migration from file system"},{"line_number":40,"context_line":"2. (Optional) Migration from another Swift, based on Swift Intenal Client."},{"line_number":41,"context_line":"3. (Optional) User defined custom driver."},{"line_number":42,"context_line":""},{"line_number":43,"context_line":"Data migration middleware run on Proxy with the configuration as follows:"}],"source_content_type":"text/x-python","patch_set":14,"id":"baada198_8d37204b","line":40,"updated":"2014-08-06 15:22:47.000000000","message":"nit: s/Intenal/Internal","commit_id":"cd8a0d5ed1e0685111396beaf3d693e136bc5997"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"363fa98d6c1b66cf6a6cc4ba77c57e4db93c9b62","unresolved":false,"context_lines":[{"line_number":37,"context_line":"Our model comes with variety of storage drivers that"},{"line_number":38,"context_line":"may be used to access remote storage to migrate data."},{"line_number":39,"context_line":"1. (Default) Migration from file system"},{"line_number":40,"context_line":"2. (Optional) Migration from another Swift, based on Swift Internal Client."},{"line_number":41,"context_line":"3. (Optional) User defined custom driver."},{"line_number":42,"context_line":""},{"line_number":43,"context_line":"Data migration middleware run on Proxy with the configuration as follows:"}],"source_content_type":"text/x-python","patch_set":16,"id":"baada198_274d9693","line":40,"updated":"2014-08-13 15:09:11.000000000","message":"nit: why is this labelled (Optional)? It is enabled in proxy-server.conf-sample","commit_id":"1e5ddde5fc3bc514d59e66a5c43592cbb502143c"},{"author":{"_account_id":9205,"name":"Gil Vernik","email":"gilv@il.ibm.com","username":"gilv"},"change_message_id":"8f1bbdcf5a4962e0cdf319f9941ca972103cf051","unresolved":false,"context_lines":[{"line_number":37,"context_line":"Our model comes with variety of storage drivers that"},{"line_number":38,"context_line":"may be used to access remote storage to migrate data."},{"line_number":39,"context_line":"1. (Default) Migration from file system"},{"line_number":40,"context_line":"2. (Optional) Migration from another Swift, based on Swift Internal Client."},{"line_number":41,"context_line":"3. (Optional) User defined custom driver."},{"line_number":42,"context_line":""},{"line_number":43,"context_line":"Data migration middleware run on Proxy with the configuration as follows:"}],"source_content_type":"text/x-python","patch_set":16,"id":"baada198_906e53fd","line":40,"in_reply_to":"baada198_274d9693","updated":"2014-08-24 08:10:02.000000000","message":"Done","commit_id":"1e5ddde5fc3bc514d59e66a5c43592cbb502143c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"363fa98d6c1b66cf6a6cc4ba77c57e4db93c9b62","unresolved":false,"context_lines":[{"line_number":57,"context_line":"# will be loaded dynamically during middleware initialization )"},{"line_number":58,"context_line":""},{"line_number":59,"context_line":"# File System driver"},{"line_number":60,"context_line":"driver_fsystem_keys \u003d path"},{"line_number":61,"context_line":"driver_fsystem_module \u003d"},{"line_number":62,"context_line":"swift.common.data_migrator_drivers:FileSystemAccessDriver"},{"line_number":63,"context_line":""}],"source_content_type":"text/x-python","patch_set":16,"id":"baada198_86c46843","line":60,"updated":"2014-08-13 15:09:11.000000000","message":"q: why do we need to specify path in addition to source in fsystem case, i can achieve same by just extending the path in source? (and it would save me typing one more header :)","commit_id":"1e5ddde5fc3bc514d59e66a5c43592cbb502143c"},{"author":{"_account_id":9205,"name":"Gil Vernik","email":"gilv@il.ibm.com","username":"gilv"},"change_message_id":"8f1bbdcf5a4962e0cdf319f9941ca972103cf051","unresolved":false,"context_lines":[{"line_number":57,"context_line":"# will be loaded dynamically during middleware initialization )"},{"line_number":58,"context_line":""},{"line_number":59,"context_line":"# File System driver"},{"line_number":60,"context_line":"driver_fsystem_keys \u003d path"},{"line_number":61,"context_line":"driver_fsystem_module \u003d"},{"line_number":62,"context_line":"swift.common.data_migrator_drivers:FileSystemAccessDriver"},{"line_number":63,"context_line":""}],"source_content_type":"text/x-python","patch_set":16,"id":"7ac7a964_472b51eb","line":60,"in_reply_to":"baada198_86c46843","updated":"2014-08-24 08:10:02.000000000","message":"good point. no need for this one for the filesystem driver.\nDone.","commit_id":"1e5ddde5fc3bc514d59e66a5c43592cbb502143c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"363fa98d6c1b66cf6a6cc4ba77c57e4db93c9b62","unresolved":false,"context_lines":[{"line_number":61,"context_line":"driver_fsystem_module \u003d"},{"line_number":62,"context_line":"swift.common.data_migrator_drivers:FileSystemAccessDriver"},{"line_number":63,"context_line":""},{"line_number":64,"context_line":"# Swift driver based on Swift Internal Client (Optional)"},{"line_number":65,"context_line":"driver_swift_keys \u003d auth-type,token-url,user,key"},{"line_number":66,"context_line":"driver_swift_module \u003d"},{"line_number":67,"context_line":"swift.common.data_migrator_drivers:SwiftAccessDriver"}],"source_content_type":"text/x-python","patch_set":16,"id":"baada198_415efa79","line":64,"updated":"2014-08-13 15:09:11.000000000","message":"nit: you\u0027re using swiftclient - \u0027swift internal client\u0027  is as separate thing: swift/common/internal_client.py","commit_id":"1e5ddde5fc3bc514d59e66a5c43592cbb502143c"},{"author":{"_account_id":9205,"name":"Gil Vernik","email":"gilv@il.ibm.com","username":"gilv"},"change_message_id":"8f1bbdcf5a4962e0cdf319f9941ca972103cf051","unresolved":false,"context_lines":[{"line_number":61,"context_line":"driver_fsystem_module \u003d"},{"line_number":62,"context_line":"swift.common.data_migrator_drivers:FileSystemAccessDriver"},{"line_number":63,"context_line":""},{"line_number":64,"context_line":"# Swift driver based on Swift Internal Client (Optional)"},{"line_number":65,"context_line":"driver_swift_keys \u003d auth-type,token-url,user,key"},{"line_number":66,"context_line":"driver_swift_module \u003d"},{"line_number":67,"context_line":"swift.common.data_migrator_drivers:SwiftAccessDriver"}],"source_content_type":"text/x-python","patch_set":16,"id":"baada198_d5bb60bc","line":64,"in_reply_to":"baada198_415efa79","updated":"2014-08-24 08:10:02.000000000","message":"Done","commit_id":"1e5ddde5fc3bc514d59e66a5c43592cbb502143c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"363fa98d6c1b66cf6a6cc4ba77c57e4db93c9b62","unresolved":false,"context_lines":[{"line_number":132,"context_line":"        if self._response_headers is None:"},{"line_number":133,"context_line":"            self._response_headers \u003d []"},{"line_number":134,"context_line":""},{"line_number":135,"context_line":"        if self.container_md is not None:"},{"line_number":136,"context_line":"            for key, val in self.container_md.iteritems():"},{"line_number":137,"context_line":"                if key.startswith(\u0027migration\u0027):"},{"line_number":138,"context_line":"                    self._response_headers.append((\u0027X-Container-\u0027 +"}],"source_content_type":"text/x-python","patch_set":16,"id":"baada198_eed26732","line":135,"updated":"2014-08-13 15:09:11.000000000","message":"just an idea - you could obscure the x-container-migration-key value when copying to outbound headers so user sees  x-container-migration-key: ********","commit_id":"1e5ddde5fc3bc514d59e66a5c43592cbb502143c"},{"author":{"_account_id":9205,"name":"Gil Vernik","email":"gilv@il.ibm.com","username":"gilv"},"change_message_id":"8f1bbdcf5a4962e0cdf319f9941ca972103cf051","unresolved":false,"context_lines":[{"line_number":132,"context_line":"        if self._response_headers is None:"},{"line_number":133,"context_line":"            self._response_headers \u003d []"},{"line_number":134,"context_line":""},{"line_number":135,"context_line":"        if self.container_md is not None:"},{"line_number":136,"context_line":"            for key, val in self.container_md.iteritems():"},{"line_number":137,"context_line":"                if key.startswith(\u0027migration\u0027):"},{"line_number":138,"context_line":"                    self._response_headers.append((\u0027X-Container-\u0027 +"}],"source_content_type":"text/x-python","patch_set":16,"id":"1abeadc6_62c33a57","line":135,"in_reply_to":"baada198_eed26732","updated":"2014-08-24 08:10:02.000000000","message":"I decided to remove it completely from response. It\u0027s not wise to return those v keys / values at all.","commit_id":"1e5ddde5fc3bc514d59e66a5c43592cbb502143c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"363fa98d6c1b66cf6a6cc4ba77c57e4db93c9b62","unresolved":false,"context_lines":[{"line_number":172,"context_line":"            if req.method \u003d\u003d \u0027HEAD\u0027:"},{"line_number":173,"context_line":"                container_info \u003d get_container_info(req.environ, self.app,"},{"line_number":174,"context_line":"                                                    swift_source\u003d\u0027DM\u0027)"},{"line_number":175,"context_line":"                container_md \u003d container_info.get(\u0027sysmeta\u0027, {})"},{"line_number":176,"context_line":"            elif req.method in [\u0027PUT\u0027, \u0027POST\u0027]:"},{"line_number":177,"context_line":"                prov \u003d req.headers.get(\u0027X-Container-Migration-Provider\u0027)"},{"line_number":178,"context_line":"                mig_source \u003d req.headers.get(\u0027X-Container-Migration-Source\u0027)"}],"source_content_type":"text/x-python","patch_set":16,"id":"baada198_a46b64b6","line":175,"updated":"2014-08-13 15:09:11.000000000","message":"not sure this is necessary - the HEAD response will contain the sysmeta headers and they can be copied in DataMigrationContext.handle_original_call","commit_id":"1e5ddde5fc3bc514d59e66a5c43592cbb502143c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"363fa98d6c1b66cf6a6cc4ba77c57e4db93c9b62","unresolved":false,"context_lines":[{"line_number":176,"context_line":"            elif req.method in [\u0027PUT\u0027, \u0027POST\u0027]:"},{"line_number":177,"context_line":"                prov \u003d req.headers.get(\u0027X-Container-Migration-Provider\u0027)"},{"line_number":178,"context_line":"                mig_source \u003d req.headers.get(\u0027X-Container-Migration-Source\u0027)"},{"line_number":179,"context_line":"                if ((not mig_source) and (prov is not None)):"},{"line_number":180,"context_line":"                    return HTTPPreconditionFailed(\u0027Migration source is \u0027 +"},{"line_number":181,"context_line":"                                                  \u0027missing\u0027)(env,"},{"line_number":182,"context_line":"                                                             start_response)"}],"source_content_type":"text/x-python","patch_set":16,"id":"baada198_c1868ab9","line":179,"updated":"2014-08-13 15:09:11.000000000","message":"nit: outer () are redundant","commit_id":"1e5ddde5fc3bc514d59e66a5c43592cbb502143c"},{"author":{"_account_id":9205,"name":"Gil Vernik","email":"gilv@il.ibm.com","username":"gilv"},"change_message_id":"8f1bbdcf5a4962e0cdf319f9941ca972103cf051","unresolved":false,"context_lines":[{"line_number":176,"context_line":"            elif req.method in [\u0027PUT\u0027, \u0027POST\u0027]:"},{"line_number":177,"context_line":"                prov \u003d req.headers.get(\u0027X-Container-Migration-Provider\u0027)"},{"line_number":178,"context_line":"                mig_source \u003d req.headers.get(\u0027X-Container-Migration-Source\u0027)"},{"line_number":179,"context_line":"                if ((not mig_source) and (prov is not None)):"},{"line_number":180,"context_line":"                    return HTTPPreconditionFailed(\u0027Migration source is \u0027 +"},{"line_number":181,"context_line":"                                                  \u0027missing\u0027)(env,"},{"line_number":182,"context_line":"                                                             start_response)"}],"source_content_type":"text/x-python","patch_set":16,"id":"baada198_35557c52","line":179,"in_reply_to":"baada198_c1868ab9","updated":"2014-08-24 08:10:02.000000000","message":"Done","commit_id":"1e5ddde5fc3bc514d59e66a5c43592cbb502143c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"363fa98d6c1b66cf6a6cc4ba77c57e4db93c9b62","unresolved":false,"context_lines":[{"line_number":180,"context_line":"                    return HTTPPreconditionFailed(\u0027Migration source is \u0027 +"},{"line_number":181,"context_line":"                                                  \u0027missing\u0027)(env,"},{"line_number":182,"context_line":"                                                             start_response)"},{"line_number":183,"context_line":"                elif ((not prov) and (mig_source is not None)):"},{"line_number":184,"context_line":"                    return HTTPPreconditionFailed(\u0027Migration provider is \u0027 +"},{"line_number":185,"context_line":"                                                  \u0027missing\u0027)(env,"},{"line_number":186,"context_line":"                                                             start_response)"}],"source_content_type":"text/x-python","patch_set":16,"id":"baada198_61bdbe66","line":183,"updated":"2014-08-13 15:09:11.000000000","message":"nit: outer () are redundant","commit_id":"1e5ddde5fc3bc514d59e66a5c43592cbb502143c"},{"author":{"_account_id":9205,"name":"Gil Vernik","email":"gilv@il.ibm.com","username":"gilv"},"change_message_id":"8f1bbdcf5a4962e0cdf319f9941ca972103cf051","unresolved":false,"context_lines":[{"line_number":180,"context_line":"                    return HTTPPreconditionFailed(\u0027Migration source is \u0027 +"},{"line_number":181,"context_line":"                                                  \u0027missing\u0027)(env,"},{"line_number":182,"context_line":"                                                             start_response)"},{"line_number":183,"context_line":"                elif ((not prov) and (mig_source is not None)):"},{"line_number":184,"context_line":"                    return HTTPPreconditionFailed(\u0027Migration provider is \u0027 +"},{"line_number":185,"context_line":"                                                  \u0027missing\u0027)(env,"},{"line_number":186,"context_line":"                                                             start_response)"}],"source_content_type":"text/x-python","patch_set":16,"id":"baada198_95d2c8cd","line":183,"in_reply_to":"baada198_61bdbe66","updated":"2014-08-24 08:10:02.000000000","message":"Done","commit_id":"1e5ddde5fc3bc514d59e66a5c43592cbb502143c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"363fa98d6c1b66cf6a6cc4ba77c57e4db93c9b62","unresolved":false,"context_lines":[{"line_number":186,"context_line":"                                                             start_response)"},{"line_number":187,"context_line":"                elif (prov is not None) and (mig_source is not None):"},{"line_number":188,"context_line":"                    if prov not in self.migration_conf:"},{"line_number":189,"context_line":"                        return HTTPServiceUnavailable(body\u003d\u0027Invalid provide\u0027 +"},{"line_number":190,"context_line":"                                                      \u0027r\u0027)(env, start_response)"},{"line_number":191,"context_line":"                    if self.migration_conf[prov][\u0027driver_loaded\u0027] is False:"},{"line_number":192,"context_line":"                        return HTTPServiceUnavailable(body\u003d\u0027Invalid access \u0027 +"}],"source_content_type":"text/x-python","patch_set":16,"id":"baada198_d8bcb709","line":189,"updated":"2014-08-13 15:09:11.000000000","message":"I\u0027m not sure this should be 503. Perhaps 400? swiftclient will retry by default on receiving a 503, but this request will never work so the retries are just annoying","commit_id":"1e5ddde5fc3bc514d59e66a5c43592cbb502143c"},{"author":{"_account_id":9205,"name":"Gil Vernik","email":"gilv@il.ibm.com","username":"gilv"},"change_message_id":"8f1bbdcf5a4962e0cdf319f9941ca972103cf051","unresolved":false,"context_lines":[{"line_number":186,"context_line":"                                                             start_response)"},{"line_number":187,"context_line":"                elif (prov is not None) and (mig_source is not None):"},{"line_number":188,"context_line":"                    if prov not in self.migration_conf:"},{"line_number":189,"context_line":"                        return HTTPServiceUnavailable(body\u003d\u0027Invalid provide\u0027 +"},{"line_number":190,"context_line":"                                                      \u0027r\u0027)(env, start_response)"},{"line_number":191,"context_line":"                    if self.migration_conf[prov][\u0027driver_loaded\u0027] is False:"},{"line_number":192,"context_line":"                        return HTTPServiceUnavailable(body\u003d\u0027Invalid access \u0027 +"}],"source_content_type":"text/x-python","patch_set":16,"id":"7ac7a964_e7ac5d38","line":189,"in_reply_to":"baada198_d8bcb709","updated":"2014-08-24 08:10:02.000000000","message":"Done","commit_id":"1e5ddde5fc3bc514d59e66a5c43592cbb502143c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"363fa98d6c1b66cf6a6cc4ba77c57e4db93c9b62","unresolved":false,"context_lines":[{"line_number":189,"context_line":"                        return HTTPServiceUnavailable(body\u003d\u0027Invalid provide\u0027 +"},{"line_number":190,"context_line":"                                                      \u0027r\u0027)(env, start_response)"},{"line_number":191,"context_line":"                    if self.migration_conf[prov][\u0027driver_loaded\u0027] is False:"},{"line_number":192,"context_line":"                        return HTTPServiceUnavailable(body\u003d\u0027Invalid access \u0027 +"},{"line_number":193,"context_line":"                                                      \u0027driver\u0027)(env,"},{"line_number":194,"context_line":"                                                                start_response)"},{"line_number":195,"context_line":""}],"source_content_type":"text/x-python","patch_set":16,"id":"baada198_184aaf23","line":192,"updated":"2014-08-13 15:09:11.000000000","message":"ditto re 503 vs 400","commit_id":"1e5ddde5fc3bc514d59e66a5c43592cbb502143c"},{"author":{"_account_id":9205,"name":"Gil Vernik","email":"gilv@il.ibm.com","username":"gilv"},"change_message_id":"8f1bbdcf5a4962e0cdf319f9941ca972103cf051","unresolved":false,"context_lines":[{"line_number":189,"context_line":"                        return HTTPServiceUnavailable(body\u003d\u0027Invalid provide\u0027 +"},{"line_number":190,"context_line":"                                                      \u0027r\u0027)(env, start_response)"},{"line_number":191,"context_line":"                    if self.migration_conf[prov][\u0027driver_loaded\u0027] is False:"},{"line_number":192,"context_line":"                        return HTTPServiceUnavailable(body\u003d\u0027Invalid access \u0027 +"},{"line_number":193,"context_line":"                                                      \u0027driver\u0027)(env,"},{"line_number":194,"context_line":"                                                                start_response)"},{"line_number":195,"context_line":""}],"source_content_type":"text/x-python","patch_set":16,"id":"7ac7a964_c75da14a","line":192,"in_reply_to":"baada198_184aaf23","updated":"2014-08-24 08:10:02.000000000","message":"Done","commit_id":"1e5ddde5fc3bc514d59e66a5c43592cbb502143c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"363fa98d6c1b66cf6a6cc4ba77c57e4db93c9b62","unresolved":false,"context_lines":[{"line_number":213,"context_line":""},{"line_number":214,"context_line":"        original_resp_status \u003d env[\u0027swift.original_status\u0027]"},{"line_number":215,"context_line":"        del env[\u0027swift.original_status\u0027]"},{"line_number":216,"context_line":"        if (original_resp_status \u003d\u003d NOT_FOUND):"},{"line_number":217,"context_line":"            if obj and container and account and req.method in [\u0027GET\u0027, \u0027HEAD\u0027]:"},{"line_number":218,"context_line":"                container_info \u003d get_container_info(req.environ, self.app,"},{"line_number":219,"context_line":"                                                    swift_source\u003d\u0027DM\u0027)"}],"source_content_type":"text/x-python","patch_set":16,"id":"baada198_71fd5309","line":216,"updated":"2014-08-13 15:09:11.000000000","message":"nit: redundant ( )","commit_id":"1e5ddde5fc3bc514d59e66a5c43592cbb502143c"},{"author":{"_account_id":9205,"name":"Gil Vernik","email":"gilv@il.ibm.com","username":"gilv"},"change_message_id":"8f1bbdcf5a4962e0cdf319f9941ca972103cf051","unresolved":false,"context_lines":[{"line_number":213,"context_line":""},{"line_number":214,"context_line":"        original_resp_status \u003d env[\u0027swift.original_status\u0027]"},{"line_number":215,"context_line":"        del env[\u0027swift.original_status\u0027]"},{"line_number":216,"context_line":"        if (original_resp_status \u003d\u003d NOT_FOUND):"},{"line_number":217,"context_line":"            if obj and container and account and req.method in [\u0027GET\u0027, \u0027HEAD\u0027]:"},{"line_number":218,"context_line":"                container_info \u003d get_container_info(req.environ, self.app,"},{"line_number":219,"context_line":"                                                    swift_source\u003d\u0027DM\u0027)"}],"source_content_type":"text/x-python","patch_set":16,"id":"baada198_55e8307b","line":216,"in_reply_to":"baada198_71fd5309","updated":"2014-08-24 08:10:02.000000000","message":"Done","commit_id":"1e5ddde5fc3bc514d59e66a5c43592cbb502143c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"363fa98d6c1b66cf6a6cc4ba77c57e4db93c9b62","unresolved":false,"context_lines":[{"line_number":225,"context_line":"                                                       container, obj,"},{"line_number":226,"context_line":"                                                       container_md,"},{"line_number":227,"context_line":"                                                       start_response)"},{"line_number":228,"context_line":"                        except Exception as e:"},{"line_number":229,"context_line":"                            self.logger.error(e)"},{"line_number":230,"context_line":"                            rh \u003d {\u0027X-Migration-Status\u0027: str(e)}"},{"line_number":231,"context_line":"                            return \\"}],"source_content_type":"text/x-python","patch_set":16,"id":"baada198_e9b981ec","line":228,"updated":"2014-08-13 15:09:11.000000000","message":"Use more specific exception: except DataMigrationError as e - otherwise unexpected exceptions get obscured - e.g. see comment line 325","commit_id":"1e5ddde5fc3bc514d59e66a5c43592cbb502143c"},{"author":{"_account_id":9205,"name":"Gil Vernik","email":"gilv@il.ibm.com","username":"gilv"},"change_message_id":"8f1bbdcf5a4962e0cdf319f9941ca972103cf051","unresolved":false,"context_lines":[{"line_number":225,"context_line":"                                                       container, obj,"},{"line_number":226,"context_line":"                                                       container_md,"},{"line_number":227,"context_line":"                                                       start_response)"},{"line_number":228,"context_line":"                        except Exception as e:"},{"line_number":229,"context_line":"                            self.logger.error(e)"},{"line_number":230,"context_line":"                            rh \u003d {\u0027X-Migration-Status\u0027: str(e)}"},{"line_number":231,"context_line":"                            return \\"}],"source_content_type":"text/x-python","patch_set":16,"id":"7ac7a964_eeb98eea","line":228,"in_reply_to":"baada198_e9b981ec","updated":"2014-08-24 08:10:02.000000000","message":"Done","commit_id":"1e5ddde5fc3bc514d59e66a5c43592cbb502143c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"363fa98d6c1b66cf6a6cc4ba77c57e4db93c9b62","unresolved":false,"context_lines":[{"line_number":234,"context_line":""},{"line_number":235,"context_line":"        return original_resp"},{"line_number":236,"context_line":""},{"line_number":237,"context_line":"    def GETorHEAD_miss(self, original_env, account, container,"},{"line_number":238,"context_line":"                       obj, container_md, start_response):"},{"line_number":239,"context_line":"        \"\"\""},{"line_number":240,"context_line":"        Data migration logic for GET or HEAD requests. Identifies correct"}],"source_content_type":"text/x-python","patch_set":16,"id":"baada198_8e700b43","line":237,"updated":"2014-08-13 15:09:11.000000000","message":"account, container are unused","commit_id":"1e5ddde5fc3bc514d59e66a5c43592cbb502143c"},{"author":{"_account_id":9205,"name":"Gil Vernik","email":"gilv@il.ibm.com","username":"gilv"},"change_message_id":"8f1bbdcf5a4962e0cdf319f9941ca972103cf051","unresolved":false,"context_lines":[{"line_number":234,"context_line":""},{"line_number":235,"context_line":"        return original_resp"},{"line_number":236,"context_line":""},{"line_number":237,"context_line":"    def GETorHEAD_miss(self, original_env, account, container,"},{"line_number":238,"context_line":"                       obj, container_md, start_response):"},{"line_number":239,"context_line":"        \"\"\""},{"line_number":240,"context_line":"        Data migration logic for GET or HEAD requests. Identifies correct"}],"source_content_type":"text/x-python","patch_set":16,"id":"7ac7a964_4a3195eb","line":237,"in_reply_to":"baada198_8e700b43","updated":"2014-08-24 08:10:02.000000000","message":"Done","commit_id":"1e5ddde5fc3bc514d59e66a5c43592cbb502143c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"363fa98d6c1b66cf6a6cc4ba77c57e4db93c9b62","unresolved":false,"context_lines":[{"line_number":241,"context_line":"        access driver and then reads an object from another storage. Received"},{"line_number":242,"context_line":"        object will be written in local Swift and returned back to the user."},{"line_number":243,"context_line":""},{"line_number":244,"context_line":"        :params original_env: environ of the application"},{"line_number":245,"context_line":"        :params account: account name in Swift"},{"line_number":246,"context_line":"        :params container: container name in Swift"},{"line_number":247,"context_line":"        :params obj: object name"}],"source_content_type":"text/x-python","patch_set":16,"id":"baada198_43f32ce8","line":244,"updated":"2014-08-13 15:09:11.000000000","message":"Throughout - the correct sphinx directive/label is :param rather than :params","commit_id":"1e5ddde5fc3bc514d59e66a5c43592cbb502143c"},{"author":{"_account_id":9205,"name":"Gil Vernik","email":"gilv@il.ibm.com","username":"gilv"},"change_message_id":"8f1bbdcf5a4962e0cdf319f9941ca972103cf051","unresolved":false,"context_lines":[{"line_number":241,"context_line":"        access driver and then reads an object from another storage. Received"},{"line_number":242,"context_line":"        object will be written in local Swift and returned back to the user."},{"line_number":243,"context_line":""},{"line_number":244,"context_line":"        :params original_env: environ of the application"},{"line_number":245,"context_line":"        :params account: account name in Swift"},{"line_number":246,"context_line":"        :params container: container name in Swift"},{"line_number":247,"context_line":"        :params obj: object name"}],"source_content_type":"text/x-python","patch_set":16,"id":"baada198_80d7e84f","line":244,"in_reply_to":"baada198_43f32ce8","updated":"2014-08-24 08:10:02.000000000","message":"Done","commit_id":"1e5ddde5fc3bc514d59e66a5c43592cbb502143c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"363fa98d6c1b66cf6a6cc4ba77c57e4db93c9b62","unresolved":false,"context_lines":[{"line_number":256,"context_line":"        metadata, read_length, body_stream, \\"},{"line_number":257,"context_line":"            content_type, timestamp \u003d access_resolver.get_object(obj)"},{"line_number":258,"context_line":""},{"line_number":259,"context_line":"        if body_stream is None:"},{"line_number":260,"context_line":"            raise DataMigrationError(\u0027Failed to retrieve object for migration\u0027)"},{"line_number":261,"context_line":"        sys_metadata \u003d dict()"},{"line_number":262,"context_line":"        sys_metadata[\u0027Migration-Import-Time\u0027] \u003d str(time.time())"}],"source_content_type":"text/x-python","patch_set":16,"id":"baada198_5145af42","line":259,"updated":"2014-08-13 15:09:11.000000000","message":"do we need access_resolver.finalize() here before leaving method?","commit_id":"1e5ddde5fc3bc514d59e66a5c43592cbb502143c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"b414dd5bd497f20cec15cc2dc8bf4fb3275b1f61","unresolved":false,"context_lines":[{"line_number":256,"context_line":"        metadata, read_length, body_stream, \\"},{"line_number":257,"context_line":"            content_type, timestamp \u003d access_resolver.get_object(obj)"},{"line_number":258,"context_line":""},{"line_number":259,"context_line":"        if body_stream is None:"},{"line_number":260,"context_line":"            raise DataMigrationError(\u0027Failed to retrieve object for migration\u0027)"},{"line_number":261,"context_line":"        sys_metadata \u003d dict()"},{"line_number":262,"context_line":"        sys_metadata[\u0027Migration-Import-Time\u0027] \u003d str(time.time())"}],"source_content_type":"text/x-python","patch_set":16,"id":"fa98f980_3e7b3df6","line":259,"in_reply_to":"baada198_4077908e","updated":"2014-09-03 16:30:35.000000000","message":"sorry, my comment wasn\u0027t clear - i meant to point out that if this exception is raised then access_resolver.finalize() will not be called, but maybe it should be?","commit_id":"1e5ddde5fc3bc514d59e66a5c43592cbb502143c"},{"author":{"_account_id":9205,"name":"Gil Vernik","email":"gilv@il.ibm.com","username":"gilv"},"change_message_id":"8f1bbdcf5a4962e0cdf319f9941ca972103cf051","unresolved":false,"context_lines":[{"line_number":256,"context_line":"        metadata, read_length, body_stream, \\"},{"line_number":257,"context_line":"            content_type, timestamp \u003d access_resolver.get_object(obj)"},{"line_number":258,"context_line":""},{"line_number":259,"context_line":"        if body_stream is None:"},{"line_number":260,"context_line":"            raise DataMigrationError(\u0027Failed to retrieve object for migration\u0027)"},{"line_number":261,"context_line":"        sys_metadata \u003d dict()"},{"line_number":262,"context_line":"        sys_metadata[\u0027Migration-Import-Time\u0027] \u003d str(time.time())"}],"source_content_type":"text/x-python","patch_set":16,"id":"baada198_4077908e","line":259,"in_reply_to":"baada198_5145af42","updated":"2014-08-24 08:10:02.000000000","message":"The idea that each driver will have the same interface. There should be method finalize for the driver to clean, close connection, etc.\nI would prefer to keep it, if you don\u0027t mind.","commit_id":"1e5ddde5fc3bc514d59e66a5c43592cbb502143c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"363fa98d6c1b66cf6a6cc4ba77c57e4db93c9b62","unresolved":false,"context_lines":[{"line_number":297,"context_line":"        new_env[\u0027CONTENT_TYPE\u0027] \u003d content_type"},{"line_number":298,"context_line":"        new_env[\u0027swift.source\u0027] \u003d \u0027DM\u0027"},{"line_number":299,"context_line":"        create_obj_req \u003d Request.blank(new_env[\u0027PATH_INFO\u0027], new_env)"},{"line_number":300,"context_line":"        create_obj_req.headers[\u0027X-Timestamp\u0027] \u003d min(time.time(), timestamp)"},{"line_number":301,"context_line":"        for key in metadata.keys():"},{"line_number":302,"context_line":"            if key.lower().startswith(\u0027x-object-meta-\u0027):"},{"line_number":303,"context_line":"                create_obj_req.headers[key] \u003d metadata[key]"}],"source_content_type":"text/x-python","patch_set":16,"id":"baada198_80260fca","line":300,"updated":"2014-08-13 15:09:11.000000000","message":"convert to float --\u003e min(time.time(), float(timestamp)) because the swift driver returns string timestamp which is always \u003e time.time().\n\nBUT problem here is that if the migrated object is deleted from the swift container, then you try to get again, the migration will fail because there is a tombstone in swift with same timestamp as migration source (in fact, the upload_object gets a 201 depsite the tombstone, but the original object GET returns 404 because of the tombstone, which may be a bug with swift).\n\nIf you want to be able to migrate an object that has been deleted from target container then you will need to use time.time() for timestamp, and perhaps record the source timestamp as migration obj sysmeta??","commit_id":"1e5ddde5fc3bc514d59e66a5c43592cbb502143c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"b414dd5bd497f20cec15cc2dc8bf4fb3275b1f61","unresolved":false,"context_lines":[{"line_number":297,"context_line":"        new_env[\u0027CONTENT_TYPE\u0027] \u003d content_type"},{"line_number":298,"context_line":"        new_env[\u0027swift.source\u0027] \u003d \u0027DM\u0027"},{"line_number":299,"context_line":"        create_obj_req \u003d Request.blank(new_env[\u0027PATH_INFO\u0027], new_env)"},{"line_number":300,"context_line":"        create_obj_req.headers[\u0027X-Timestamp\u0027] \u003d min(time.time(), timestamp)"},{"line_number":301,"context_line":"        for key in metadata.keys():"},{"line_number":302,"context_line":"            if key.lower().startswith(\u0027x-object-meta-\u0027):"},{"line_number":303,"context_line":"                create_obj_req.headers[key] \u003d metadata[key]"}],"source_content_type":"text/x-python","patch_set":16,"id":"fa98f980_7e6b05f1","line":300,"in_reply_to":"7ac7a964_4db60f3f","updated":"2014-09-03 16:30:35.000000000","message":"OK, I get it - importantly, the swift driver behaves the same as filesystem driver w.r.t. not migrating again once object is deleted.\n\nSomething to consider: The backend obj server returns X-Backend-Timestamp \u003d delete time with a 404 which could maybe be used to make the migration conditional on the migration source object being newer than deleted object - that would save getting the source object, uploading and then the upload fails ... but I\u0027d suggest leaving that optimisation for a follow on patch ;)","commit_id":"1e5ddde5fc3bc514d59e66a5c43592cbb502143c"},{"author":{"_account_id":9205,"name":"Gil Vernik","email":"gilv@il.ibm.com","username":"gilv"},"change_message_id":"8f1bbdcf5a4962e0cdf319f9941ca972103cf051","unresolved":false,"context_lines":[{"line_number":297,"context_line":"        new_env[\u0027CONTENT_TYPE\u0027] \u003d content_type"},{"line_number":298,"context_line":"        new_env[\u0027swift.source\u0027] \u003d \u0027DM\u0027"},{"line_number":299,"context_line":"        create_obj_req \u003d Request.blank(new_env[\u0027PATH_INFO\u0027], new_env)"},{"line_number":300,"context_line":"        create_obj_req.headers[\u0027X-Timestamp\u0027] \u003d min(time.time(), timestamp)"},{"line_number":301,"context_line":"        for key in metadata.keys():"},{"line_number":302,"context_line":"            if key.lower().startswith(\u0027x-object-meta-\u0027):"},{"line_number":303,"context_line":"                create_obj_req.headers[key] \u003d metadata[key]"}],"source_content_type":"text/x-python","patch_set":16,"id":"7ac7a964_4db60f3f","line":300,"in_reply_to":"baada198_80260fca","updated":"2014-08-24 08:10:02.000000000","message":"Conversion to  float - Done\n\nIn theory, local Swift should be most updated than old storage, so if object was migrated and than deleted, it shouldn\u0027t be migrated again.\n\nIdeally it would be good to get some 404 with noticed that it was deleted, to prevent call to old storage.\n\nThe need for min (time.. ) was your suggestion in the past to make sort of validation.\n\nThe need to put X-Timestamp with old timestap is to prevent the problematic case that migration may ovewrite local object that was created or deleted locally ( has higher timestamp than one in the old storage, of course trying to do it best, since time may not be synchronized ).","commit_id":"1e5ddde5fc3bc514d59e66a5c43592cbb502143c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"363fa98d6c1b66cf6a6cc4ba77c57e4db93c9b62","unresolved":false,"context_lines":[{"line_number":322,"context_line":"        \"\"\""},{"line_number":323,"context_line":"        migration_provider \u003d container_md.get(\u0027migration-provider\u0027, \u0027\u0027).lower()"},{"line_number":324,"context_line":"        data_source \u003d container_md.get(\u0027migration-source\u0027, \u0027\u0027).lower()"},{"line_number":325,"context_line":"        the_class \u003d migration_conf[migration_provider][\u0027the_class\u0027]"},{"line_number":326,"context_line":"        params \u003d dict()"},{"line_number":327,"context_line":"        if migration_conf[migration_provider][\u0027driver_loaded\u0027] is True:"},{"line_number":328,"context_line":"            for key in migration_conf[migration_provider][\u0027keys\u0027]:"}],"source_content_type":"text/x-python","patch_set":16,"id":"baada198_a91149fa","line":325,"updated":"2014-08-13 15:09:11.000000000","message":"Need to check that migration_provider is in migration_conf (provider may have been disabled in middleware config, but still referenced in container metadata) - otherwise this method may throw an IndexError. See unit tests to reveal this https://gist.github.com/alistairncoles/d50c8c3bf981cacbf583","commit_id":"1e5ddde5fc3bc514d59e66a5c43592cbb502143c"},{"author":{"_account_id":9205,"name":"Gil Vernik","email":"gilv@il.ibm.com","username":"gilv"},"change_message_id":"8f1bbdcf5a4962e0cdf319f9941ca972103cf051","unresolved":false,"context_lines":[{"line_number":322,"context_line":"        \"\"\""},{"line_number":323,"context_line":"        migration_provider \u003d container_md.get(\u0027migration-provider\u0027, \u0027\u0027).lower()"},{"line_number":324,"context_line":"        data_source \u003d container_md.get(\u0027migration-source\u0027, \u0027\u0027).lower()"},{"line_number":325,"context_line":"        the_class \u003d migration_conf[migration_provider][\u0027the_class\u0027]"},{"line_number":326,"context_line":"        params \u003d dict()"},{"line_number":327,"context_line":"        if migration_conf[migration_provider][\u0027driver_loaded\u0027] is True:"},{"line_number":328,"context_line":"            for key in migration_conf[migration_provider][\u0027keys\u0027]:"}],"source_content_type":"text/x-python","patch_set":16,"id":"7ac7a964_9ee76fe4","line":325,"in_reply_to":"baada198_a91149fa","updated":"2014-08-24 08:10:02.000000000","message":"Done","commit_id":"1e5ddde5fc3bc514d59e66a5c43592cbb502143c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"363fa98d6c1b66cf6a6cc4ba77c57e4db93c9b62","unresolved":false,"context_lines":[{"line_number":358,"context_line":"        migration_conf[dr] \u003d {\u0027keys\u0027: keys, \u0027the_class\u0027: the_class,"},{"line_number":359,"context_line":"                              \u0027driver_loaded\u0027: the_class.driver_loaded}"},{"line_number":360,"context_line":"        if (the_class.driver_loaded):"},{"line_number":361,"context_line":"            swift_info[dr] \u003d \u0027enable\u0027"},{"line_number":362,"context_line":"    register_swift_info(\u0027data_migration\u0027, False, **swift_info)"},{"line_number":363,"context_line":""},{"line_number":364,"context_line":"    def data_migration_filter(app):"}],"source_content_type":"text/x-python","patch_set":16,"id":"baada198_12a28e96","line":361,"updated":"2014-08-13 15:09:11.000000000","message":"nit: maybe \u0027enabled\u0027 ?\n\nAlso nit: there\u0027s a slight inconsistency here - if swiftclient cannot be imported then it ignores that driver and continue. But if I specify another optional driver and it can\u0027t be imported (line 356) then an exception will be raised.\n\nI think we may have discussed this before :) - how about ignoring/logging error for any drivers that fail to load? swift info will show what has successfully loaded?","commit_id":"1e5ddde5fc3bc514d59e66a5c43592cbb502143c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"b414dd5bd497f20cec15cc2dc8bf4fb3275b1f61","unresolved":false,"context_lines":[{"line_number":358,"context_line":"        migration_conf[dr] \u003d {\u0027keys\u0027: keys, \u0027the_class\u0027: the_class,"},{"line_number":359,"context_line":"                              \u0027driver_loaded\u0027: the_class.driver_loaded}"},{"line_number":360,"context_line":"        if (the_class.driver_loaded):"},{"line_number":361,"context_line":"            swift_info[dr] \u003d \u0027enable\u0027"},{"line_number":362,"context_line":"    register_swift_info(\u0027data_migration\u0027, False, **swift_info)"},{"line_number":363,"context_line":""},{"line_number":364,"context_line":"    def data_migration_filter(app):"}],"source_content_type":"text/x-python","patch_set":16,"id":"fa98f980_617b648c","line":361,"in_reply_to":"7ac7a964_2d9f1361","updated":"2014-09-03 16:30:35.000000000","message":"I think what I meant was that in the drivers module, if swiftclient cannot be imported then there is no exception - the swift driver will load but never succeed in getting an object - would it be more consistent to throw exception if swiftclient is not available, same as throwing exception if driver class cannot be imported?\n\n(not a blocker btw)","commit_id":"1e5ddde5fc3bc514d59e66a5c43592cbb502143c"},{"author":{"_account_id":9205,"name":"Gil Vernik","email":"gilv@il.ibm.com","username":"gilv"},"change_message_id":"8f1bbdcf5a4962e0cdf319f9941ca972103cf051","unresolved":false,"context_lines":[{"line_number":358,"context_line":"        migration_conf[dr] \u003d {\u0027keys\u0027: keys, \u0027the_class\u0027: the_class,"},{"line_number":359,"context_line":"                              \u0027driver_loaded\u0027: the_class.driver_loaded}"},{"line_number":360,"context_line":"        if (the_class.driver_loaded):"},{"line_number":361,"context_line":"            swift_info[dr] \u003d \u0027enable\u0027"},{"line_number":362,"context_line":"    register_swift_info(\u0027data_migration\u0027, False, **swift_info)"},{"line_number":363,"context_line":""},{"line_number":364,"context_line":"    def data_migration_filter(app):"}],"source_content_type":"text/x-python","patch_set":16,"id":"7ac7a964_2d9f1361","line":361,"in_reply_to":"baada198_12a28e96","updated":"2014-08-24 08:10:02.000000000","message":"if driver is specified in the configuration and failed to load here, than exception is thrown by __import__ and Swift will not start.\n\nLines 356 checking that you specify some driver that you want to use for migration. if it was not loaded - throw exception.\n\nWhy do you think the  inconsistency here?","commit_id":"1e5ddde5fc3bc514d59e66a5c43592cbb502143c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"bc14f3b3b1132d94c7079f61e8fee122b68defcc","unresolved":false,"context_lines":[{"line_number":136,"context_line":"                    if key.replace(\u0027migration-\u0027,"},{"line_number":137,"context_line":"                                   \u0027\u0027) not in self.mconf.get(\u0027keys\u0027, {}):"},{"line_number":138,"context_line":"                        self._response_headers.append((\u0027X-Container-\u0027 +"},{"line_number":139,"context_line":"                                                       key.title(), val))"},{"line_number":140,"context_line":"        start_response(self._response_status,"},{"line_number":141,"context_line":"                       self._response_headers,"},{"line_number":142,"context_line":"                       self._response_exc_info)"}],"source_content_type":"text/x-python","patch_set":20,"id":"fa98f980_2f7cc171","line":139,"updated":"2014-09-04 15:07:44.000000000","message":"this only adds migration headers to a HEAD response - need to do same for GET","commit_id":"d20cd4f09c40fd195da3e94aad079e0baa9fb2e4"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"bc14f3b3b1132d94c7079f61e8fee122b68defcc","unresolved":false,"context_lines":[{"line_number":205,"context_line":"                for key, val in req.headers.iteritems():"},{"line_number":206,"context_line":"                    if key.startswith(\u0027X-Container-Migration-\u0027):"},{"line_number":207,"context_line":"                        req.headers[key.replace(\u0027X-Container-\u0027,"},{"line_number":208,"context_line":"                                                \u0027X-Container-Sysmeta-\u0027)] \u003d val"},{"line_number":209,"context_line":""},{"line_number":210,"context_line":"        original_env \u003d req.environ.copy()"},{"line_number":211,"context_line":"        ctx \u003d DataMigrationContext(self.app, container_md,"}],"source_content_type":"text/x-python","patch_set":20,"id":"fa98f980_540c3c3b","line":208,"updated":"2014-09-04 15:07:44.000000000","message":"This allows user to set arbitrary sysmeta e.g. x-container-migration-apples with no restriction on the number. We should only accept/translate to sysmeta the headers that we expect i.e. -active, -provider, -source and the driver-specific keys.","commit_id":"d20cd4f09c40fd195da3e94aad079e0baa9fb2e4"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"bc14f3b3b1132d94c7079f61e8fee122b68defcc","unresolved":false,"context_lines":[{"line_number":278,"context_line":"            raise DataMigrationError(\u0027Failed to create local object.\u0027 +"},{"line_number":279,"context_line":"                                     \u0027Status {0}\u0027.format(status))"},{"line_number":280,"context_line":""},{"line_number":281,"context_line":"        raise DataMigrationError(\u0027Data migration failed. \u0027 +"},{"line_number":282,"context_line":"                                 \u0027Failed to access old storage\u0027)"},{"line_number":283,"context_line":""},{"line_number":284,"context_line":"    def upload_object(self, env, data, length, metadata, sys_metadata,"}],"source_content_type":"text/x-python","patch_set":20,"id":"fa98f980_3d8ec6d0","line":281,"updated":"2014-09-04 15:07:44.000000000","message":"this line cannot be reached","commit_id":"d20cd4f09c40fd195da3e94aad079e0baa9fb2e4"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"bc14f3b3b1132d94c7079f61e8fee122b68defcc","unresolved":false,"context_lines":[{"line_number":360,"context_line":"                if key.strip()]"},{"line_number":361,"context_line":"        module_name \u003d conf.get(\u0027driver_\u0027 + dr + \u0027_module\u0027, \u0027\u0027)"},{"line_number":362,"context_line":"        mo \u003d module_name[:module_name.rfind(\u0027:\u0027)]"},{"line_number":363,"context_line":"        cl \u003d module_name[module_name.rfind(\u0027:\u0027) + 1:]"},{"line_number":364,"context_line":"        module \u003d __import__(mo, fromlist\u003d[cl])"},{"line_number":365,"context_line":"        the_class \u003d getattr(module, cl)"},{"line_number":366,"context_line":"        migration_conf[dr] \u003d {\u0027keys\u0027: keys, \u0027the_class\u0027: the_class,"}],"source_content_type":"text/x-python","patch_set":20,"id":"fa98f980_a0a9a625","line":363,"updated":"2014-09-04 15:07:44.000000000","message":"Q: could the keys be class variables that are read from the loaded class here? I\u0027m not sure I see the value of them being in the config file because they don\u0027t seem to be configurable i.e. the keys must be provided for the driver to work.","commit_id":"d20cd4f09c40fd195da3e94aad079e0baa9fb2e4"},{"author":{"_account_id":9625,"name":"Thiago da Silva","email":"thiagodasilva@gmail.com","username":"thiago"},"change_message_id":"a2a5457425f0ff8074b00d7f9652038db33707d9","unresolved":false,"context_lines":[{"line_number":256,"context_line":"        :returns Response with migrated object."},{"line_number":257,"context_line":"        :raises DataMigrationError when operation fails"},{"line_number":258,"context_line":"        \"\"\""},{"line_number":259,"context_line":"        orig_env \u003d dict(original_env)"},{"line_number":260,"context_line":"        access_resolver \u003d self.remote_driver_resolver(container_md,"},{"line_number":261,"context_line":"                                                      self.migration_conf)"},{"line_number":262,"context_line":"        try:"}],"source_content_type":"text/x-python","patch_set":22,"id":"fa98f980_2b7644a4","line":259,"updated":"2014-09-16 01:51:51.000000000","message":"i think creating this new dictionary might not be necessary","commit_id":"63a19a6152b5eae20641d541aa9fa1c7b9977208"},{"author":{"_account_id":9205,"name":"Gil Vernik","email":"gilv@il.ibm.com","username":"gilv"},"change_message_id":"ef8d0a38b273c71af3b4f33398ee79747a421983","unresolved":false,"context_lines":[{"line_number":256,"context_line":"        :returns Response with migrated object."},{"line_number":257,"context_line":"        :raises DataMigrationError when operation fails"},{"line_number":258,"context_line":"        \"\"\""},{"line_number":259,"context_line":"        orig_env \u003d dict(original_env)"},{"line_number":260,"context_line":"        access_resolver \u003d self.remote_driver_resolver(container_md,"},{"line_number":261,"context_line":"                                                      self.migration_conf)"},{"line_number":262,"context_line":"        try:"}],"source_content_type":"text/x-python","patch_set":22,"id":"fa98f980_e1a7fb4d","line":259,"in_reply_to":"fa98f980_2b7644a4","updated":"2014-09-16 05:20:51.000000000","message":"The problem with environment, that if you send one request - it spoil the environment dictionary. In my patch, user send one request and my patch performs PUT. So i need to preserve the original environment to return response to the user.","commit_id":"63a19a6152b5eae20641d541aa9fa1c7b9977208"},{"author":{"_account_id":9625,"name":"Thiago da Silva","email":"thiagodasilva@gmail.com","username":"thiago"},"change_message_id":"a2a5457425f0ff8074b00d7f9652038db33707d9","unresolved":false,"context_lines":[{"line_number":296,"context_line":"        :param timestamp: timestamp of an object in the old storage"},{"line_number":297,"context_line":"        :returns HTTP code result of an upload operation"},{"line_number":298,"context_line":"        \"\"\""},{"line_number":299,"context_line":"        new_env \u003d dict(env)"},{"line_number":300,"context_line":"        new_env[\u0027REQUEST_METHOD\u0027] \u003d \u0027PUT\u0027"},{"line_number":301,"context_line":"        new_env[\u0027wsgi.input\u0027] \u003d data"},{"line_number":302,"context_line":"        new_env[\u0027CONTENT_LENGTH\u0027] \u003d length"}],"source_content_type":"text/x-python","patch_set":22,"id":"fa98f980_cb92189e","line":299,"updated":"2014-09-16 01:51:51.000000000","message":"don\u0027t think creating this new dict is necessary","commit_id":"63a19a6152b5eae20641d541aa9fa1c7b9977208"},{"author":{"_account_id":9205,"name":"Gil Vernik","email":"gilv@il.ibm.com","username":"gilv"},"change_message_id":"ef8d0a38b273c71af3b4f33398ee79747a421983","unresolved":false,"context_lines":[{"line_number":296,"context_line":"        :param timestamp: timestamp of an object in the old storage"},{"line_number":297,"context_line":"        :returns HTTP code result of an upload operation"},{"line_number":298,"context_line":"        \"\"\""},{"line_number":299,"context_line":"        new_env \u003d dict(env)"},{"line_number":300,"context_line":"        new_env[\u0027REQUEST_METHOD\u0027] \u003d \u0027PUT\u0027"},{"line_number":301,"context_line":"        new_env[\u0027wsgi.input\u0027] \u003d data"},{"line_number":302,"context_line":"        new_env[\u0027CONTENT_LENGTH\u0027] \u003d length"}],"source_content_type":"text/x-python","patch_set":22,"id":"fa98f980_e17e1bae","line":299,"in_reply_to":"fa98f980_cb92189e","updated":"2014-09-16 05:20:51.000000000","message":"I will check if this one can be avoided, since i already have copy in the caller function.","commit_id":"63a19a6152b5eae20641d541aa9fa1c7b9977208"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"e387ba656abe913c3625351bde0b0b80d871365f","unresolved":false,"context_lines":[{"line_number":315,"context_line":"            if key not in reserved and key.startswith(\u0027driver_\u0027 + dr):"},{"line_number":316,"context_line":"                add_params[key] \u003d val"},{"line_number":317,"context_line":"        # validate driver specific config with the driver class"},{"line_number":318,"context_line":"        the_class.check_conf(add_params)"},{"line_number":319,"context_line":"        keys \u003d the_class.get_keys() or []"},{"line_number":320,"context_line":"        migration_conf[dr] \u003d {\u0027keys\u0027: keys, \u0027the_class\u0027: the_class,"},{"line_number":321,"context_line":"                              \u0027driver_loaded\u0027: the_class.driver_loaded,"}],"source_content_type":"text/x-python","patch_set":23,"id":"da9df570_50e3f806","line":318,"updated":"2014-09-23 05:49:21.000000000","message":"the check_conf method potentially raises a DataMigrationDriverError, did we want to catch this here, log an exception and continue attempting to load the next driver? or just fail if the error is raised?","commit_id":"4a4004287583df78997ec351b17896fc611d9fca"},{"author":{"_account_id":9205,"name":"Gil Vernik","email":"gilv@il.ibm.com","username":"gilv"},"change_message_id":"ac94e5e0f9df518bba58a726d3f3b737f3e68ee0","unresolved":false,"context_lines":[{"line_number":315,"context_line":"            if key not in reserved and key.startswith(\u0027driver_\u0027 + dr):"},{"line_number":316,"context_line":"                add_params[key] \u003d val"},{"line_number":317,"context_line":"        # validate driver specific config with the driver class"},{"line_number":318,"context_line":"        the_class.check_conf(add_params)"},{"line_number":319,"context_line":"        keys \u003d the_class.get_keys() or []"},{"line_number":320,"context_line":"        migration_conf[dr] \u003d {\u0027keys\u0027: keys, \u0027the_class\u0027: the_class,"},{"line_number":321,"context_line":"                              \u0027driver_loaded\u0027: the_class.driver_loaded,"}],"source_content_type":"text/x-python","patch_set":23,"id":"da9df570_1031705a","line":318,"in_reply_to":"da9df570_50e3f806","updated":"2014-09-23 07:36:07.000000000","message":"It was discussed in the past, in some very early patches. It depends how you want to approach this type of error. The flow is that user defines some migration driver in the proxy-conf and than start proxy server. If there is something wrong, like missing mandatory parameters or wrong driver class  - then scream :) and stop everything, so user will know he did a mistake in the middleware configuration. It\u0027s the best way to find out that driver is not configured as it suppose to be.","commit_id":"4a4004287583df78997ec351b17896fc611d9fca"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"54baaf7d3eee021bebc6c045801f7d4257b569c3","unresolved":false,"context_lines":[{"line_number":310,"context_line":"        mo \u003d module_name[:module_name.rfind(\u0027:\u0027)]"},{"line_number":311,"context_line":"        cl \u003d module_name[module_name.rfind(\u0027:\u0027) + 1:]"},{"line_number":312,"context_line":"        module \u003d __import__(mo, fromlist\u003d[cl])"},{"line_number":313,"context_line":"        the_class \u003d getattr(module, cl)"},{"line_number":314,"context_line":"        add_params \u003d {}"},{"line_number":315,"context_line":"        for key, val in conf.iteritems():"},{"line_number":316,"context_line":"            if key not in reserved and key.startswith(\u0027driver_\u0027 + dr):"}],"source_content_type":"text/x-python","patch_set":27,"id":"9aa7fdbe_fb5f030d","line":313,"updated":"2014-10-24 09:42:41.000000000","message":"i would perfer these plugins be exposed via entry points and loaded with pkg_resources.","commit_id":"52eeff33144174bbd51593163c5391ce48e08473"},{"author":{"_account_id":9205,"name":"Gil Vernik","email":"gilv@il.ibm.com","username":"gilv"},"change_message_id":"9f05a155ac4d16a8c2b6652f81a409a5c67e71bc","unresolved":false,"context_lines":[{"line_number":310,"context_line":"        mo \u003d module_name[:module_name.rfind(\u0027:\u0027)]"},{"line_number":311,"context_line":"        cl \u003d module_name[module_name.rfind(\u0027:\u0027) + 1:]"},{"line_number":312,"context_line":"        module \u003d __import__(mo, fromlist\u003d[cl])"},{"line_number":313,"context_line":"        the_class \u003d getattr(module, cl)"},{"line_number":314,"context_line":"        add_params \u003d {}"},{"line_number":315,"context_line":"        for key, val in conf.iteritems():"},{"line_number":316,"context_line":"            if key not in reserved and key.startswith(\u0027driver_\u0027 + dr):"}],"source_content_type":"text/x-python","patch_set":27,"id":"7a8c0949_c7900fef","line":313,"in_reply_to":"9aa7fdbe_fb5f030d","updated":"2014-11-02 02:27:13.000000000","message":"Can you explain me please what do you mean? I am not sure i understand the suggestion, can you point me to some example?","commit_id":"52eeff33144174bbd51593163c5391ce48e08473"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"54baaf7d3eee021bebc6c045801f7d4257b569c3","unresolved":false,"context_lines":[{"line_number":323,"context_line":"                              \u0027additional_params\u0027: add_params}"},{"line_number":324,"context_line":"        if (the_class.driver_loaded):"},{"line_number":325,"context_line":"            swift_info[dr] \u003d \u0027enabled\u0027"},{"line_number":326,"context_line":"    register_swift_info(\u0027data_migration\u0027, False, **swift_info)"},{"line_number":327,"context_line":""},{"line_number":328,"context_line":"    def data_migration_filter(app):"},{"line_number":329,"context_line":"        return DataMigrationMiddleware(app, conf, migration_conf)"}],"source_content_type":"text/x-python","patch_set":27,"id":"9aa7fdbe_7ba31300","line":326,"updated":"2014-10-24 09:42:41.000000000","message":"Can you have multiple data migration middlewares in the pipeline at the same time?  What does registration look like in this case?","commit_id":"52eeff33144174bbd51593163c5391ce48e08473"},{"author":{"_account_id":9205,"name":"Gil Vernik","email":"gilv@il.ibm.com","username":"gilv"},"change_message_id":"9f05a155ac4d16a8c2b6652f81a409a5c67e71bc","unresolved":false,"context_lines":[{"line_number":323,"context_line":"                              \u0027additional_params\u0027: add_params}"},{"line_number":324,"context_line":"        if (the_class.driver_loaded):"},{"line_number":325,"context_line":"            swift_info[dr] \u003d \u0027enabled\u0027"},{"line_number":326,"context_line":"    register_swift_info(\u0027data_migration\u0027, False, **swift_info)"},{"line_number":327,"context_line":""},{"line_number":328,"context_line":"    def data_migration_filter(app):"},{"line_number":329,"context_line":"        return DataMigrationMiddleware(app, conf, migration_conf)"}],"source_content_type":"text/x-python","patch_set":27,"id":"7a8c0949_a79303f9","line":326,"in_reply_to":"9aa7fdbe_7ba31300","updated":"2014-11-02 02:27:13.000000000","message":"It\u0027s not different from any other middleware. Everyone can define the same middleware many times and probably get wrong behavior at some point. \nmigration middleware suppose to be defined once and not multiple times, is there way in Swift to make sure a middleware is defined once in the pipeline and not multiple times?","commit_id":"52eeff33144174bbd51593163c5391ce48e08473"}],"test/unit/common/middleware/test_data_migration.py":[{"author":{"_account_id":6889,"name":"Fabien Boucher","email":"fboucher@redhat.com","username":"fabien-boucher"},"change_message_id":"6989cb8bfbcafb02f9c9a44fa05b2f1bed71f27d","unresolved":false,"context_lines":[{"line_number":22,"context_line":"import imp"},{"line_number":23,"context_line":""},{"line_number":24,"context_line":"local_files \u003d [\u0027obj1.dat\u0027]"},{"line_number":25,"context_line":""},{"line_number":26,"context_line":""},{"line_number":27,"context_line":"class FakeApp(object):"},{"line_number":28,"context_line":"    GET_counter \u003d 0"}],"source_content_type":"text/x-python","patch_set":5,"id":"AAAAUH%2F%2F6aw%3D","line":25,"updated":"2014-01-06 21:57:30.000000000","message":"I think it is better to have two separate test files. One for swift/common/data_migrator_drivers.py and the other for the middleware with the proper mocks to data_migrator drivers.","commit_id":"94be0c854c189f22f9f1debf0f4a022c09fc3f40"},{"author":{"_account_id":9205,"name":"Gil Vernik","email":"gilv@il.ibm.com","username":"gilv"},"change_message_id":"5c8750ba8b7931c1aa887dbab94bb58ebf67c7d3","unresolved":false,"context_lines":[{"line_number":22,"context_line":"import imp"},{"line_number":23,"context_line":""},{"line_number":24,"context_line":"local_files \u003d [\u0027obj1.dat\u0027]"},{"line_number":25,"context_line":""},{"line_number":26,"context_line":""},{"line_number":27,"context_line":"class FakeApp(object):"},{"line_number":28,"context_line":"    GET_counter \u003d 0"}],"source_content_type":"text/x-python","patch_set":5,"id":"AAAAUH%2F%2F49Y%3D","line":25,"in_reply_to":"AAAAUH%2F%2F6aw%3D","updated":"2014-01-07 07:34:33.000000000","message":"I agree and  I am working in parallel to improve testing, in particular i am trying to figure out the best way to test drivers itself, since it\u0027s not so trivial to simulate other clouds,","commit_id":"94be0c854c189f22f9f1debf0f4a022c09fc3f40"},{"author":{"_account_id":6889,"name":"Fabien Boucher","email":"fboucher@redhat.com","username":"fabien-boucher"},"change_message_id":"6989cb8bfbcafb02f9c9a44fa05b2f1bed71f27d","unresolved":false,"context_lines":[{"line_number":194,"context_line":"        self.assertEquals(res.status_int, 404)"},{"line_number":195,"context_line":""},{"line_number":196,"context_line":""},{"line_number":197,"context_line":"class TestDataMigrationLibCloudAmazonS3(unittest.TestCase):"},{"line_number":198,"context_line":"    global_conf \u003d {\u0027log_name\u0027: \u0027proxy-server\u0027,"},{"line_number":199,"context_line":"                   \u0027eventlet_debug\u0027: \u0027true\u0027,"},{"line_number":200,"context_line":"                   \u0027bind_port\u0027: \u00278080\u0027,"}],"source_content_type":"text/x-python","patch_set":5,"id":"AAAAUH%2F%2F6Zk%3D","line":197,"updated":"2014-01-06 21:57:30.000000000","message":"global_conf \u0026 local_conf are duplicated TestDataMigration*. Maybe you refactor a bit here.","commit_id":"94be0c854c189f22f9f1debf0f4a022c09fc3f40"},{"author":{"_account_id":9205,"name":"Gil Vernik","email":"gilv@il.ibm.com","username":"gilv"},"change_message_id":"5c8750ba8b7931c1aa887dbab94bb58ebf67c7d3","unresolved":false,"context_lines":[{"line_number":194,"context_line":"        self.assertEquals(res.status_int, 404)"},{"line_number":195,"context_line":""},{"line_number":196,"context_line":""},{"line_number":197,"context_line":"class TestDataMigrationLibCloudAmazonS3(unittest.TestCase):"},{"line_number":198,"context_line":"    global_conf \u003d {\u0027log_name\u0027: \u0027proxy-server\u0027,"},{"line_number":199,"context_line":"                   \u0027eventlet_debug\u0027: \u0027true\u0027,"},{"line_number":200,"context_line":"                   \u0027bind_port\u0027: \u00278080\u0027,"}],"source_content_type":"text/x-python","patch_set":5,"id":"AAAAUH%2F%2F484%3D","line":197,"in_reply_to":"AAAAUH%2F%2F6Zk%3D","updated":"2014-01-07 07:34:33.000000000","message":"Done","commit_id":"94be0c854c189f22f9f1debf0f4a022c09fc3f40"},{"author":{"_account_id":6889,"name":"Fabien Boucher","email":"fboucher@redhat.com","username":"fabien-boucher"},"change_message_id":"6989cb8bfbcafb02f9c9a44fa05b2f1bed71f27d","unresolved":false,"context_lines":[{"line_number":292,"context_line":"    def setUp(self):"},{"line_number":293,"context_line":"        self.app \u003d data_migration.filter_factory(self.global_conf,"},{"line_number":294,"context_line":"                                                 **self.local_conf)(FakeApp())"},{"line_number":295,"context_line":"        self._orig_get_container_info \u003d data_migration.get_container_info"},{"line_number":296,"context_line":"        data_migration.get_container_info \u003d mock_get_container_info"},{"line_number":297,"context_line":""},{"line_number":298,"context_line":"    def tearDown(self):"}],"source_content_type":"text/x-python","patch_set":5,"id":"AAAAUH%2F%2F7bI%3D","line":295,"updated":"2014-01-06 21:57:30.000000000","message":"Is it possible to use mock.patch and the with statement to avoid this save/restore(in tearDown) ?","commit_id":"94be0c854c189f22f9f1debf0f4a022c09fc3f40"},{"author":{"_account_id":9205,"name":"Gil Vernik","email":"gilv@il.ibm.com","username":"gilv"},"change_message_id":"5c8750ba8b7931c1aa887dbab94bb58ebf67c7d3","unresolved":false,"context_lines":[{"line_number":292,"context_line":"    def setUp(self):"},{"line_number":293,"context_line":"        self.app \u003d data_migration.filter_factory(self.global_conf,"},{"line_number":294,"context_line":"                                                 **self.local_conf)(FakeApp())"},{"line_number":295,"context_line":"        self._orig_get_container_info \u003d data_migration.get_container_info"},{"line_number":296,"context_line":"        data_migration.get_container_info \u003d mock_get_container_info"},{"line_number":297,"context_line":""},{"line_number":298,"context_line":"    def tearDown(self):"}],"source_content_type":"text/x-python","patch_set":5,"id":"AAAAUH%2F%2F49E%3D","line":295,"in_reply_to":"AAAAUH%2F%2F7bI%3D","updated":"2014-01-07 07:34:33.000000000","message":"This \u0027trick\u0027 of replacing get_container_info call appears in other tests and i just used the same techniques. If you think there is some other better usage, can you please point me to other test that implements it the way you want?","commit_id":"94be0c854c189f22f9f1debf0f4a022c09fc3f40"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"65aa23baee7da7a8badb71a751918fed1c676868","unresolved":false,"context_lines":[{"line_number":76,"context_line":"                        request\u003dreq)(env, start_response)"},{"line_number":77,"context_line":""},{"line_number":78,"context_line":""},{"line_number":79,"context_line":"def verify_object(req, container_name, object_name):"},{"line_number":80,"context_line":"    newpath \u003d os.path.join(tempfile.gettempdir(), \u0027migsource\u0027)"},{"line_number":81,"context_line":"    file_path \u003d os.path.join(newpath, object_name)"},{"line_number":82,"context_line":"    f \u003d open(file_path, \u0027r+\u0027)"}],"source_content_type":"text/x-python","patch_set":22,"id":"fa98f980_dc39f240","line":79,"updated":"2014-09-16 15:13:47.000000000","message":"When does this get called? I couldn\u0027t find a path to it in tests","commit_id":"63a19a6152b5eae20641d541aa9fa1c7b9977208"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"65aa23baee7da7a8badb71a751918fed1c676868","unresolved":false,"context_lines":[{"line_number":106,"context_line":"                        \u0027X-Container-Migration-Key\u0027: \u0027testing\u0027}"},{"line_number":107,"context_line":""},{"line_number":108,"context_line":"    if (container \u003d\u003d \u0027cfs1\u0027):"},{"line_number":109,"context_line":"        container_info \u003d {\u0027meta\u0027: fsystem_migration_info.copy()}"},{"line_number":110,"context_line":"        container_info[\u0027meta\u0027][\u0027migration-source\u0027] \u003d \\"},{"line_number":111,"context_line":"            os.path.join(tempfile.gettempdir(), \u0027migsource\u0027)"},{"line_number":112,"context_line":"        container_info.setdefault(\u0027status\u0027, 200)"}],"source_content_type":"text/x-python","patch_set":22,"id":"fa98f980_7c0ac67a","line":109,"updated":"2014-09-16 15:13:47.000000000","message":"shouldn\u0027t the key be \u0027sysmeta\u0027 throughout?","commit_id":"63a19a6152b5eae20641d541aa9fa1c7b9977208"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"65aa23baee7da7a8badb71a751918fed1c676868","unresolved":false,"context_lines":[{"line_number":177,"context_line":"        req \u003d Request.blank(\u0027/v1/a/cfswrongpath/\u0027 + local_files[0],"},{"line_number":178,"context_line":"                            environ\u003d{\u0027REQUEST_METHOD\u0027: \u0027GET\u0027})"},{"line_number":179,"context_line":"        res \u003d req.get_response(self.app)"},{"line_number":180,"context_line":"        self.assertEquals(res.status_int, 404)"},{"line_number":181,"context_line":""},{"line_number":182,"context_line":""},{"line_number":183,"context_line":"class TestDataMigrationSwiftClient(unittest.TestCase):"}],"source_content_type":"text/x-python","patch_set":22,"id":"fa98f980_9cebea77","line":180,"updated":"2014-09-16 15:13:47.000000000","message":"I think we need a test_data_migration_fsystem_GET_right_path() - at the moment there is no test coverage for the GETorHEADmiss or upload_object methods","commit_id":"63a19a6152b5eae20641d541aa9fa1c7b9977208"}]}