)]}'
{"doc/source/overview_encryption.rst":[{"author":{"_account_id":12279,"name":"Janie Richling","email":"jrichli@us.ibm.com","username":"jrichli"},"change_message_id":"75b8423e746d709cf4217abbb1f6f1d53d1406d1","unresolved":false,"context_lines":[{"line_number":8,"context_line":""},{"line_number":9,"context_line":"Encryption of data at rest is implemented by a set of three middleware modules"},{"line_number":10,"context_line":"that may be included in the proxy server WSGI pipeline. The feature is internal"},{"line_number":11,"context_line":"to a Swift cluster and not not exposed through the API. Client\u0027s are unaware"},{"line_number":12,"context_line":"that data is encrypted while at rest in Swift; encrypted data should never be"},{"line_number":13,"context_line":"returned to clients via the Swift API."},{"line_number":14,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"1a122d0e_210b1226","line":11,"updated":"2016-05-03 17:41:18.000000000","message":"Technically, the BYOK push adds an optional header to the API for supplying the key.  Is it still accurate to say that this is not exposed through the API?","commit_id":"56b893a7a4ebe44db82b898106796bc9d256ba0e"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"71a49160abb770f268fc73405ddecab2c35bdfe0","unresolved":false,"context_lines":[{"line_number":8,"context_line":""},{"line_number":9,"context_line":"Encryption of data at rest is implemented by a set of three middleware modules"},{"line_number":10,"context_line":"that may be included in the proxy server WSGI pipeline. The feature is internal"},{"line_number":11,"context_line":"to a Swift cluster and not not exposed through the API. Client\u0027s are unaware"},{"line_number":12,"context_line":"that data is encrypted while at rest in Swift; encrypted data should never be"},{"line_number":13,"context_line":"returned to clients via the Swift API."},{"line_number":14,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"1a122d0e_2a8110fc","line":11,"in_reply_to":"1a122d0e_210b1226","updated":"2016-05-04 08:38:19.000000000","message":"The notes here say BYOK is for v2, so I am assuming we will not be proposing that to master initially and have not considered it in this doc.","commit_id":"56b893a7a4ebe44db82b898106796bc9d256ba0e"},{"author":{"_account_id":12279,"name":"Janie Richling","email":"jrichli@us.ibm.com","username":"jrichli"},"change_message_id":"75b8423e746d709cf4217abbb1f6f1d53d1406d1","unresolved":false,"context_lines":[{"line_number":19,"context_line":"* all custom user metadata values i.e. metadata sent using X-Object-Meta-"},{"line_number":20,"context_line":"  prefixed headers with PUT or POST requests"},{"line_number":21,"context_line":""},{"line_number":22,"context_line":"Any items data not included in the list above are not encrypted, including:"},{"line_number":23,"context_line":""},{"line_number":24,"context_line":"* account, container and object names"},{"line_number":25,"context_line":"* account and container custom user metadata"}],"source_content_type":"text/x-rst","patch_set":1,"id":"1a122d0e_41560e07","line":22,"updated":"2016-05-03 17:41:18.000000000","message":"change the \"items data\" here.","commit_id":"56b893a7a4ebe44db82b898106796bc9d256ba0e"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"71a49160abb770f268fc73405ddecab2c35bdfe0","unresolved":false,"context_lines":[{"line_number":19,"context_line":"* all custom user metadata values i.e. metadata sent using X-Object-Meta-"},{"line_number":20,"context_line":"  prefixed headers with PUT or POST requests"},{"line_number":21,"context_line":""},{"line_number":22,"context_line":"Any items data not included in the list above are not encrypted, including:"},{"line_number":23,"context_line":""},{"line_number":24,"context_line":"* account, container and object names"},{"line_number":25,"context_line":"* account and container custom user metadata"}],"source_content_type":"text/x-rst","patch_set":1,"id":"1a122d0e_4a760c40","line":22,"in_reply_to":"1a122d0e_41560e07","updated":"2016-05-04 08:38:19.000000000","message":"Done","commit_id":"56b893a7a4ebe44db82b898106796bc9d256ba0e"},{"author":{"_account_id":12279,"name":"Janie Richling","email":"jrichli@us.ibm.com","username":"jrichli"},"change_message_id":"75b8423e746d709cf4217abbb1f6f1d53d1406d1","unresolved":false,"context_lines":[{"line_number":54,"context_line":"The keymaster config option ``encryption_root_secret`` MUST be set before the"},{"line_number":55,"context_line":"middleware is used and MUST be consistent across all proxy servers. Its value"},{"line_number":56,"context_line":"(and consequently, the proxy-server.conf file) should not be stored on any disk"},{"line_number":57,"context_line":"that is in any account, container or object ring."},{"line_number":58,"context_line":""},{"line_number":59,"context_line":"Once deployed, the encrypter will encrypt object data and metadata when PUT and"},{"line_number":60,"context_line":"POST requests are made to the proxy server and the decrypter will decrypt"}],"source_content_type":"text/x-rst","patch_set":1,"id":"1a122d0e_c1a97e26","line":57,"updated":"2016-05-03 17:41:18.000000000","message":"To use encryption successfully, you also have to change the following in swift.conf:\n[swift-constraints]\nextra_header_count \u003d 372\n\nunless we change the default MAX.  See https://trello.com/c/4AyGQrHH/46-increase-the-default-max-num-headers","commit_id":"56b893a7a4ebe44db82b898106796bc9d256ba0e"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"71a49160abb770f268fc73405ddecab2c35bdfe0","unresolved":false,"context_lines":[{"line_number":54,"context_line":"The keymaster config option ``encryption_root_secret`` MUST be set before the"},{"line_number":55,"context_line":"middleware is used and MUST be consistent across all proxy servers. Its value"},{"line_number":56,"context_line":"(and consequently, the proxy-server.conf file) should not be stored on any disk"},{"line_number":57,"context_line":"that is in any account, container or object ring."},{"line_number":58,"context_line":""},{"line_number":59,"context_line":"Once deployed, the encrypter will encrypt object data and metadata when PUT and"},{"line_number":60,"context_line":"POST requests are made to the proxy server and the decrypter will decrypt"}],"source_content_type":"text/x-rst","patch_set":1,"id":"1a122d0e_0a37f44d","line":57,"in_reply_to":"1a122d0e_c1a97e26","updated":"2016-05-04 08:38:19.000000000","message":"Done","commit_id":"56b893a7a4ebe44db82b898106796bc9d256ba0e"},{"author":{"_account_id":12279,"name":"Janie Richling","email":"jrichli@us.ibm.com","username":"jrichli"},"change_message_id":"75b8423e746d709cf4217abbb1f6f1d53d1406d1","unresolved":false,"context_lines":[{"line_number":72,"context_line":"TODO - so what can an operator do if they wish to cease encrypting new objects?"},{"line_number":73,"context_line":"If they remove just the encrypter then existing encrypted objects could be"},{"line_number":74,"context_line":"read and new objects would not be encrypted, but conditional GETs on existing"},{"line_number":75,"context_line":"encrypted objects would not work"},{"line_number":76,"context_line":""},{"line_number":77,"context_line":"--------------------------"},{"line_number":78,"context_line":"Performance Considerations"}],"source_content_type":"text/x-rst","patch_set":1,"id":"1a122d0e_04dc2417","line":75,"updated":"2016-05-03 17:41:18.000000000","message":"Good catch.  The thing that immediately comes to my mind is replacing the encrypter with a new middleware - so that all it would do is handle this \"no longer encrypting\" case.  For now, that is just conditional etag handling, but there may be other things to handle as development moves on.","commit_id":"56b893a7a4ebe44db82b898106796bc9d256ba0e"},{"author":{"_account_id":20553,"name":"Brian Ober","email":"bober@us.ibm.com","username":"ober37"},"change_message_id":"49ffe994909c378eb3ce2a1f442514b90d8dd0e2","unresolved":false,"context_lines":[{"line_number":73,"context_line":"If they remove just the encrypter then existing encrypted objects could be"},{"line_number":74,"context_line":"read and new objects would not be encrypted, but conditional GETs on existing"},{"line_number":75,"context_line":"encrypted objects would not work"},{"line_number":76,"context_line":""},{"line_number":77,"context_line":"--------------------------"},{"line_number":78,"context_line":"Performance Considerations"},{"line_number":79,"context_line":"--------------------------"}],"source_content_type":"text/x-rst","patch_set":1,"id":"1a122d0e_f31c436c","line":76,"updated":"2016-05-03 20:04:47.000000000","message":"Would it make sense to add a parameter under the encrypter for this?\n\n[filter:encrypter]\nuse \u003d egg:swift#encrypter\ndisable_encryption \u003d true","commit_id":"56b893a7a4ebe44db82b898106796bc9d256ba0e"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"71a49160abb770f268fc73405ddecab2c35bdfe0","unresolved":false,"context_lines":[{"line_number":73,"context_line":"If they remove just the encrypter then existing encrypted objects could be"},{"line_number":74,"context_line":"read and new objects would not be encrypted, but conditional GETs on existing"},{"line_number":75,"context_line":"encrypted objects would not work"},{"line_number":76,"context_line":""},{"line_number":77,"context_line":"--------------------------"},{"line_number":78,"context_line":"Performance Considerations"},{"line_number":79,"context_line":"--------------------------"}],"source_content_type":"text/x-rst","patch_set":1,"id":"1a122d0e_ea10b8ad","line":76,"in_reply_to":"1a122d0e_f31c436c","updated":"2016-05-04 08:38:19.000000000","message":"yes something like this makes sense - I will leave it as a TODO in doc until we implement it though since that will serve as a reminder to actually implement it!","commit_id":"56b893a7a4ebe44db82b898106796bc9d256ba0e"},{"author":{"_account_id":12279,"name":"Janie Richling","email":"jrichli@us.ibm.com","username":"jrichli"},"change_message_id":"75b8423e746d709cf4217abbb1f6f1d53d1406d1","unresolved":false,"context_lines":[{"line_number":98,"context_line":""},{"line_number":99,"context_line":"where ``E`` is the encryption function, ``k`` is an encryption key and ``n`` is"},{"line_number":100,"context_line":"a unique nonce chosen for each encryption operation. The nonce is stored as"},{"line_number":101,"context_line":"metadata of the encrypted item so that it is available for decryption::"},{"line_number":102,"context_line":""},{"line_number":103,"context_line":"  plaintext \u003d D(ciphertext, k, n)"},{"line_number":104,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"1a122d0e_44b57c7c","line":101,"updated":"2016-05-03 17:41:18.000000000","message":"I am thinking we stick with \u0027iv\u0027 instead of nonce.  Technically speaking, one could argue that we are supplying the first iv for the first block of the encyrption context.  The reason being that the size of this value is a whole block size.  The size of just what is referred to as \"nonce\" is less than a block size.  What we supply is the first iv, being nonce plus the first counter.\n\nBut I am open to whatever communicates more effectively.","commit_id":"56b893a7a4ebe44db82b898106796bc9d256ba0e"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"71a49160abb770f268fc73405ddecab2c35bdfe0","unresolved":false,"context_lines":[{"line_number":98,"context_line":""},{"line_number":99,"context_line":"where ``E`` is the encryption function, ``k`` is an encryption key and ``n`` is"},{"line_number":100,"context_line":"a unique nonce chosen for each encryption operation. The nonce is stored as"},{"line_number":101,"context_line":"metadata of the encrypted item so that it is available for decryption::"},{"line_number":102,"context_line":""},{"line_number":103,"context_line":"  plaintext \u003d D(ciphertext, k, n)"},{"line_number":104,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"1a122d0e_0aa1b446","line":101,"in_reply_to":"1a122d0e_44b57c7c","updated":"2016-05-04 08:38:19.000000000","message":"Done","commit_id":"56b893a7a4ebe44db82b898106796bc9d256ba0e"},{"author":{"_account_id":12279,"name":"Janie Richling","email":"jrichli@us.ibm.com","username":"jrichli"},"change_message_id":"75b8423e746d709cf4217abbb1f6f1d53d1406d1","unresolved":false,"context_lines":[{"line_number":112,"context_line":"dictionary::"},{"line_number":113,"context_line":""},{"line_number":114,"context_line":"  crypto_metadata \u003d {\u0027nonce\u0027: \u003c16 byte value\u003e,"},{"line_number":115,"context_line":"                     \u0027key_id\u0027: H(k)}"},{"line_number":116,"context_line":""},{"line_number":117,"context_line":"Key management"},{"line_number":118,"context_line":"--------------"}],"source_content_type":"text/x-rst","patch_set":1,"id":"1a122d0e_a47bb0bd","line":115,"updated":"2016-05-03 17:41:18.000000000","message":"maybe note that the entity tag section will mention a deviation (since its not a unique nonce technically)","commit_id":"56b893a7a4ebe44db82b898106796bc9d256ba0e"},{"author":{"_account_id":20553,"name":"Brian Ober","email":"bober@us.ibm.com","username":"ober37"},"change_message_id":"49ffe994909c378eb3ce2a1f442514b90d8dd0e2","unresolved":false,"context_lines":[{"line_number":121,"context_line":"object key for each encryption and decryption operation. These are made"},{"line_number":122,"context_line":"available to the encrypter and decrypter via a callback function that the"},{"line_number":123,"context_line":"keymaster installs in the WSGI request environ."},{"line_number":124,"context_line":""},{"line_number":125,"context_line":"The current keymaster implementation derives container and object keys from the"},{"line_number":126,"context_line":"``encryption_root_secret`` in a deterministic way. Other strategies may be"},{"line_number":127,"context_line":"employed by future implementations of alternative keymaster middleware."}],"source_content_type":"text/x-rst","patch_set":1,"id":"1a122d0e_f3b8c32c","line":124,"updated":"2016-05-03 20:04:47.000000000","message":"Should this be expanded to cover the case where the user provides keys via API?","commit_id":"56b893a7a4ebe44db82b898106796bc9d256ba0e"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"71a49160abb770f268fc73405ddecab2c35bdfe0","unresolved":false,"context_lines":[{"line_number":121,"context_line":"object key for each encryption and decryption operation. These are made"},{"line_number":122,"context_line":"available to the encrypter and decrypter via a callback function that the"},{"line_number":123,"context_line":"keymaster installs in the WSGI request environ."},{"line_number":124,"context_line":""},{"line_number":125,"context_line":"The current keymaster implementation derives container and object keys from the"},{"line_number":126,"context_line":"``encryption_root_secret`` in a deterministic way. Other strategies may be"},{"line_number":127,"context_line":"employed by future implementations of alternative keymaster middleware."}],"source_content_type":"text/x-rst","patch_set":1,"id":"1a122d0e_0a46d47a","line":124,"in_reply_to":"1a122d0e_f3b8c32c","updated":"2016-05-04 08:38:19.000000000","message":"We\u0027re not planning to include BYOK in first release, at least that\u0027s what I took away from Austin discussions:\n\nhttps://etherpad.openstack.org/p/swift-newton-work-session-1","commit_id":"56b893a7a4ebe44db82b898106796bc9d256ba0e"},{"author":{"_account_id":20553,"name":"Brian Ober","email":"bober@us.ibm.com","username":"ober37"},"change_message_id":"49ffe994909c378eb3ce2a1f442514b90d8dd0e2","unresolved":false,"context_lines":[{"line_number":137,"context_line":"wrapping minimizes the amount of data encrypted using those keys to just other"},{"line_number":138,"context_line":"randomly chosen keys which can be re-wrapped efficiently without needing to"},{"line_number":139,"context_line":"re-encrypt the larger amounts of data that were encrypted using the random"},{"line_number":140,"context_line":"keys."},{"line_number":141,"context_line":""},{"line_number":142,"context_line":"For example, as described below, the object body is encrypted using a random"},{"line_number":143,"context_line":"key which is then wrapped using the object key. If re-keying requires the"}],"source_content_type":"text/x-rst","patch_set":1,"id":"1a122d0e_13d57f1a","line":140,"updated":"2016-05-03 20:04:47.000000000","message":"This makes sense, clever solution.    I am not entirely clear what happens when the user decides that they wish to change their key at a container level.   Is the intention that this is done only during creation and cannot be updated via POST?","commit_id":"56b893a7a4ebe44db82b898106796bc9d256ba0e"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"71a49160abb770f268fc73405ddecab2c35bdfe0","unresolved":false,"context_lines":[{"line_number":137,"context_line":"wrapping minimizes the amount of data encrypted using those keys to just other"},{"line_number":138,"context_line":"randomly chosen keys which can be re-wrapped efficiently without needing to"},{"line_number":139,"context_line":"re-encrypt the larger amounts of data that were encrypted using the random"},{"line_number":140,"context_line":"keys."},{"line_number":141,"context_line":""},{"line_number":142,"context_line":"For example, as described below, the object body is encrypted using a random"},{"line_number":143,"context_line":"key which is then wrapped using the object key. If re-keying requires the"}],"source_content_type":"text/x-rst","patch_set":1,"id":"1a122d0e_65c33b8b","line":140,"in_reply_to":"1a122d0e_13d57f1a","updated":"2016-05-04 08:38:19.000000000","message":"I think you hit on one reason why deferring BYOK makes sense - there\u0027s a number of \"what if the user does X?\" issues to work through with BYOK.\n\nThat said, even with keymaster managed keys we do not have support for re-keying here. All we are doing now is wrapping keys with a view to the future because wrapping the keys should avoid re-encryption of large amounts of data if/when someone figures out re-keying.","commit_id":"56b893a7a4ebe44db82b898106796bc9d256ba0e"},{"author":{"_account_id":12279,"name":"Janie Richling","email":"jrichli@us.ibm.com","username":"jrichli"},"change_message_id":"75b8423e746d709cf4217abbb1f6f1d53d1406d1","unresolved":false,"context_lines":[{"line_number":142,"context_line":"For example, as described below, the object body is encrypted using a random"},{"line_number":143,"context_line":"key which is then wrapped using the object key. If re-keying requires the"},{"line_number":144,"context_line":"object key to be replaced the only random key needs to be re-encrypted but not"},{"line_number":145,"context_line":"the object body, which is potentially a large amount of data."},{"line_number":146,"context_line":""},{"line_number":147,"context_line":".. note::"},{"line_number":148,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"1a122d0e_efde210f","line":145,"updated":"2016-05-03 17:41:18.000000000","message":"I added a trello card for key wrapping https://trello.com/c/Dq2dZvbP/89-change-to-use-random-object-key-that-is-wrapped-with-the-container-key\n\nBut I am concerned that once we attempt this implementation, we will re-discover the areas where we are relying upon the fact that object keys are derived and that we don\u0027t need to retrieve any object metadata in order to encrypt using the same key - such as on POST.","commit_id":"56b893a7a4ebe44db82b898106796bc9d256ba0e"},{"author":{"_account_id":12279,"name":"Janie Richling","email":"jrichli@us.ibm.com","username":"jrichli"},"change_message_id":"2a99464ecf2183fa6491f503e82f5ed517b1d9aa","unresolved":false,"context_lines":[{"line_number":142,"context_line":"For example, as described below, the object body is encrypted using a random"},{"line_number":143,"context_line":"key which is then wrapped using the object key. If re-keying requires the"},{"line_number":144,"context_line":"object key to be replaced the only random key needs to be re-encrypted but not"},{"line_number":145,"context_line":"the object body, which is potentially a large amount of data."},{"line_number":146,"context_line":""},{"line_number":147,"context_line":".. note::"},{"line_number":148,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"dab17558_06d42345","line":145,"in_reply_to":"1a122d0e_a519d324","updated":"2016-05-08 22:00:29.000000000","message":"Ah, I see.  The metadata is not using the wrapped key.  But if it were, then we have to access some stored wrapped key on a POST - unless we use a new key to encrypt just the metadata on the POST.  This would be an argument for using a different key for metadata versus body.  Of course, if someday we can update individual metadata, then we\u0027d want to add the metadata key to the crypto-meta for that item.","commit_id":"56b893a7a4ebe44db82b898106796bc9d256ba0e"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"71a49160abb770f268fc73405ddecab2c35bdfe0","unresolved":false,"context_lines":[{"line_number":142,"context_line":"For example, as described below, the object body is encrypted using a random"},{"line_number":143,"context_line":"key which is then wrapped using the object key. If re-keying requires the"},{"line_number":144,"context_line":"object key to be replaced the only random key needs to be re-encrypted but not"},{"line_number":145,"context_line":"the object body, which is potentially a large amount of data."},{"line_number":146,"context_line":""},{"line_number":147,"context_line":".. note::"},{"line_number":148,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"1a122d0e_a519d324","line":145,"in_reply_to":"1a122d0e_efde210f","updated":"2016-05-04 08:38:19.000000000","message":"I noted below that we are not currently proposing to use a wrapped key for metadata (which is all that changes on a POST). However, I think we could. As long as the wrapped key is always available when you would want to decrypt the item that it encrypts, then its juts a matter of indirection - the derived keys gets you the random key which gets you the data.","commit_id":"56b893a7a4ebe44db82b898106796bc9d256ba0e"},{"author":{"_account_id":12279,"name":"Janie Richling","email":"jrichli@us.ibm.com","username":"jrichli"},"change_message_id":"75b8423e746d709cf4217abbb1f6f1d53d1406d1","unresolved":false,"context_lines":[{"line_number":179,"context_line":"++++++++++++++++++++"},{"line_number":180,"context_line":""},{"line_number":181,"context_line":"TODO - metadata is now the only place where the object key is used to directly"},{"line_number":182,"context_line":"encrypt user data, rather than a wrapped randon key. That is ok because"},{"line_number":183,"context_line":"re-keying metadata is not expensive, but we could equally choose a random key"},{"line_number":184,"context_line":"for metadata."},{"line_number":185,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"1a122d0e_6f39713f","line":182,"updated":"2016-05-03 17:41:18.000000000","message":"s/randon/random/","commit_id":"56b893a7a4ebe44db82b898106796bc9d256ba0e"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"71a49160abb770f268fc73405ddecab2c35bdfe0","unresolved":false,"context_lines":[{"line_number":179,"context_line":"++++++++++++++++++++"},{"line_number":180,"context_line":""},{"line_number":181,"context_line":"TODO - metadata is now the only place where the object key is used to directly"},{"line_number":182,"context_line":"encrypt user data, rather than a wrapped randon key. That is ok because"},{"line_number":183,"context_line":"re-keying metadata is not expensive, but we could equally choose a random key"},{"line_number":184,"context_line":"for metadata."},{"line_number":185,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"1a122d0e_851497e9","line":182,"in_reply_to":"1a122d0e_6f39713f","updated":"2016-05-04 08:38:19.000000000","message":"Done","commit_id":"56b893a7a4ebe44db82b898106796bc9d256ba0e"},{"author":{"_account_id":12279,"name":"Janie Richling","email":"jrichli@us.ibm.com","username":"jrichli"},"change_message_id":"75b8423e746d709cf4217abbb1f6f1d53d1406d1","unresolved":false,"context_lines":[{"line_number":224,"context_line":"While encrypting the object body the encrypter also calculates the hexdigest of"},{"line_number":225,"context_line":"the md5 sum of the plaintext body. This value is encrypted using a container"},{"line_number":226,"context_line":"listing key that has been read from the container metadata, and a nonce that is"},{"line_number":227,"context_line":"derived from the object\u0027s path, and saved as an item of system metadata::"},{"line_number":228,"context_line":""},{"line_number":229,"context_line":"  X-Object-Sysmeta-Crypto-Etag: E(md5(plaintext), listing_key, F(path))"},{"line_number":230,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"1a122d0e_50fd326d","line":227,"updated":"2016-05-03 17:41:18.000000000","message":"I added a trello card for changing to only use container key on all etag values. https://trello.com/c/SqcHL7YZ/90-change-the-etag-currently-encrypted-with-object-key-to-instead-use-a-listing-container-key","commit_id":"56b893a7a4ebe44db82b898106796bc9d256ba0e"},{"author":{"_account_id":12279,"name":"Janie Richling","email":"jrichli@us.ibm.com","username":"jrichli"},"change_message_id":"75b8423e746d709cf4217abbb1f6f1d53d1406d1","unresolved":false,"context_lines":[{"line_number":230,"context_line":""},{"line_number":231,"context_line":"The crypto metadata for this transformation comprises only the listing key"},{"line_number":232,"context_line":"identifier because the nonce can be determined from the object path. This"},{"line_number":233,"context_line":"crypto metadata is stored as an item of system metadata::"},{"line_number":234,"context_line":""},{"line_number":235,"context_line":"  X-Object-Sysmeta-Crypto-Meta-Etag: {\u0027key_id\u0027: H(listing_key)}"},{"line_number":236,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"1a122d0e_d092e2f5","line":233,"updated":"2016-05-03 17:41:18.000000000","message":"Could we still record the nonce used in the crypto_meta?  I think that is more future proof and makes things more consistent.  It would think it doesn\u0027t hurt to always record the exact nonce used when encrypting.","commit_id":"56b893a7a4ebe44db82b898106796bc9d256ba0e"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"71a49160abb770f268fc73405ddecab2c35bdfe0","unresolved":false,"context_lines":[{"line_number":230,"context_line":""},{"line_number":231,"context_line":"The crypto metadata for this transformation comprises only the listing key"},{"line_number":232,"context_line":"identifier because the nonce can be determined from the object path. This"},{"line_number":233,"context_line":"crypto metadata is stored as an item of system metadata::"},{"line_number":234,"context_line":""},{"line_number":235,"context_line":"  X-Object-Sysmeta-Crypto-Meta-Etag: {\u0027key_id\u0027: H(listing_key)}"},{"line_number":236,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"1a122d0e_65c97b5a","line":233,"in_reply_to":"1a122d0e_d092e2f5","updated":"2016-05-04 08:38:19.000000000","message":"OK.\n\nI wanted to be efficient with what we need to append as metadata to the etag value that gets stored in the container db but there\u0027s no prob having it duplicated in the object metadata.","commit_id":"56b893a7a4ebe44db82b898106796bc9d256ba0e"},{"author":{"_account_id":12279,"name":"Janie Richling","email":"jrichli@us.ibm.com","username":"jrichli"},"change_message_id":"75b8423e746d709cf4217abbb1f6f1d53d1406d1","unresolved":false,"context_lines":[{"line_number":247,"context_line":"the associated crypto metadata appended to the encrypted etag value::"},{"line_number":248,"context_line":""},{"line_number":249,"context_line":"  X-Object-Sysmeta-Container-Update-Override-Etag:"},{"line_number":250,"context_line":"      E(md5(plaintext), container_key, F(path)); meta\u003d{\u0027key_id\u0027: H(listing_key)}"},{"line_number":251,"context_line":""},{"line_number":252,"context_line":"The reason a nonce derived from the object\u0027s path is used when encrypting the"},{"line_number":253,"context_line":"etag is to allow the encrypter to perform the same transformation on entity tag"}],"source_content_type":"text/x-rst","patch_set":1,"id":"1a122d0e_d02e62a4","line":250,"updated":"2016-05-03 17:41:18.000000000","message":"should we explain the rationale why we could not use the backend values - and had to go with sysmeta instead?","commit_id":"56b893a7a4ebe44db82b898106796bc9d256ba0e"},{"author":{"_account_id":12279,"name":"Janie Richling","email":"jrichli@us.ibm.com","username":"jrichli"},"change_message_id":"2a99464ecf2183fa6491f503e82f5ed517b1d9aa","unresolved":false,"context_lines":[{"line_number":247,"context_line":"the associated crypto metadata appended to the encrypted etag value::"},{"line_number":248,"context_line":""},{"line_number":249,"context_line":"  X-Object-Sysmeta-Container-Update-Override-Etag:"},{"line_number":250,"context_line":"      E(md5(plaintext), container_key, F(path)); meta\u003d{\u0027key_id\u0027: H(listing_key)}"},{"line_number":251,"context_line":""},{"line_number":252,"context_line":"The reason a nonce derived from the object\u0027s path is used when encrypting the"},{"line_number":253,"context_line":"etag is to allow the encrypter to perform the same transformation on entity tag"}],"source_content_type":"text/x-rst","patch_set":1,"id":"dab17558_46ce2bb5","line":250,"in_reply_to":"1a122d0e_25d023c9","updated":"2016-05-08 22:00:29.000000000","message":"gotcha.","commit_id":"56b893a7a4ebe44db82b898106796bc9d256ba0e"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"71a49160abb770f268fc73405ddecab2c35bdfe0","unresolved":false,"context_lines":[{"line_number":247,"context_line":"the associated crypto metadata appended to the encrypted etag value::"},{"line_number":248,"context_line":""},{"line_number":249,"context_line":"  X-Object-Sysmeta-Container-Update-Override-Etag:"},{"line_number":250,"context_line":"      E(md5(plaintext), container_key, F(path)); meta\u003d{\u0027key_id\u0027: H(listing_key)}"},{"line_number":251,"context_line":""},{"line_number":252,"context_line":"The reason a nonce derived from the object\u0027s path is used when encrypting the"},{"line_number":253,"context_line":"etag is to allow the encrypter to perform the same transformation on entity tag"}],"source_content_type":"text/x-rst","patch_set":1,"id":"1a122d0e_25d023c9","line":250,"in_reply_to":"1a122d0e_d02e62a4","updated":"2016-05-04 08:38:19.000000000","message":"I see that as a change to the proxy/object server protocol that was prompted by crypto but is not a feature of crypto. With the benefit of hindsight it should have been done that way originally.\n\nSo while it is useful to have rationale for the crypto related design in this doc, I would prefer not to creep into justifying other changes in swift core code. Likewise I do not propose to discuss transient sysmeta in this doc.","commit_id":"56b893a7a4ebe44db82b898106796bc9d256ba0e"},{"author":{"_account_id":12279,"name":"Janie Richling","email":"jrichli@us.ibm.com","username":"jrichli"},"change_message_id":"75b8423e746d709cf4217abbb1f6f1d53d1406d1","unresolved":false,"context_lines":[{"line_number":287,"context_line":"#. inspects the container metadata for any"},{"line_number":288,"context_line":"   ``X-Container-Sysmeta-Crypto-Listing-Key-*`` headers"},{"line_number":289,"context_line":"#. if found decrypt those wrapped listing keys using the container key obtained"},{"line_number":290,"context_line":"   from the keymaster"},{"line_number":291,"context_line":"#. decrypt the response body etag entries using the listing key(s)"},{"line_number":292,"context_line":""},{"line_number":293,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"1a122d0e_a4672be9","line":290,"updated":"2016-05-03 17:41:18.000000000","message":"I want to be careful here about who/ what is in control of whether or not to decrypt.  The keymaster override flag will still be consulted over all, right?  The keymaster will be the one to look for crypto-listing-key in order to determine whether or not to decrypt, right?","commit_id":"56b893a7a4ebe44db82b898106796bc9d256ba0e"},{"author":{"_account_id":12279,"name":"Janie Richling","email":"jrichli@us.ibm.com","username":"jrichli"},"change_message_id":"2a99464ecf2183fa6491f503e82f5ed517b1d9aa","unresolved":false,"context_lines":[{"line_number":287,"context_line":"#. inspects the container metadata for any"},{"line_number":288,"context_line":"   ``X-Container-Sysmeta-Crypto-Listing-Key-*`` headers"},{"line_number":289,"context_line":"#. if found decrypt those wrapped listing keys using the container key obtained"},{"line_number":290,"context_line":"   from the keymaster"},{"line_number":291,"context_line":"#. decrypt the response body etag entries using the listing key(s)"},{"line_number":292,"context_line":""},{"line_number":293,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"dab17558_c6ba1b0a","line":290,"in_reply_to":"1a122d0e_053d27dd","updated":"2016-05-08 22:00:29.000000000","message":"I see.","commit_id":"56b893a7a4ebe44db82b898106796bc9d256ba0e"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"71a49160abb770f268fc73405ddecab2c35bdfe0","unresolved":false,"context_lines":[{"line_number":287,"context_line":"#. inspects the container metadata for any"},{"line_number":288,"context_line":"   ``X-Container-Sysmeta-Crypto-Listing-Key-*`` headers"},{"line_number":289,"context_line":"#. if found decrypt those wrapped listing keys using the container key obtained"},{"line_number":290,"context_line":"   from the keymaster"},{"line_number":291,"context_line":"#. decrypt the response body etag entries using the listing key(s)"},{"line_number":292,"context_line":""},{"line_number":293,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"1a122d0e_053d27dd","line":290,"in_reply_to":"1a122d0e_a4672be9","updated":"2016-05-04 08:38:19.000000000","message":"Good point that I have neglected to mention the override.\n\nHowever, I would argue that the listing key is not a concern of the keymaster - it is generated by the encrypter. Before we had the listing key, how would the keymaster have set the override flag?\n\nIn my thinking, the keymaster is managing the derived keys that the encrypter chooses to use to encrypt either other random keys or metadata values. The keymaster does not need to know that we happen to be wrapping keys. At least, not in this version.\n\nI accept that implementation may challenge my thinking, which is why the doc is WIP.","commit_id":"56b893a7a4ebe44db82b898106796bc9d256ba0e"},{"author":{"_account_id":12279,"name":"Janie Richling","email":"jrichli@us.ibm.com","username":"jrichli"},"change_message_id":"75b8423e746d709cf4217abbb1f6f1d53d1406d1","unresolved":false,"context_lines":[{"line_number":312,"context_line":"-----------------------------"},{"line_number":313,"context_line":""},{"line_number":314,"context_line":"TODO - discuss implications of moving objects from one location to another"},{"line_number":315,"context_line":"without the decrypter/encrypter being involved. key_ids should ensure correct"},{"line_number":316,"context_line":"keys are used. what about the etag nonce which is assumed to be the path??"}],"source_content_type":"text/x-rst","patch_set":1,"id":"1a122d0e_4a08e328","line":315,"updated":"2016-05-03 17:41:18.000000000","message":"technically, i guess we called it \"crypto-id\" in the keymaster.","commit_id":"56b893a7a4ebe44db82b898106796bc9d256ba0e"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"71a49160abb770f268fc73405ddecab2c35bdfe0","unresolved":false,"context_lines":[{"line_number":312,"context_line":"-----------------------------"},{"line_number":313,"context_line":""},{"line_number":314,"context_line":"TODO - discuss implications of moving objects from one location to another"},{"line_number":315,"context_line":"without the decrypter/encrypter being involved. key_ids should ensure correct"},{"line_number":316,"context_line":"keys are used. what about the etag nonce which is assumed to be the path??"}],"source_content_type":"text/x-rst","patch_set":1,"id":"1a122d0e_c09cbdc1","line":315,"in_reply_to":"1a122d0e_4a08e328","updated":"2016-05-04 08:38:19.000000000","message":"You\u0027re right!\n\nBut I think we could embed this in crypto meta now rather than having a separate header??","commit_id":"56b893a7a4ebe44db82b898106796bc9d256ba0e"},{"author":{"_account_id":12279,"name":"Janie Richling","email":"jrichli@us.ibm.com","username":"jrichli"},"change_message_id":"2a99464ecf2183fa6491f503e82f5ed517b1d9aa","unresolved":false,"context_lines":[{"line_number":312,"context_line":"-----------------------------"},{"line_number":313,"context_line":""},{"line_number":314,"context_line":"TODO - discuss implications of moving objects from one location to another"},{"line_number":315,"context_line":"without the decrypter/encrypter being involved. key_ids should ensure correct"},{"line_number":316,"context_line":"keys are used. what about the etag nonce which is assumed to be the path??"}],"source_content_type":"text/x-rst","patch_set":1,"id":"dab17558_a64c3743","line":315,"in_reply_to":"1a122d0e_c09cbdc1","updated":"2016-05-08 22:00:29.000000000","message":"sure, that makes sense.  I added https://trello.com/c/IWbTC2nj/94-add-crypto-id-to-the-object-body-and-metadata-crypto-meta-instead-of-adding-a-new-header","commit_id":"56b893a7a4ebe44db82b898106796bc9d256ba0e"},{"author":{"_account_id":12279,"name":"Janie Richling","email":"jrichli@us.ibm.com","username":"jrichli"},"change_message_id":"75b8423e746d709cf4217abbb1f6f1d53d1406d1","unresolved":false,"context_lines":[{"line_number":313,"context_line":""},{"line_number":314,"context_line":"TODO - discuss implications of moving objects from one location to another"},{"line_number":315,"context_line":"without the decrypter/encrypter being involved. key_ids should ensure correct"},{"line_number":316,"context_line":"keys are used. what about the etag nonce which is assumed to be the path??"}],"source_content_type":"text/x-rst","patch_set":1,"id":"1a122d0e_4a6d83cb","line":316,"updated":"2016-05-03 17:41:18.000000000","message":"Again, good catch.  we really need a test to expose the case where a path changes and we must rely on the crypto-id.  I suspect we need to update the etag handling to use some method on the crypto utils to make use of the crypto-id to add yet another possibly etag value to check (we do get a list of possible tags, so that is good).","commit_id":"56b893a7a4ebe44db82b898106796bc9d256ba0e"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"71a49160abb770f268fc73405ddecab2c35bdfe0","unresolved":false,"context_lines":[{"line_number":313,"context_line":""},{"line_number":314,"context_line":"TODO - discuss implications of moving objects from one location to another"},{"line_number":315,"context_line":"without the decrypter/encrypter being involved. key_ids should ensure correct"},{"line_number":316,"context_line":"keys are used. what about the etag nonce which is assumed to be the path??"}],"source_content_type":"text/x-rst","patch_set":1,"id":"1a122d0e_604a0929","line":316,"in_reply_to":"1a122d0e_4a6d83cb","updated":"2016-05-04 08:38:19.000000000","message":"I haven\u0027t yet thought of a case where an object moves without the encrypter/decrypter being involved. My intention was to point out to devs that if that was ever to be implemented then some careful thought is needed.","commit_id":"56b893a7a4ebe44db82b898106796bc9d256ba0e"},{"author":{"_account_id":12279,"name":"Janie Richling","email":"jrichli@us.ibm.com","username":"jrichli"},"change_message_id":"2a99464ecf2183fa6491f503e82f5ed517b1d9aa","unresolved":false,"context_lines":[{"line_number":313,"context_line":""},{"line_number":314,"context_line":"TODO - discuss implications of moving objects from one location to another"},{"line_number":315,"context_line":"without the decrypter/encrypter being involved. key_ids should ensure correct"},{"line_number":316,"context_line":"keys are used. what about the etag nonce which is assumed to be the path??"}],"source_content_type":"text/x-rst","patch_set":1,"id":"dab17558_6659affe","line":316,"in_reply_to":"1a122d0e_604a0929","updated":"2016-05-08 22:00:29.000000000","message":"yes, the encrypter and decrypter will still be involved for auto-tiering because they will use internal client and that will have crypto middleware: but, we dont want to re-encrypt.  So, the original path would still be needed.  And I had said we could make use of the crypto-id, but we could not - since on the conditional get, we wouldn\u0027t have access to the saved crypto-id.\n\nSo, I still see this as an issue for auto-tiering conditional GETs.  For the first move, it might be ok, because we could put the \"from path\" information on the move request.  but for subsequent moves, the original path that was used for encryption might be lost.  I guess we would need to save all paths that had been used for an object.","commit_id":"56b893a7a4ebe44db82b898106796bc9d256ba0e"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"bf1ecdf42e40c9e97fca416c4fe55436996fc956","unresolved":false,"context_lines":[{"line_number":313,"context_line":""},{"line_number":314,"context_line":"TODO - discuss implications of moving objects from one location to another"},{"line_number":315,"context_line":"without the decrypter/encrypter being involved. key_ids should ensure correct"},{"line_number":316,"context_line":"keys are used. what about the etag nonce which is assumed to be the path??"}],"source_content_type":"text/x-rst","patch_set":1,"id":"dab17558_82273fb5","line":316,"in_reply_to":"dab17558_6659affe","updated":"2016-05-09 17:41:51.000000000","message":"Yep, conditional GETs is the show stopper if objects move. However, if/when we have tiering with symlinks then the answer may be for symlink resolution to occur to right of encryption so that by the if(none)match values are correct for the symlink and then get forwarded to the target whose etags are encrypted using the symlink path. So all works out ok.","commit_id":"56b893a7a4ebe44db82b898106796bc9d256ba0e"},{"author":{"_account_id":330,"name":"John Dickinson","email":"me@not.mn","username":"notmyname"},"change_message_id":"dc837be60e94a2c3457000773426138ae9e9f3e3","unresolved":false,"context_lines":[{"line_number":4,"context_line":""},{"line_number":5,"context_line":"Swift supports the optional encryption of object data at rest on storage nodes."},{"line_number":6,"context_line":"The encryption of object data is intended to mitigate the risk of users\u0027 data"},{"line_number":7,"context_line":"being read if an unauthorised party was to gain physical access to a disk."},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"Encryption of data at rest is implemented by a set of three middleware modules"},{"line_number":10,"context_line":"that may be included in the proxy server WSGI pipeline. The feature is internal"}],"source_content_type":"text/x-rst","patch_set":3,"id":"dab17558_ebb7c482","line":7,"updated":"2016-05-16 21:19:29.000000000","message":"grammar nit: s/was/were/","commit_id":"7d1e4d793070a8aa5b2c920efd8460d1142c83a4"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"66b86565b9beb836576ce441b455d96a7f56cea4","unresolved":false,"context_lines":[{"line_number":4,"context_line":""},{"line_number":5,"context_line":"Swift supports the optional encryption of object data at rest on storage nodes."},{"line_number":6,"context_line":"The encryption of object data is intended to mitigate the risk of users\u0027 data"},{"line_number":7,"context_line":"being read if an unauthorised party was to gain physical access to a disk."},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"Encryption of data at rest is implemented by a set of three middleware modules"},{"line_number":10,"context_line":"that may be included in the proxy server WSGI pipeline. The feature is internal"}],"source_content_type":"text/x-rst","patch_set":3,"id":"bab6814e_a690f796","line":7,"in_reply_to":"dab17558_ebb7c482","updated":"2016-05-18 12:57:28.000000000","message":"TIL https://www.grammarly.com/handbook/sentences/conditional-sentences/6/conditional-sentences-was-instead-of-were-in-the-u/","commit_id":"7d1e4d793070a8aa5b2c920efd8460d1142c83a4"},{"author":{"_account_id":330,"name":"John Dickinson","email":"me@not.mn","username":"notmyname"},"change_message_id":"dc837be60e94a2c3457000773426138ae9e9f3e3","unresolved":false,"context_lines":[{"line_number":7,"context_line":"being read if an unauthorised party was to gain physical access to a disk."},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"Encryption of data at rest is implemented by a set of three middleware modules"},{"line_number":10,"context_line":"that may be included in the proxy server WSGI pipeline. The feature is internal"},{"line_number":11,"context_line":"to a Swift cluster and not not exposed through the API. Client\u0027s are unaware"},{"line_number":12,"context_line":"that data is encrypted while at rest in Swift; encrypted data should never be"},{"line_number":13,"context_line":"returned to clients via the Swift API."}],"source_content_type":"text/x-rst","patch_set":3,"id":"dab17558_5a9cd70d","line":10,"updated":"2016-05-16 21:19:29.000000000","message":"passive to active:\n\nThree middleware modules in the proxy server WSGI pipeline implement data-at-rest encryption.","commit_id":"7d1e4d793070a8aa5b2c920efd8460d1142c83a4"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"66b86565b9beb836576ce441b455d96a7f56cea4","unresolved":false,"context_lines":[{"line_number":7,"context_line":"being read if an unauthorised party was to gain physical access to a disk."},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"Encryption of data at rest is implemented by a set of three middleware modules"},{"line_number":10,"context_line":"that may be included in the proxy server WSGI pipeline. The feature is internal"},{"line_number":11,"context_line":"to a Swift cluster and not not exposed through the API. Client\u0027s are unaware"},{"line_number":12,"context_line":"that data is encrypted while at rest in Swift; encrypted data should never be"},{"line_number":13,"context_line":"returned to clients via the Swift API."}],"source_content_type":"text/x-rst","patch_set":3,"id":"bab6814e_e1b949ee","line":10,"in_reply_to":"dab17558_5a9cd70d","updated":"2016-05-18 12:57:28.000000000","message":"I prefer the passive form in the context of this introduction because it keeps the emphasis on what is being achieved (encryption) rather than how (middleware). It would also be rather clunky to say more precisely:\n\n\"Three middleware modules that may be included in the proxy server WSGI pipeline combine to implement data-at-rest encryption.\"\n\nI don\u0027t want to imply that the middleware is there in the pipeline by default.","commit_id":"7d1e4d793070a8aa5b2c920efd8460d1142c83a4"},{"author":{"_account_id":21231,"name":"Bryan Keller","email":"kellerbr@us.ibm.com","username":"bkeller"},"change_message_id":"33ddef5d6e748e24522b9fe2051cc84cee35167f","unresolved":false,"context_lines":[{"line_number":8,"context_line":""},{"line_number":9,"context_line":"Encryption of data at rest is implemented by a set of three middleware modules"},{"line_number":10,"context_line":"that may be included in the proxy server WSGI pipeline. The feature is internal"},{"line_number":11,"context_line":"to a Swift cluster and not not exposed through the API. Client\u0027s are unaware"},{"line_number":12,"context_line":"that data is encrypted while at rest in Swift; encrypted data should never be"},{"line_number":13,"context_line":"returned to clients via the Swift API."},{"line_number":14,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"1a122d0e_b7b3b209","line":11,"updated":"2016-05-04 15:35:08.000000000","message":"(nits) \"not not\" -\u003e \"not\", \"Client\u0027s\" -\u003e \"Clients\"","commit_id":"7d1e4d793070a8aa5b2c920efd8460d1142c83a4"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"66b86565b9beb836576ce441b455d96a7f56cea4","unresolved":false,"context_lines":[{"line_number":8,"context_line":""},{"line_number":9,"context_line":"Encryption of data at rest is implemented by a set of three middleware modules"},{"line_number":10,"context_line":"that may be included in the proxy server WSGI pipeline. The feature is internal"},{"line_number":11,"context_line":"to a Swift cluster and not not exposed through the API. Client\u0027s are unaware"},{"line_number":12,"context_line":"that data is encrypted while at rest in Swift; encrypted data should never be"},{"line_number":13,"context_line":"returned to clients via the Swift API."},{"line_number":14,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"bab6814e_061463f3","line":11,"in_reply_to":"1a122d0e_b7b3b209","updated":"2016-05-18 12:57:28.000000000","message":"Done","commit_id":"7d1e4d793070a8aa5b2c920efd8460d1142c83a4"},{"author":{"_account_id":330,"name":"John Dickinson","email":"me@not.mn","username":"notmyname"},"change_message_id":"dc837be60e94a2c3457000773426138ae9e9f3e3","unresolved":false,"context_lines":[{"line_number":10,"context_line":"that may be included in the proxy server WSGI pipeline. The feature is internal"},{"line_number":11,"context_line":"to a Swift cluster and not not exposed through the API. Client\u0027s are unaware"},{"line_number":12,"context_line":"that data is encrypted while at rest in Swift; encrypted data should never be"},{"line_number":13,"context_line":"returned to clients via the Swift API."},{"line_number":14,"context_line":""},{"line_number":15,"context_line":"The following items of data are encrypted while at rest in Swift:"},{"line_number":16,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"dab17558_c5458843","line":13,"updated":"2016-05-16 21:19:29.000000000","message":"footnote?: Yes, if the client encrypts the data before storing it in Swift, Swift will return the encrypted data back to the client.","commit_id":"7d1e4d793070a8aa5b2c920efd8460d1142c83a4"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"66b86565b9beb836576ce441b455d96a7f56cea4","unresolved":false,"context_lines":[{"line_number":10,"context_line":"that may be included in the proxy server WSGI pipeline. The feature is internal"},{"line_number":11,"context_line":"to a Swift cluster and not not exposed through the API. Client\u0027s are unaware"},{"line_number":12,"context_line":"that data is encrypted while at rest in Swift; encrypted data should never be"},{"line_number":13,"context_line":"returned to clients via the Swift API."},{"line_number":14,"context_line":""},{"line_number":15,"context_line":"The following items of data are encrypted while at rest in Swift:"},{"line_number":16,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"bab6814e_bccbe61d","line":13,"in_reply_to":"dab17558_c5458843","updated":"2016-05-18 12:57:28.000000000","message":"I see your point. Swift would return decrypted encrypted data to the client :P I\u0027ll clarify the encrypted data to mean internally encrypted data.","commit_id":"7d1e4d793070a8aa5b2c920efd8460d1142c83a4"},{"author":{"_account_id":9625,"name":"Thiago da Silva","email":"thiagodasilva@gmail.com","username":"thiago"},"change_message_id":"7e01aeab531a2ea568395805cc17176c612101ff","unresolved":false,"context_lines":[{"line_number":12,"context_line":"that data is encrypted while at rest in Swift; encrypted data should never be"},{"line_number":13,"context_line":"returned to clients via the Swift API."},{"line_number":14,"context_line":""},{"line_number":15,"context_line":"The following items of data are encrypted while at rest in Swift:"},{"line_number":16,"context_line":""},{"line_number":17,"context_line":"* object data i.e. the content of an object PUT request\u0027s body"},{"line_number":18,"context_line":"* the Etag of the object"}],"source_content_type":"text/x-rst","patch_set":3,"id":"dab17558_03231ea8","line":15,"updated":"2016-05-10 16:09:55.000000000","message":"just wondering why we refer to \u0027items of data\u0027 as opposed to just \u0027object data\u0027 and \u0027object metadata\u0027","commit_id":"7d1e4d793070a8aa5b2c920efd8460d1142c83a4"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"66b86565b9beb836576ce441b455d96a7f56cea4","unresolved":false,"context_lines":[{"line_number":12,"context_line":"that data is encrypted while at rest in Swift; encrypted data should never be"},{"line_number":13,"context_line":"returned to clients via the Swift API."},{"line_number":14,"context_line":""},{"line_number":15,"context_line":"The following items of data are encrypted while at rest in Swift:"},{"line_number":16,"context_line":""},{"line_number":17,"context_line":"* object data i.e. the content of an object PUT request\u0027s body"},{"line_number":18,"context_line":"* the Etag of the object"}],"source_content_type":"text/x-rst","patch_set":3,"id":"bab6814e_31827801","line":15,"in_reply_to":"dab17558_03231ea8","updated":"2016-05-18 12:57:28.000000000","message":"Done","commit_id":"7d1e4d793070a8aa5b2c920efd8460d1142c83a4"},{"author":{"_account_id":21231,"name":"Bryan Keller","email":"kellerbr@us.ibm.com","username":"bkeller"},"change_message_id":"33ddef5d6e748e24522b9fe2051cc84cee35167f","unresolved":false,"context_lines":[{"line_number":14,"context_line":""},{"line_number":15,"context_line":"The following items of data are encrypted while at rest in Swift:"},{"line_number":16,"context_line":""},{"line_number":17,"context_line":"* object data i.e. the content of an object PUT request\u0027s body"},{"line_number":18,"context_line":"* the Etag of the object"},{"line_number":19,"context_line":"* all custom user metadata values i.e. metadata sent using X-Object-Meta-"},{"line_number":20,"context_line":"  prefixed headers with PUT or POST requests"}],"source_content_type":"text/x-rst","patch_set":3,"id":"1a122d0e_57333e13","line":17,"updated":"2016-05-04 15:35:08.000000000","message":"Should probably capitalize list items","commit_id":"7d1e4d793070a8aa5b2c920efd8460d1142c83a4"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"66b86565b9beb836576ce441b455d96a7f56cea4","unresolved":false,"context_lines":[{"line_number":14,"context_line":""},{"line_number":15,"context_line":"The following items of data are encrypted while at rest in Swift:"},{"line_number":16,"context_line":""},{"line_number":17,"context_line":"* object data i.e. the content of an object PUT request\u0027s body"},{"line_number":18,"context_line":"* the Etag of the object"},{"line_number":19,"context_line":"* all custom user metadata values i.e. metadata sent using X-Object-Meta-"},{"line_number":20,"context_line":"  prefixed headers with PUT or POST requests"}],"source_content_type":"text/x-rst","patch_set":3,"id":"bab6814e_62e4380d","line":17,"in_reply_to":"1a122d0e_57333e13","updated":"2016-05-18 12:57:28.000000000","message":"Done","commit_id":"7d1e4d793070a8aa5b2c920efd8460d1142c83a4"},{"author":{"_account_id":21231,"name":"Bryan Keller","email":"kellerbr@us.ibm.com","username":"bkeller"},"change_message_id":"bb7a46269004e660f9e0ce65f8daa9ab697b3389","unresolved":false,"context_lines":[{"line_number":15,"context_line":"The following items of data are encrypted while at rest in Swift:"},{"line_number":16,"context_line":""},{"line_number":17,"context_line":"* object data i.e. the content of an object PUT request\u0027s body"},{"line_number":18,"context_line":"* the Etag of the object"},{"line_number":19,"context_line":"* all custom user metadata values i.e. metadata sent using X-Object-Meta-"},{"line_number":20,"context_line":"  prefixed headers with PUT or POST requests"},{"line_number":21,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"1a122d0e_85d3e0d4","line":18,"updated":"2016-05-04 15:36:08.000000000","message":"Etag is lowercase in the rest of the document","commit_id":"7d1e4d793070a8aa5b2c920efd8460d1142c83a4"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"66b86565b9beb836576ce441b455d96a7f56cea4","unresolved":false,"context_lines":[{"line_number":15,"context_line":"The following items of data are encrypted while at rest in Swift:"},{"line_number":16,"context_line":""},{"line_number":17,"context_line":"* object data i.e. the content of an object PUT request\u0027s body"},{"line_number":18,"context_line":"* the Etag of the object"},{"line_number":19,"context_line":"* all custom user metadata values i.e. metadata sent using X-Object-Meta-"},{"line_number":20,"context_line":"  prefixed headers with PUT or POST requests"},{"line_number":21,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"bab6814e_22e23004","line":18,"in_reply_to":"1a122d0e_85d3e0d4","updated":"2016-05-18 12:57:28.000000000","message":"replaced with \"ETag\" throughout since that is the term used in HTTP RFC, apart from using \"entity tag\" here and defining ETag and when used in a heading.","commit_id":"7d1e4d793070a8aa5b2c920efd8460d1142c83a4"},{"author":{"_account_id":330,"name":"John Dickinson","email":"me@not.mn","username":"notmyname"},"change_message_id":"dc837be60e94a2c3457000773426138ae9e9f3e3","unresolved":false,"context_lines":[{"line_number":50,"context_line":"  # disable_encryption \u003d False"},{"line_number":51,"context_line":""},{"line_number":52,"context_line":"For further details on the positioning of those middlewares relative to other"},{"line_number":53,"context_line":"middleware please see the example pipeline in `proxy-server.conf-sample`."},{"line_number":54,"context_line":""},{"line_number":55,"context_line":"The keymaster config option ``encryption_root_secret`` MUST be set before the"},{"line_number":56,"context_line":"middleware is used and MUST be consistent across all proxy servers. Its value"}],"source_content_type":"text/x-rst","patch_set":3,"id":"dab17558_653e5c8c","line":53,"updated":"2016-05-16 21:19:29.000000000","message":"\"See the example pipeline ... for further details on the positioning...\" (ie swap the clauses)","commit_id":"7d1e4d793070a8aa5b2c920efd8460d1142c83a4"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"66b86565b9beb836576ce441b455d96a7f56cea4","unresolved":false,"context_lines":[{"line_number":50,"context_line":"  # disable_encryption \u003d False"},{"line_number":51,"context_line":""},{"line_number":52,"context_line":"For further details on the positioning of those middlewares relative to other"},{"line_number":53,"context_line":"middleware please see the example pipeline in `proxy-server.conf-sample`."},{"line_number":54,"context_line":""},{"line_number":55,"context_line":"The keymaster config option ``encryption_root_secret`` MUST be set before the"},{"line_number":56,"context_line":"middleware is used and MUST be consistent across all proxy servers. Its value"}],"source_content_type":"text/x-rst","patch_set":3,"id":"bab6814e_31e8b87c","line":53,"in_reply_to":"dab17558_653e5c8c","updated":"2016-05-18 12:57:28.000000000","message":"Done","commit_id":"7d1e4d793070a8aa5b2c920efd8460d1142c83a4"},{"author":{"_account_id":12279,"name":"Janie Richling","email":"jrichli@us.ibm.com","username":"jrichli"},"change_message_id":"a61669e3ca327fec5633d28ab0d87bd4c2131ed3","unresolved":false,"context_lines":[{"line_number":62,"context_line":"following option should be added (or modified) in the `swift.conf` file::"},{"line_number":63,"context_line":""},{"line_number":64,"context_line":"  [swift-constraints]"},{"line_number":65,"context_line":"  extra_header_count \u003d 372"},{"line_number":66,"context_line":""},{"line_number":67,"context_line":"Once deployed, the encrypter will by default encrypt object data and metadata"},{"line_number":68,"context_line":"when PUT and POST requests are made to the proxy server and the decrypter will"}],"source_content_type":"text/x-rst","patch_set":3,"id":"dab17558_a9682cf6","line":65,"updated":"2016-05-16 15:56:44.000000000","message":"This value can probably be something like 95.  I will test to find the right number.  Although, our current implementation doesn\u0027t fail without this - and i have yet to explain why.  haven\u0027t had the chance to trace that down yet.","commit_id":"7d1e4d793070a8aa5b2c920efd8460d1142c83a4"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"66b86565b9beb836576ce441b455d96a7f56cea4","unresolved":false,"context_lines":[{"line_number":62,"context_line":"following option should be added (or modified) in the `swift.conf` file::"},{"line_number":63,"context_line":""},{"line_number":64,"context_line":"  [swift-constraints]"},{"line_number":65,"context_line":"  extra_header_count \u003d 372"},{"line_number":66,"context_line":""},{"line_number":67,"context_line":"Once deployed, the encrypter will by default encrypt object data and metadata"},{"line_number":68,"context_line":"when PUT and POST requests are made to the proxy server and the decrypter will"}],"source_content_type":"text/x-rst","patch_set":3,"id":"bab6814e_51ad7c30","line":65,"in_reply_to":"dab17558_a9682cf6","updated":"2016-05-18 12:57:28.000000000","message":"this praragraph is no longer required","commit_id":"7d1e4d793070a8aa5b2c920efd8460d1142c83a4"},{"author":{"_account_id":330,"name":"John Dickinson","email":"me@not.mn","username":"notmyname"},"change_message_id":"dc837be60e94a2c3457000773426138ae9e9f3e3","unresolved":false,"context_lines":[{"line_number":69,"context_line":"decrypt object data and metadata when handling GET and HEAD requests."},{"line_number":70,"context_line":""},{"line_number":71,"context_line":"Objects that existed in the cluster prior to the crypto middleware being"},{"line_number":72,"context_line":"deployed may still be read with GET and HEAD requests. Those existing objects"},{"line_number":73,"context_line":"will not be encrypted unless they are written again by a PUT, POST or COPY"},{"line_number":74,"context_line":"request."},{"line_number":75,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"dab17558_a02672cb","line":72,"updated":"2016-05-16 21:19:29.000000000","message":"s/read/readable/","commit_id":"7d1e4d793070a8aa5b2c920efd8460d1142c83a4"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"66b86565b9beb836576ce441b455d96a7f56cea4","unresolved":false,"context_lines":[{"line_number":69,"context_line":"decrypt object data and metadata when handling GET and HEAD requests."},{"line_number":70,"context_line":""},{"line_number":71,"context_line":"Objects that existed in the cluster prior to the crypto middleware being"},{"line_number":72,"context_line":"deployed may still be read with GET and HEAD requests. Those existing objects"},{"line_number":73,"context_line":"will not be encrypted unless they are written again by a PUT, POST or COPY"},{"line_number":74,"context_line":"request."},{"line_number":75,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"bab6814e_712520c1","line":72,"in_reply_to":"dab17558_a02672cb","updated":"2016-05-18 12:57:28.000000000","message":"Done","commit_id":"7d1e4d793070a8aa5b2c920efd8460d1142c83a4"},{"author":{"_account_id":9625,"name":"Thiago da Silva","email":"thiagodasilva@gmail.com","username":"thiago"},"change_message_id":"7e01aeab531a2ea568395805cc17176c612101ff","unresolved":false,"context_lines":[{"line_number":80,"context_line":"Encryption of inbound object data may be disabled by setting the encrypter"},{"line_number":81,"context_line":"``disable_encryption`` option to ``True``, in which case existing encrypted"},{"line_number":82,"context_line":"objects will remain encrypted but new data written with PUT, POST or COPY"},{"line_number":83,"context_line":"requests will not be encrypted. The encrypter middleware should remain in the"},{"line_number":84,"context_line":"pipeline so that it can handle conditional GET requests that may be for"},{"line_number":85,"context_line":"previously encrypted objects."},{"line_number":86,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"dab17558_31cf8c2a","line":83,"updated":"2016-05-10 16:09:55.000000000","message":"this seems a little contradictory because in a paragraph above it was stated that the decrypter decrypts data. why is encrypter needed for conditional GETs?","commit_id":"7d1e4d793070a8aa5b2c920efd8460d1142c83a4"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"56852bcd2a2a33d29d2ad4a92baccb4da67d0588","unresolved":false,"context_lines":[{"line_number":80,"context_line":"Encryption of inbound object data may be disabled by setting the encrypter"},{"line_number":81,"context_line":"``disable_encryption`` option to ``True``, in which case existing encrypted"},{"line_number":82,"context_line":"objects will remain encrypted but new data written with PUT, POST or COPY"},{"line_number":83,"context_line":"requests will not be encrypted. The encrypter middleware should remain in the"},{"line_number":84,"context_line":"pipeline so that it can handle conditional GET requests that may be for"},{"line_number":85,"context_line":"previously encrypted objects."},{"line_number":86,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"dab17558_fa10dd5b","line":83,"in_reply_to":"dab17558_31cf8c2a","updated":"2016-05-12 08:39:46.000000000","message":"The encrypter needs to encrypt the If-[None-]Match header values on the inbound conditional GET or HEAD in order for the conditional request to work with a previously encrypted object.\n\nTBH I am not convinced we need the separation of encrypter and decrypter middlewares and it would make for simpler docs and less error prone deployment if they were in one module, but that is how it is for now.","commit_id":"7d1e4d793070a8aa5b2c920efd8460d1142c83a4"},{"author":{"_account_id":12279,"name":"Janie Richling","email":"jrichli@us.ibm.com","username":"jrichli"},"change_message_id":"a61669e3ca327fec5633d28ab0d87bd4c2131ed3","unresolved":false,"context_lines":[{"line_number":82,"context_line":"objects will remain encrypted but new data written with PUT, POST or COPY"},{"line_number":83,"context_line":"requests will not be encrypted. The encrypter middleware should remain in the"},{"line_number":84,"context_line":"pipeline so that it can handle conditional GET requests that may be for"},{"line_number":85,"context_line":"previously encrypted objects."},{"line_number":86,"context_line":""},{"line_number":87,"context_line":"--------------------------"},{"line_number":88,"context_line":"Performance Considerations"}],"source_content_type":"text/x-rst","patch_set":3,"id":"dab17558_7c31e4ec","line":85,"updated":"2016-05-16 15:56:44.000000000","message":"We need to add a section that speaks about container-sync. You need to add crytpo to the internal-client pipeline for container-sync.  In addition, the proxy-config having the root secret would need to be on a separate disk in the container-server from all the data at rest.","commit_id":"7d1e4d793070a8aa5b2c920efd8460d1142c83a4"},{"author":{"_account_id":12279,"name":"Janie Richling","email":"jrichli@us.ibm.com","username":"jrichli"},"change_message_id":"8da49d1f9441d505924881af4a106be63d438326","unresolved":false,"context_lines":[{"line_number":82,"context_line":"objects will remain encrypted but new data written with PUT, POST or COPY"},{"line_number":83,"context_line":"requests will not be encrypted. The encrypter middleware should remain in the"},{"line_number":84,"context_line":"pipeline so that it can handle conditional GET requests that may be for"},{"line_number":85,"context_line":"previously encrypted objects."},{"line_number":86,"context_line":""},{"line_number":87,"context_line":"--------------------------"},{"line_number":88,"context_line":"Performance Considerations"}],"source_content_type":"text/x-rst","patch_set":3,"id":"bab6814e_1b32cef1","line":85,"in_reply_to":"bab6814e_fc0447c7","updated":"2016-05-18 20:30:30.000000000","message":"The instructions are identical to what you have under the \"Deployment and operation\" section of this document, only those mods go into the internal-client.conf file.","commit_id":"7d1e4d793070a8aa5b2c920efd8460d1142c83a4"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"66b86565b9beb836576ce441b455d96a7f56cea4","unresolved":false,"context_lines":[{"line_number":82,"context_line":"objects will remain encrypted but new data written with PUT, POST or COPY"},{"line_number":83,"context_line":"requests will not be encrypted. The encrypter middleware should remain in the"},{"line_number":84,"context_line":"pipeline so that it can handle conditional GET requests that may be for"},{"line_number":85,"context_line":"previously encrypted objects."},{"line_number":86,"context_line":""},{"line_number":87,"context_line":"--------------------------"},{"line_number":88,"context_line":"Performance Considerations"}],"source_content_type":"text/x-rst","patch_set":3,"id":"bab6814e_fc0447c7","line":85,"in_reply_to":"dab17558_7c31e4ec","updated":"2016-05-18 12:57:28.000000000","message":"I\u0027ll add a TBD - could you suggest an example internal-client conf? The root secret would need to be duplicated in the internal client conf, correct?","commit_id":"7d1e4d793070a8aa5b2c920efd8460d1142c83a4"},{"author":{"_account_id":9625,"name":"Thiago da Silva","email":"thiagodasilva@gmail.com","username":"thiago"},"change_message_id":"7e01aeab531a2ea568395805cc17176c612101ff","unresolved":false,"context_lines":[{"line_number":112,"context_line":"The IV is stored as metadata of the encrypted item so that it is available for"},{"line_number":113,"context_line":"decryption::"},{"line_number":114,"context_line":""},{"line_number":115,"context_line":"  plaintext \u003d D(ciphertext, k, n)"},{"line_number":116,"context_line":""},{"line_number":117,"context_line":"where ``D`` is the decryption function."},{"line_number":118,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"dab17558_2c6baff2","line":115,"updated":"2016-05-10 16:09:55.000000000","message":"should n be changed to iv here?","commit_id":"7d1e4d793070a8aa5b2c920efd8460d1142c83a4"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"66b86565b9beb836576ce441b455d96a7f56cea4","unresolved":false,"context_lines":[{"line_number":112,"context_line":"The IV is stored as metadata of the encrypted item so that it is available for"},{"line_number":113,"context_line":"decryption::"},{"line_number":114,"context_line":""},{"line_number":115,"context_line":"  plaintext \u003d D(ciphertext, k, n)"},{"line_number":116,"context_line":""},{"line_number":117,"context_line":"where ``D`` is the decryption function."},{"line_number":118,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"bab6814e_1781244b","line":115,"in_reply_to":"dab17558_2c6baff2","updated":"2016-05-18 12:57:28.000000000","message":"Done","commit_id":"7d1e4d793070a8aa5b2c920efd8460d1142c83a4"},{"author":{"_account_id":9625,"name":"Thiago da Silva","email":"thiagodasilva@gmail.com","username":"thiago"},"change_message_id":"7e01aeab531a2ea568395805cc17176c612101ff","unresolved":false,"context_lines":[{"line_number":118,"context_line":""},{"line_number":119,"context_line":"A key identifier which is the hash of the encryption key ``H(k)`` is also"},{"line_number":120,"context_line":"stored as metadata of the encrypted data so that the decrypter can verify that"},{"line_number":121,"context_line":"it has the correct key when decrypting any item."},{"line_number":122,"context_line":""},{"line_number":123,"context_line":"In general any encrypted item has accompanying crypto-metadata in the form of a"},{"line_number":124,"context_line":"dictionary::"}],"source_content_type":"text/x-rst","patch_set":3,"id":"dab17558_b7324cf0","line":121,"updated":"2016-05-10 16:09:55.000000000","message":"why does the decrypter need to verify that it has the correct key as opposed to just trying to decrypt?\n\nI understand that it is to keep the proxy from returning garbage to the client (especially if it\u0027s a big object), I think this could be called out...","commit_id":"7d1e4d793070a8aa5b2c920efd8460d1142c83a4"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"56852bcd2a2a33d29d2ad4a92baccb4da67d0588","unresolved":false,"context_lines":[{"line_number":118,"context_line":""},{"line_number":119,"context_line":"A key identifier which is the hash of the encryption key ``H(k)`` is also"},{"line_number":120,"context_line":"stored as metadata of the encrypted data so that the decrypter can verify that"},{"line_number":121,"context_line":"it has the correct key when decrypting any item."},{"line_number":122,"context_line":""},{"line_number":123,"context_line":"In general any encrypted item has accompanying crypto-metadata in the form of a"},{"line_number":124,"context_line":"dictionary::"}],"source_content_type":"text/x-rst","patch_set":3,"id":"dab17558_05dc6245","line":121,"in_reply_to":"dab17558_b7324cf0","updated":"2016-05-12 08:39:46.000000000","message":"This paragraph (and any other reference to key ids) may be removed. We needed it for BYOK, in order to verify that the client has sent the correct key and not return garbage otherwise.\n\nWith the server managed keys (our current keymaster implementation) the keys provided would only change from encryption time to decryption time if there was a bug or the encryption_root_secret got changed. Protecting against those errors could be a follow on feature once we are on master if need be.\n\nThe other case for having key id is to support possible future re-keying implementation - it would enable periods of transition when more than one key was in use for a resources (as an old key is retired and a new one is introduced). Again, can be a follow-on as and when we get to that - the absence of any key id would indicate that the very first key is in use.)","commit_id":"7d1e4d793070a8aa5b2c920efd8460d1142c83a4"},{"author":{"_account_id":330,"name":"John Dickinson","email":"me@not.mn","username":"notmyname"},"change_message_id":"dc837be60e94a2c3457000773426138ae9e9f3e3","unresolved":false,"context_lines":[{"line_number":124,"context_line":"dictionary::"},{"line_number":125,"context_line":""},{"line_number":126,"context_line":"  crypto_metadata \u003d {\u0027iv\u0027: \u003c16 byte value\u003e,"},{"line_number":127,"context_line":"                     \u0027key_id\u0027: H(k)}"},{"line_number":128,"context_line":""},{"line_number":129,"context_line":"Key management"},{"line_number":130,"context_line":"--------------"}],"source_content_type":"text/x-rst","patch_set":3,"id":"dab17558_144ea36e","line":127,"updated":"2016-05-16 21:19:29.000000000","message":"\"in the form of a dictionary\" seems odd. Just talk about the metadata and not the actual data structure implementation.","commit_id":"7d1e4d793070a8aa5b2c920efd8460d1142c83a4"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"66b86565b9beb836576ce441b455d96a7f56cea4","unresolved":false,"context_lines":[{"line_number":124,"context_line":"dictionary::"},{"line_number":125,"context_line":""},{"line_number":126,"context_line":"  crypto_metadata \u003d {\u0027iv\u0027: \u003c16 byte value\u003e,"},{"line_number":127,"context_line":"                     \u0027key_id\u0027: H(k)}"},{"line_number":128,"context_line":""},{"line_number":129,"context_line":"Key management"},{"line_number":130,"context_line":"--------------"}],"source_content_type":"text/x-rst","patch_set":3,"id":"bab6814e_c95f040d","line":127,"in_reply_to":"dab17558_144ea36e","updated":"2016-05-18 12:57:28.000000000","message":"Done","commit_id":"7d1e4d793070a8aa5b2c920efd8460d1142c83a4"},{"author":{"_account_id":21231,"name":"Bryan Keller","email":"kellerbr@us.ibm.com","username":"bkeller"},"change_message_id":"33ddef5d6e748e24522b9fe2051cc84cee35167f","unresolved":false,"context_lines":[{"line_number":129,"context_line":"Key management"},{"line_number":130,"context_line":"--------------"},{"line_number":131,"context_line":""},{"line_number":132,"context_line":"The keymaster middleware is responsible for providing a container key and an"},{"line_number":133,"context_line":"object key for each encryption and decryption operation. These are made"},{"line_number":134,"context_line":"available to the encrypter and decrypter via a callback function that the"},{"line_number":135,"context_line":"keymaster installs in the WSGI request environ."}],"source_content_type":"text/x-rst","patch_set":3,"id":"1a122d0e_1a8e0bd0","line":132,"updated":"2016-05-04 15:35:08.000000000","message":"Maybe mention encryption is per-object. It took me until the end of this section to figure that out.","commit_id":"7d1e4d793070a8aa5b2c920efd8460d1142c83a4"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"66b86565b9beb836576ce441b455d96a7f56cea4","unresolved":false,"context_lines":[{"line_number":129,"context_line":"Key management"},{"line_number":130,"context_line":"--------------"},{"line_number":131,"context_line":""},{"line_number":132,"context_line":"The keymaster middleware is responsible for providing a container key and an"},{"line_number":133,"context_line":"object key for each encryption and decryption operation. These are made"},{"line_number":134,"context_line":"available to the encrypter and decrypter via a callback function that the"},{"line_number":135,"context_line":"keymaster installs in the WSGI request environ."}],"source_content_type":"text/x-rst","patch_set":3,"id":"bab6814e_9703540f","line":132,"in_reply_to":"1a122d0e_1a8e0bd0","updated":"2016-05-18 12:57:28.000000000","message":"Done","commit_id":"7d1e4d793070a8aa5b2c920efd8460d1142c83a4"},{"author":{"_account_id":9625,"name":"Thiago da Silva","email":"thiagodasilva@gmail.com","username":"thiago"},"change_message_id":"7e01aeab531a2ea568395805cc17176c612101ff","unresolved":false,"context_lines":[{"line_number":160,"context_line":"For example, as described below, the object body is encrypted using a random"},{"line_number":161,"context_line":"key which is then wrapped using the object key. If re-keying requires the"},{"line_number":162,"context_line":"object key to be replaced the only random key needs to be re-encrypted but not"},{"line_number":163,"context_line":"the object body, which is potentially a large amount of data."},{"line_number":164,"context_line":""},{"line_number":165,"context_line":".. note::"},{"line_number":166,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"dab17558_37a93c91","line":163,"updated":"2016-05-10 16:09:55.000000000","message":"nice!!!","commit_id":"7d1e4d793070a8aa5b2c920efd8460d1142c83a4"},{"author":{"_account_id":21231,"name":"Bryan Keller","email":"kellerbr@us.ibm.com","username":"bkeller"},"change_message_id":"33ddef5d6e748e24522b9fe2051cc84cee35167f","unresolved":false,"context_lines":[{"line_number":180,"context_line":"container key obtained from the keymaster and the wrapped key is stored as an"},{"line_number":181,"context_line":"item of container system metadata::"},{"line_number":182,"context_line":""},{"line_number":183,"context_line":"X-Container-Sysmeta-Crypto-Listing-Key-\u003cH(listing_key)\u003e:"},{"line_number":184,"context_line":"      {\u0027iv\u0027: random_iv,"},{"line_number":185,"context_line":"       \u0027key_id\u0027: H(container_key),"},{"line_number":186,"context_line":"       \u0027listing_key\u0027: E(listing_key, container_key, random_iv)}"}],"source_content_type":"text/x-rst","patch_set":3,"id":"1a122d0e_857000a6","line":183,"updated":"2016-05-04 15:35:08.000000000","message":"Shouldn\u0027t this line be indented?","commit_id":"7d1e4d793070a8aa5b2c920efd8460d1142c83a4"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"66b86565b9beb836576ce441b455d96a7f56cea4","unresolved":false,"context_lines":[{"line_number":180,"context_line":"container key obtained from the keymaster and the wrapped key is stored as an"},{"line_number":181,"context_line":"item of container system metadata::"},{"line_number":182,"context_line":""},{"line_number":183,"context_line":"X-Container-Sysmeta-Crypto-Listing-Key-\u003cH(listing_key)\u003e:"},{"line_number":184,"context_line":"      {\u0027iv\u0027: random_iv,"},{"line_number":185,"context_line":"       \u0027key_id\u0027: H(container_key),"},{"line_number":186,"context_line":"       \u0027listing_key\u0027: E(listing_key, container_key, random_iv)}"}],"source_content_type":"text/x-rst","patch_set":3,"id":"bab6814e_89933ca3","line":183,"in_reply_to":"1a122d0e_857000a6","updated":"2016-05-18 12:57:28.000000000","message":"Done","commit_id":"7d1e4d793070a8aa5b2c920efd8460d1142c83a4"},{"author":{"_account_id":9625,"name":"Thiago da Silva","email":"thiagodasilva@gmail.com","username":"thiago"},"change_message_id":"7e01aeab531a2ea568395805cc17176c612101ff","unresolved":false,"context_lines":[{"line_number":201,"context_line":"re-keying metadata is not expensive, but we could equally choose a random key"},{"line_number":202,"context_line":"for metadata."},{"line_number":203,"context_line":""},{"line_number":204,"context_line":"The encrypter encrypts object each item of custom user metadata using the"},{"line_number":205,"context_line":"object key provided by the keymaster and an IV that is randomly chosen for that"},{"line_number":206,"context_line":"metadata item. For example::"},{"line_number":207,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"dab17558_37813cc4","line":204,"updated":"2016-05-10 16:09:55.000000000","message":"remove \u0027object\u0027 ??\nThe encrypter encrypts each item of custom...","commit_id":"7d1e4d793070a8aa5b2c920efd8460d1142c83a4"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"66b86565b9beb836576ce441b455d96a7f56cea4","unresolved":false,"context_lines":[{"line_number":201,"context_line":"re-keying metadata is not expensive, but we could equally choose a random key"},{"line_number":202,"context_line":"for metadata."},{"line_number":203,"context_line":""},{"line_number":204,"context_line":"The encrypter encrypts object each item of custom user metadata using the"},{"line_number":205,"context_line":"object key provided by the keymaster and an IV that is randomly chosen for that"},{"line_number":206,"context_line":"metadata item. For example::"},{"line_number":207,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"bab6814e_09ee2c22","line":204,"in_reply_to":"dab17558_37813cc4","updated":"2016-05-18 12:57:28.000000000","message":"Done","commit_id":"7d1e4d793070a8aa5b2c920efd8460d1142c83a4"},{"author":{"_account_id":9625,"name":"Thiago da Silva","email":"thiagodasilva@gmail.com","username":"thiago"},"change_message_id":"7e01aeab531a2ea568395805cc17176c612101ff","unresolved":false,"context_lines":[{"line_number":233,"context_line":"The encrypter stores the associated crypto metadata in a system metadata"},{"line_number":234,"context_line":"header::"},{"line_number":235,"context_line":""},{"line_number":236,"context_line":"  X-Object-Sysmeta-Crypto-Meta:"},{"line_number":237,"context_line":"      {\u0027iv\u0027: body_iv,"},{"line_number":238,"context_line":"       \u0027key_id\u0027: H(object_key),"},{"line_number":239,"context_line":"       \u0027body_key\u0027: E(body_key, object_key, body_iv)}"}],"source_content_type":"text/x-rst","patch_set":3,"id":"dab17558_d7c270e8","line":236,"updated":"2016-05-10 16:09:55.000000000","message":"why is Transient-Sysmeta used for object metadata and not of object data?","commit_id":"7d1e4d793070a8aa5b2c920efd8460d1142c83a4"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"56852bcd2a2a33d29d2ad4a92baccb4da67d0588","unresolved":false,"context_lines":[{"line_number":233,"context_line":"The encrypter stores the associated crypto metadata in a system metadata"},{"line_number":234,"context_line":"header::"},{"line_number":235,"context_line":""},{"line_number":236,"context_line":"  X-Object-Sysmeta-Crypto-Meta:"},{"line_number":237,"context_line":"      {\u0027iv\u0027: body_iv,"},{"line_number":238,"context_line":"       \u0027key_id\u0027: H(object_key),"},{"line_number":239,"context_line":"       \u0027body_key\u0027: E(body_key, object_key, body_iv)}"}],"source_content_type":"text/x-rst","patch_set":3,"id":"dab17558_8575f233","line":236,"in_reply_to":"dab17558_d7c270e8","updated":"2016-05-12 08:39:46.000000000","message":"Transient-sysmeta gets replaced in whole on a POST. Sysmeta persists for as long as the object data persists. So any metadata associated with the object body data must be sysmeta and any metadata associated with custom metadata should be transient-sysmeta.","commit_id":"7d1e4d793070a8aa5b2c920efd8460d1142c83a4"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"56852bcd2a2a33d29d2ad4a92baccb4da67d0588","unresolved":false,"context_lines":[{"line_number":246,"context_line":""},{"line_number":247,"context_line":"While encrypting the object body the encrypter also calculates the hexdigest of"},{"line_number":248,"context_line":"the md5 sum of the plaintext body. This value is encrypted using a container"},{"line_number":249,"context_line":"listing key that has been read from the container metadata, and an IV that is"},{"line_number":250,"context_line":"derived from the object\u0027s path, and saved as an item of system metadata::"},{"line_number":251,"context_line":""},{"line_number":252,"context_line":"  X-Object-Sysmeta-Crypto-Etag: E(md5(plaintext), listing_key, F(path))"}],"source_content_type":"text/x-rst","patch_set":3,"id":"dab17558_e5eafeab","line":249,"updated":"2016-05-12 08:39:46.000000000","message":"change container listing key -\u003e just the container key\n\nwe may not implement listing keys in first release","commit_id":"7d1e4d793070a8aa5b2c920efd8460d1142c83a4"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"56852bcd2a2a33d29d2ad4a92baccb4da67d0588","unresolved":false,"context_lines":[{"line_number":255,"context_line":"header::"},{"line_number":256,"context_line":""},{"line_number":257,"context_line":"  X-Object-Sysmeta-Crypto-Meta-Etag: {\u0027iv\u0027: F(path),"},{"line_number":258,"context_line":"                                      \u0027key_id\u0027: H(listing_key)}"},{"line_number":259,"context_line":""},{"line_number":260,"context_line":"The reason for using the listing key for this encryption is that the"},{"line_number":261,"context_line":"encrypted etag must also be included in the object update to the container"}],"source_content_type":"text/x-rst","patch_set":3,"id":"dab17558_45f78a8c","line":258,"updated":"2016-05-12 08:39:46.000000000","message":"revisit: remove key_id?","commit_id":"7d1e4d793070a8aa5b2c920efd8460d1142c83a4"},{"author":{"_account_id":21231,"name":"Bryan Keller","email":"kellerbr@us.ibm.com","username":"bkeller"},"change_message_id":"33ddef5d6e748e24522b9fe2051cc84cee35167f","unresolved":false,"context_lines":[{"line_number":297,"context_line":"for encrypted items (revealed by crypto metadata headers), and if any are"},{"line_number":298,"context_line":"discovered then:"},{"line_number":299,"context_line":""},{"line_number":300,"context_line":"#. fetch container and object keys from the keymaster via its callback"},{"line_number":301,"context_line":"#. retrieve listing_key(s) from the container metadata"},{"line_number":302,"context_line":"#. decrypt the wrapped container listing key(s)"},{"line_number":303,"context_line":"#. decrypt the ``X-Object-Sysmeta-Crypto-Etag`` value using whichever container"}],"source_content_type":"text/x-rst","patch_set":3,"id":"1a122d0e_a5affc7b","line":300,"updated":"2016-05-04 15:35:08.000000000","message":"More capitalization","commit_id":"7d1e4d793070a8aa5b2c920efd8460d1142c83a4"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"66b86565b9beb836576ce441b455d96a7f56cea4","unresolved":false,"context_lines":[{"line_number":297,"context_line":"for encrypted items (revealed by crypto metadata headers), and if any are"},{"line_number":298,"context_line":"discovered then:"},{"line_number":299,"context_line":""},{"line_number":300,"context_line":"#. fetch container and object keys from the keymaster via its callback"},{"line_number":301,"context_line":"#. retrieve listing_key(s) from the container metadata"},{"line_number":302,"context_line":"#. decrypt the wrapped container listing key(s)"},{"line_number":303,"context_line":"#. decrypt the ``X-Object-Sysmeta-Crypto-Etag`` value using whichever container"}],"source_content_type":"text/x-rst","patch_set":3,"id":"bab6814e_89c17c83","line":300,"in_reply_to":"1a122d0e_a5affc7b","updated":"2016-05-18 12:57:28.000000000","message":"Done","commit_id":"7d1e4d793070a8aa5b2c920efd8460d1142c83a4"},{"author":{"_account_id":9625,"name":"Thiago da Silva","email":"thiagodasilva@gmail.com","username":"thiago"},"change_message_id":"7e01aeab531a2ea568395805cc17176c612101ff","unresolved":false,"context_lines":[{"line_number":357,"context_line":"not cause the object data and metadata to pass through the crypto middlewares"},{"line_number":358,"context_line":"in the proxy pipeline and be re-encrypted."},{"line_number":359,"context_line":""},{"line_number":360,"context_line":"The encrypter does include a key_id, which is a hash of the key, in the crypto"},{"line_number":361,"context_line":"metadata that is associated with each item of encrypted data. This key_id has"},{"line_number":362,"context_line":"been included in anticipation of future scenarios when it may be necessary to"},{"line_number":363,"context_line":"`verify` that a key proposed for decryption is the same as the key used for"}],"source_content_type":"text/x-rst","patch_set":3,"id":"dab17558_72747267","line":360,"updated":"2016-05-10 16:09:55.000000000","message":"nice, maybe this paragraph could be moved up where it states that the decrypter needs to verify that it has the correct key","commit_id":"7d1e4d793070a8aa5b2c920efd8460d1142c83a4"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"66b86565b9beb836576ce441b455d96a7f56cea4","unresolved":false,"context_lines":[{"line_number":357,"context_line":"not cause the object data and metadata to pass through the crypto middlewares"},{"line_number":358,"context_line":"in the proxy pipeline and be re-encrypted."},{"line_number":359,"context_line":""},{"line_number":360,"context_line":"The encrypter does include a key_id, which is a hash of the key, in the crypto"},{"line_number":361,"context_line":"metadata that is associated with each item of encrypted data. This key_id has"},{"line_number":362,"context_line":"been included in anticipation of future scenarios when it may be necessary to"},{"line_number":363,"context_line":"`verify` that a key proposed for decryption is the same as the key used for"}],"source_content_type":"text/x-rst","patch_set":3,"id":"bab6814e_a9f9e0ca","line":360,"in_reply_to":"dab17558_72747267","updated":"2016-05-18 12:57:28.000000000","message":"This description of key_id will be revised.","commit_id":"7d1e4d793070a8aa5b2c920efd8460d1142c83a4"},{"author":{"_account_id":12279,"name":"Janie Richling","email":"jrichli@us.ibm.com","username":"jrichli"},"change_message_id":"66dea609f3956bb75f403be99333a666a5eaf460","unresolved":false,"context_lines":[{"line_number":84,"context_line":"If container sync is being used then the encrytion middlewares must be added to"},{"line_number":85,"context_line":"the container sync internal client pipeline."},{"line_number":86,"context_line":""},{"line_number":87,"context_line":"[TBD add details and example config]"},{"line_number":88,"context_line":""},{"line_number":89,"context_line":"--------------------------"},{"line_number":90,"context_line":"Performance Considerations"}],"source_content_type":"text/x-rst","patch_set":4,"id":"bab6814e_b1591a5d","line":87,"updated":"2016-05-19 02:33:11.000000000","message":"I just saw that there is a mention later about container sync possibly being incompatiable.  I guess this is a TODO.  Right now, we still run those probetests and put the crypto middleware in the pipeline for internal-client.  I guess we need to decide if crypto will be supported for container-sync.  For that matter, we also need to consider what advice to give for custom middleware or other plugins that make use of internal-client.  I think storelets uses it, for example.","commit_id":"9b05163a705573bd2745195f146dccd3a33ddb0e"},{"author":{"_account_id":11330,"name":"Christian Cachin","email":"cca@zurich.ibm.com","username":"cca88"},"change_message_id":"58330b60dea82bea61f5cce687e288b00dceac42","unresolved":false,"context_lines":[{"line_number":55,"context_line":"The keymaster config option ``encryption_root_secret`` MUST be set before the"},{"line_number":56,"context_line":"middleware is used and MUST be consistent across all proxy servers. Its value"},{"line_number":57,"context_line":"(and consequently, the proxy-server.conf file) should not be stored on any disk"},{"line_number":58,"context_line":"that is in any account, container or object ring."},{"line_number":59,"context_line":""},{"line_number":60,"context_line":"Once deployed, the encrypter will by default encrypt object data and metadata"},{"line_number":61,"context_line":"when PUT and POST requests are made to the proxy server and the decrypter will"}],"source_content_type":"text/x-rst","patch_set":5,"id":"bab6814e_d8ee96b3","line":58,"updated":"2016-05-24 15:03:19.000000000","message":"This deserves more attention, as it discusses the master secret key for encryption.  I would say instead:\n---\nThe keymaster config option encryption_root_secret holds the master secret\nkey used for encryption.  The security of all encrypted data critically\ndepends on this key, therefore it should be set to a high-entropy value\ngenerated by a cryptographically secure random number generator.  For use\nwith 256-bit AES it should have at least 32 truly random bytes or more than\n42 truly random characters from the base-64 alphabet.\n\nThe encryption_root_secret value is necessary to recover any encrypted data\nfrom the storage system, and therefore, it must be guarded against\naccidental loss.\n---","commit_id":"658fc188b6803ebd990645c4de7ed3610133ee70"},{"author":{"_account_id":11330,"name":"Christian Cachin","email":"cca@zurich.ibm.com","username":"cca88"},"change_message_id":"bd3fdf971fdc32077cf6d950db9ef22f291852a0","unresolved":false,"context_lines":[{"line_number":55,"context_line":"The keymaster config option ``encryption_root_secret`` MUST be set before the"},{"line_number":56,"context_line":"middleware is used and MUST be consistent across all proxy servers. Its value"},{"line_number":57,"context_line":"(and consequently, the proxy-server.conf file) should not be stored on any disk"},{"line_number":58,"context_line":"that is in any account, container or object ring."},{"line_number":59,"context_line":""},{"line_number":60,"context_line":"Once deployed, the encrypter will by default encrypt object data and metadata"},{"line_number":61,"context_line":"when PUT and POST requests are made to the proxy server and the decrypter will"}],"source_content_type":"text/x-rst","patch_set":5,"id":"bab6814e_836c7e9d","line":58,"in_reply_to":"bab6814e_1e2c6a49","updated":"2016-05-25 06:41:39.000000000","message":"No, it is the master secret key itself. I realize this may not have been understood like this, so I suggest we make this very clear. HMAC is deterministic and takes a \"key\" and some \"data\" as input. It doesn\u0027t \"generate\" a key in any way. Our secret here becomes is the key. Hence it MUST be generated from a cryptographically strong random generator. We have to treat it as a key and it MUST have enough entropy. It can\u0027t be a human-memorable password, this would allow guessing attacks on it and considerably weaken the security.\n\nWe are using HMAC as a key derivation function according to NIST SP 800-108 here, section 5.1, with PRF \u003d\u003d HMAC.","commit_id":"658fc188b6803ebd990645c4de7ed3610133ee70"},{"author":{"_account_id":11330,"name":"Christian Cachin","email":"cca@zurich.ibm.com","username":"cca88"},"change_message_id":"679bb1f0217af8acd8f4bc40753ce7c27c04b3c7","unresolved":false,"context_lines":[{"line_number":55,"context_line":"The keymaster config option ``encryption_root_secret`` MUST be set before the"},{"line_number":56,"context_line":"middleware is used and MUST be consistent across all proxy servers. Its value"},{"line_number":57,"context_line":"(and consequently, the proxy-server.conf file) should not be stored on any disk"},{"line_number":58,"context_line":"that is in any account, container or object ring."},{"line_number":59,"context_line":""},{"line_number":60,"context_line":"Once deployed, the encrypter will by default encrypt object data and metadata"},{"line_number":61,"context_line":"when PUT and POST requests are made to the proxy server and the decrypter will"}],"source_content_type":"text/x-rst","patch_set":5,"id":"bab6814e_1d4d6699","line":58,"in_reply_to":"bab6814e_2d681187","updated":"2016-05-25 18:14:56.000000000","message":"Note that length doesn\u0027t matter at all, it is the entropy (\u003d amount of randomness contained in the key) that\u0027s relevant. All those functions are deterministic and will not throw in any randomness.\n\nThese other schemes are pointless here, they apply when the user input is a password. The HMAC already does what this is supposed to, but it requires this to be a strong key. The page [1] says the input is a human-memorable password. Passwords have very low entropy and give very weak security for storage encryption, and a password does not correspond to the model we use here. Our admins will have to copy\u0026paste the secret key into the config files. Nobody will have to type this secret into a terminal. It has to be pointed out very strongly that this is the secret key and that it must be generated appropriately.\n\nI am not aware of any recent storage encryption scheme that has the key in a config file. Generally storing keys in config files is a sort of no-no nowadays, but I have succumbed to the opinions here that this design is adequate for a first step.","commit_id":"658fc188b6803ebd990645c4de7ed3610133ee70"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"ceac169c6b395de16790d400b3f68e30852e019e","unresolved":false,"context_lines":[{"line_number":55,"context_line":"The keymaster config option ``encryption_root_secret`` MUST be set before the"},{"line_number":56,"context_line":"middleware is used and MUST be consistent across all proxy servers. Its value"},{"line_number":57,"context_line":"(and consequently, the proxy-server.conf file) should not be stored on any disk"},{"line_number":58,"context_line":"that is in any account, container or object ring."},{"line_number":59,"context_line":""},{"line_number":60,"context_line":"Once deployed, the encrypter will by default encrypt object data and metadata"},{"line_number":61,"context_line":"when PUT and POST requests are made to the proxy server and the decrypter will"}],"source_content_type":"text/x-rst","patch_set":5,"id":"bab6814e_2d681187","line":58,"in_reply_to":"bab6814e_836c7e9d","updated":"2016-05-25 13:17:42.000000000","message":"As an alternative to enforcing a minimum secret length, is there a suitable mechanism to transform a secret to a crytographically strong key that is suitable as a parameter for the HMACs e.g. are any of the functions here [1] intended for that purpose?\n\n[1] https://cryptography.io/en/latest/hazmat/primitives/key-derivation-functions/","commit_id":"658fc188b6803ebd990645c4de7ed3610133ee70"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"abe816371b77500d7fec0cb9ea06431d93531cef","unresolved":false,"context_lines":[{"line_number":55,"context_line":"The keymaster config option ``encryption_root_secret`` MUST be set before the"},{"line_number":56,"context_line":"middleware is used and MUST be consistent across all proxy servers. Its value"},{"line_number":57,"context_line":"(and consequently, the proxy-server.conf file) should not be stored on any disk"},{"line_number":58,"context_line":"that is in any account, container or object ring."},{"line_number":59,"context_line":""},{"line_number":60,"context_line":"Once deployed, the encrypter will by default encrypt object data and metadata"},{"line_number":61,"context_line":"when PUT and POST requests are made to the proxy server and the decrypter will"}],"source_content_type":"text/x-rst","patch_set":5,"id":"bab6814e_427070a4","line":58,"in_reply_to":"bab6814e_a0d87180","updated":"2016-05-26 17:15:08.000000000","message":"Christian - Understood. I have added your proposed text with a slight change - we can *suggest* in this doc that the value is at least 32 random bytes, we can *require* in code (by otherwise raising an Exception) the value to be 44 characters.","commit_id":"658fc188b6803ebd990645c4de7ed3610133ee70"},{"author":{"_account_id":11330,"name":"Christian Cachin","email":"cca@zurich.ibm.com","username":"cca88"},"change_message_id":"679bb1f0217af8acd8f4bc40753ce7c27c04b3c7","unresolved":false,"context_lines":[{"line_number":55,"context_line":"The keymaster config option ``encryption_root_secret`` MUST be set before the"},{"line_number":56,"context_line":"middleware is used and MUST be consistent across all proxy servers. Its value"},{"line_number":57,"context_line":"(and consequently, the proxy-server.conf file) should not be stored on any disk"},{"line_number":58,"context_line":"that is in any account, container or object ring."},{"line_number":59,"context_line":""},{"line_number":60,"context_line":"Once deployed, the encrypter will by default encrypt object data and metadata"},{"line_number":61,"context_line":"when PUT and POST requests are made to the proxy server and the decrypter will"}],"source_content_type":"text/x-rst","patch_set":5,"id":"bab6814e_a0d87180","line":58,"in_reply_to":"bab6814e_b870bddd","updated":"2016-05-25 18:14:56.000000000","message":"Well, I took 256 (bits of key) / 6 (information in one base-64 char) \u003d 42.6.  The encoding might add some extra char for padding, but this wont help entropy.  The 256 bits would be \"more than\" 42 in base-64.  The text I proposed above is precise but perhaps not understandable.","commit_id":"658fc188b6803ebd990645c4de7ed3610133ee70"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"ceac169c6b395de16790d400b3f68e30852e019e","unresolved":false,"context_lines":[{"line_number":55,"context_line":"The keymaster config option ``encryption_root_secret`` MUST be set before the"},{"line_number":56,"context_line":"middleware is used and MUST be consistent across all proxy servers. Its value"},{"line_number":57,"context_line":"(and consequently, the proxy-server.conf file) should not be stored on any disk"},{"line_number":58,"context_line":"that is in any account, container or object ring."},{"line_number":59,"context_line":""},{"line_number":60,"context_line":"Once deployed, the encrypter will by default encrypt object data and metadata"},{"line_number":61,"context_line":"when PUT and POST requests are made to the proxy server and the decrypter will"}],"source_content_type":"text/x-rst","patch_set":5,"id":"bab6814e_b870bddd","line":58,"in_reply_to":"bab6814e_d8ee96b3","updated":"2016-05-25 13:17:42.000000000","message":"We can enforce a minimum secret length. I\u0027m not sure we can enforce the nature of its content, but it certainly makes sense to recommend that it should be suitably random, as you suggest.\n\nDid you mean 44 random base-64 characters (rather than 42)? 44 being the length of a base-64 encoded 32 byte value.\n\nhttps://trello.com/c/KCIq3d9X/109-extend-the-root-secret-enforce-a-minimum-length","commit_id":"658fc188b6803ebd990645c4de7ed3610133ee70"},{"author":{"_account_id":12279,"name":"Janie Richling","email":"jrichli@us.ibm.com","username":"jrichli"},"change_message_id":"dff12490aeda68de430b0b0261dd345ebbe74ea4","unresolved":false,"context_lines":[{"line_number":55,"context_line":"The keymaster config option ``encryption_root_secret`` MUST be set before the"},{"line_number":56,"context_line":"middleware is used and MUST be consistent across all proxy servers. Its value"},{"line_number":57,"context_line":"(and consequently, the proxy-server.conf file) should not be stored on any disk"},{"line_number":58,"context_line":"that is in any account, container or object ring."},{"line_number":59,"context_line":""},{"line_number":60,"context_line":"Once deployed, the encrypter will by default encrypt object data and metadata"},{"line_number":61,"context_line":"when PUT and POST requests are made to the proxy server and the decrypter will"}],"source_content_type":"text/x-rst","patch_set":5,"id":"bab6814e_1e2c6a49","line":58,"in_reply_to":"bab6814e_d8ee96b3","updated":"2016-05-25 03:03:48.000000000","message":"technically, this root secret is not a key itself.  That is why we call it a secret.  it is is only an input, along with the items \"path\" in the system, to the HMAC algorithm to derive the key.  Does it still need to be held to these standards?","commit_id":"658fc188b6803ebd990645c4de7ed3610133ee70"},{"author":{"_account_id":11330,"name":"Christian Cachin","email":"cca@zurich.ibm.com","username":"cca88"},"change_message_id":"58330b60dea82bea61f5cce687e288b00dceac42","unresolved":false,"context_lines":[{"line_number":61,"context_line":"when PUT and POST requests are made to the proxy server and the decrypter will"},{"line_number":62,"context_line":"decrypt object data and metadata when handling GET and HEAD requests."},{"line_number":63,"context_line":""},{"line_number":64,"context_line":"Objects that existed in the cluster prior to the crypto middleware being"},{"line_number":65,"context_line":"deployed are still readable with GET and HEAD requests. The content of those"},{"line_number":66,"context_line":"objects will not be encrypted unless they are written again by a PUT or COPY"},{"line_number":67,"context_line":"request. Any user metadata of those objects will not be encrypted unless it is"}],"source_content_type":"text/x-rst","patch_set":5,"id":"bab6814e_01226ab9","line":64,"range":{"start_line":64,"start_character":49,"end_line":64,"end_character":66},"updated":"2016-05-24 15:03:19.000000000","message":"better: encryption middlewares \n(\"crypto\" is generic for any cryptography, we are doing data encryption here)","commit_id":"658fc188b6803ebd990645c4de7ed3610133ee70"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"ceac169c6b395de16790d400b3f68e30852e019e","unresolved":false,"context_lines":[{"line_number":61,"context_line":"when PUT and POST requests are made to the proxy server and the decrypter will"},{"line_number":62,"context_line":"decrypt object data and metadata when handling GET and HEAD requests."},{"line_number":63,"context_line":""},{"line_number":64,"context_line":"Objects that existed in the cluster prior to the crypto middleware being"},{"line_number":65,"context_line":"deployed are still readable with GET and HEAD requests. The content of those"},{"line_number":66,"context_line":"objects will not be encrypted unless they are written again by a PUT or COPY"},{"line_number":67,"context_line":"request. Any user metadata of those objects will not be encrypted unless it is"}],"source_content_type":"text/x-rst","patch_set":5,"id":"bab6814e_4d77553a","line":64,"range":{"start_line":64,"start_character":49,"end_line":64,"end_character":66},"in_reply_to":"bab6814e_01226ab9","updated":"2016-05-25 13:17:42.000000000","message":"Done","commit_id":"658fc188b6803ebd990645c4de7ed3610133ee70"},{"author":{"_account_id":11330,"name":"Christian Cachin","email":"cca@zurich.ibm.com","username":"cca88"},"change_message_id":"58330b60dea82bea61f5cce687e288b00dceac42","unresolved":false,"context_lines":[{"line_number":75,"context_line":"``disable_encryption`` option to ``True``, in which case existing encrypted"},{"line_number":76,"context_line":"objects will remain encrypted but new data written with PUT, POST or COPY"},{"line_number":77,"context_line":"requests will not be encrypted. The encrypter middleware should remain in the"},{"line_number":78,"context_line":"pipeline so that it can handle conditional GET requests that may be for"},{"line_number":79,"context_line":"previously encrypted objects."},{"line_number":80,"context_line":""},{"line_number":81,"context_line":".. _container_sync_client_config:"}],"source_content_type":"text/x-rst","patch_set":5,"id":"bab6814e_b87bd228","line":78,"range":{"start_line":78,"start_character":31,"end_line":78,"end_character":43},"updated":"2016-05-24 15:03:19.000000000","message":"why only for conditional GETs? I would think for any GET and remove \"conditional\"","commit_id":"658fc188b6803ebd990645c4de7ed3610133ee70"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"abe816371b77500d7fec0cb9ea06431d93531cef","unresolved":false,"context_lines":[{"line_number":75,"context_line":"``disable_encryption`` option to ``True``, in which case existing encrypted"},{"line_number":76,"context_line":"objects will remain encrypted but new data written with PUT, POST or COPY"},{"line_number":77,"context_line":"requests will not be encrypted. The encrypter middleware should remain in the"},{"line_number":78,"context_line":"pipeline so that it can handle conditional GET requests that may be for"},{"line_number":79,"context_line":"previously encrypted objects."},{"line_number":80,"context_line":""},{"line_number":81,"context_line":".. _container_sync_client_config:"}],"source_content_type":"text/x-rst","patch_set":5,"id":"bab6814e_229c4cbf","line":78,"range":{"start_line":78,"start_character":31,"end_line":78,"end_character":43},"in_reply_to":"bab6814e_4c5bbef6","updated":"2016-05-26 17:15:08.000000000","message":"Done. It\u0027s not stupid :) this is exactly the kind of feedback that is needed on docs. Thanks.","commit_id":"658fc188b6803ebd990645c4de7ed3610133ee70"},{"author":{"_account_id":11330,"name":"Christian Cachin","email":"cca@zurich.ibm.com","username":"cca88"},"change_message_id":"679bb1f0217af8acd8f4bc40753ce7c27c04b3c7","unresolved":false,"context_lines":[{"line_number":75,"context_line":"``disable_encryption`` option to ``True``, in which case existing encrypted"},{"line_number":76,"context_line":"objects will remain encrypted but new data written with PUT, POST or COPY"},{"line_number":77,"context_line":"requests will not be encrypted. The encrypter middleware should remain in the"},{"line_number":78,"context_line":"pipeline so that it can handle conditional GET requests that may be for"},{"line_number":79,"context_line":"previously encrypted objects."},{"line_number":80,"context_line":""},{"line_number":81,"context_line":".. _container_sync_client_config:"}],"source_content_type":"text/x-rst","patch_set":5,"id":"bab6814e_4c5bbef6","line":78,"range":{"start_line":78,"start_character":31,"end_line":78,"end_character":43},"in_reply_to":"bab6814e_4dc91578","updated":"2016-05-25 18:14:56.000000000","message":"Aha! Overlooked that.\n\nWhy not add, for the stupid, like me:\n\nThe decrypter is of course needed to handle all GET requests.","commit_id":"658fc188b6803ebd990645c4de7ed3610133ee70"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"ceac169c6b395de16790d400b3f68e30852e019e","unresolved":false,"context_lines":[{"line_number":75,"context_line":"``disable_encryption`` option to ``True``, in which case existing encrypted"},{"line_number":76,"context_line":"objects will remain encrypted but new data written with PUT, POST or COPY"},{"line_number":77,"context_line":"requests will not be encrypted. The encrypter middleware should remain in the"},{"line_number":78,"context_line":"pipeline so that it can handle conditional GET requests that may be for"},{"line_number":79,"context_line":"previously encrypted objects."},{"line_number":80,"context_line":""},{"line_number":81,"context_line":".. _container_sync_client_config:"}],"source_content_type":"text/x-rst","patch_set":5,"id":"bab6814e_4dc91578","line":78,"range":{"start_line":78,"start_character":31,"end_line":78,"end_character":43},"in_reply_to":"bab6814e_b87bd228","updated":"2016-05-25 13:17:42.000000000","message":"conditional GETs are those with an If-Match or If-None-Match header. These are the only GET requests for which the encyrpter takes any action i.e. to encrypt the entity tag value(s) in the If-[None-]Match headers. The decrypter is of course needed to handle all GET requests.","commit_id":"658fc188b6803ebd990645c4de7ed3610133ee70"},{"author":{"_account_id":11330,"name":"Christian Cachin","email":"cca@zurich.ibm.com","username":"cca88"},"change_message_id":"58330b60dea82bea61f5cce687e288b00dceac42","unresolved":false,"context_lines":[{"line_number":140,"context_line":"Key management"},{"line_number":141,"context_line":"--------------"},{"line_number":142,"context_line":""},{"line_number":143,"context_line":"The keymaster middleware is responsible for providing the keys required for"},{"line_number":144,"context_line":"each encryption and decryption operation. Unique keys are provided for each"},{"line_number":145,"context_line":"object and container. These are made available to the encrypter and decrypter"},{"line_number":146,"context_line":"via a callback function that the keymaster installs in the WSGI request"}],"source_content_type":"text/x-rst","patch_set":5,"id":"bab6814e_deb31f09","line":143,"range":{"start_line":143,"start_character":0,"end_line":143,"end_character":3},"updated":"2016-05-24 15:03:19.000000000","message":"Let\u0027s talk about any keymaster here, so I\u0027d say:\n\nA keymaster middleware ...","commit_id":"658fc188b6803ebd990645c4de7ed3610133ee70"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"ceac169c6b395de16790d400b3f68e30852e019e","unresolved":false,"context_lines":[{"line_number":140,"context_line":"Key management"},{"line_number":141,"context_line":"--------------"},{"line_number":142,"context_line":""},{"line_number":143,"context_line":"The keymaster middleware is responsible for providing the keys required for"},{"line_number":144,"context_line":"each encryption and decryption operation. Unique keys are provided for each"},{"line_number":145,"context_line":"object and container. These are made available to the encrypter and decrypter"},{"line_number":146,"context_line":"via a callback function that the keymaster installs in the WSGI request"}],"source_content_type":"text/x-rst","patch_set":5,"id":"bab6814e_8d601d37","line":143,"range":{"start_line":143,"start_character":0,"end_line":143,"end_character":3},"in_reply_to":"bab6814e_deb31f09","updated":"2016-05-25 13:17:42.000000000","message":"Done","commit_id":"658fc188b6803ebd990645c4de7ed3610133ee70"},{"author":{"_account_id":11330,"name":"Christian Cachin","email":"cca@zurich.ibm.com","username":"cca88"},"change_message_id":"58330b60dea82bea61f5cce687e288b00dceac42","unresolved":false,"context_lines":[{"line_number":141,"context_line":"--------------"},{"line_number":142,"context_line":""},{"line_number":143,"context_line":"The keymaster middleware is responsible for providing the keys required for"},{"line_number":144,"context_line":"each encryption and decryption operation. Unique keys are provided for each"},{"line_number":145,"context_line":"object and container. These are made available to the encrypter and decrypter"},{"line_number":146,"context_line":"via a callback function that the keymaster installs in the WSGI request"},{"line_number":147,"context_line":"environ."},{"line_number":148,"context_line":""}],"source_content_type":"text/x-rst","patch_set":5,"id":"bab6814e_de579fc8","line":145,"range":{"start_line":144,"start_character":42,"end_line":145,"end_character":20},"updated":"2016-05-24 15:03:19.000000000","message":"This depends on the keymaster implementation, which comes later. I believe the code permits that different keys are provided for a/c/o. If that is correct, then I would say:\n\nThe keymaster middleware should provide different keys for each object, account, and container.","commit_id":"658fc188b6803ebd990645c4de7ed3610133ee70"},{"author":{"_account_id":11330,"name":"Christian Cachin","email":"cca@zurich.ibm.com","username":"cca88"},"change_message_id":"679bb1f0217af8acd8f4bc40753ce7c27c04b3c7","unresolved":false,"context_lines":[{"line_number":141,"context_line":"--------------"},{"line_number":142,"context_line":""},{"line_number":143,"context_line":"The keymaster middleware is responsible for providing the keys required for"},{"line_number":144,"context_line":"each encryption and decryption operation. Unique keys are provided for each"},{"line_number":145,"context_line":"object and container. These are made available to the encrypter and decrypter"},{"line_number":146,"context_line":"via a callback function that the keymaster installs in the WSGI request"},{"line_number":147,"context_line":"environ."},{"line_number":148,"context_line":""}],"source_content_type":"text/x-rst","patch_set":5,"id":"bab6814e_2cd3da39","line":145,"range":{"start_line":144,"start_character":42,"end_line":145,"end_character":20},"in_reply_to":"bab6814e_0d8a6dbf","updated":"2016-05-25 18:14:56.000000000","message":"OK, but the discussion here is also about the general mechanism, where a future keymaster might provide those. Could we say something nevertheless?\n---\nThe current keymaster provides unique keys for each object and container.  Future keymasters may provide different keys for each account, container, and object.","commit_id":"658fc188b6803ebd990645c4de7ed3610133ee70"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"abe816371b77500d7fec0cb9ea06431d93531cef","unresolved":false,"context_lines":[{"line_number":141,"context_line":"--------------"},{"line_number":142,"context_line":""},{"line_number":143,"context_line":"The keymaster middleware is responsible for providing the keys required for"},{"line_number":144,"context_line":"each encryption and decryption operation. Unique keys are provided for each"},{"line_number":145,"context_line":"object and container. These are made available to the encrypter and decrypter"},{"line_number":146,"context_line":"via a callback function that the keymaster installs in the WSGI request"},{"line_number":147,"context_line":"environ."},{"line_number":148,"context_line":""}],"source_content_type":"text/x-rst","patch_set":5,"id":"bab6814e_9db11992","line":145,"range":{"start_line":144,"start_character":42,"end_line":145,"end_character":20},"in_reply_to":"bab6814e_2cd3da39","updated":"2016-05-26 17:15:08.000000000","message":"I\u0027m reluctant to discuss future features - to do so I\u0027d need to explain why a keymaster would ever need to provide account keys i.e that one day we might encrypt account metadata. I prefer to keep this documentation to statements about what is implemented now rather than any statement of intent.","commit_id":"658fc188b6803ebd990645c4de7ed3610133ee70"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"ceac169c6b395de16790d400b3f68e30852e019e","unresolved":false,"context_lines":[{"line_number":141,"context_line":"--------------"},{"line_number":142,"context_line":""},{"line_number":143,"context_line":"The keymaster middleware is responsible for providing the keys required for"},{"line_number":144,"context_line":"each encryption and decryption operation. Unique keys are provided for each"},{"line_number":145,"context_line":"object and container. These are made available to the encrypter and decrypter"},{"line_number":146,"context_line":"via a callback function that the keymaster installs in the WSGI request"},{"line_number":147,"context_line":"environ."},{"line_number":148,"context_line":""}],"source_content_type":"text/x-rst","patch_set":5,"id":"bab6814e_0d8a6dbf","line":145,"range":{"start_line":144,"start_character":42,"end_line":145,"end_character":20},"in_reply_to":"bab6814e_de579fc8","updated":"2016-05-25 13:17:42.000000000","message":"OK, except we do not currently encrypt any account metadata and therefore we do not require, nor do we provide, an account key. We do not encrypt container metadata but we do require a container key for encrypting the etag values that are stored in the container listing.","commit_id":"658fc188b6803ebd990645c4de7ed3610133ee70"},{"author":{"_account_id":11330,"name":"Christian Cachin","email":"cca@zurich.ibm.com","username":"cca88"},"change_message_id":"58330b60dea82bea61f5cce687e288b00dceac42","unresolved":false,"context_lines":[{"line_number":158,"context_line":""},{"line_number":159,"context_line":"The encrypter uses the object key to `wrap` other randomly generated keys that"},{"line_number":160,"context_line":"are used to encrypt object data. A random key is `wrapped` by encrypting it"},{"line_number":161,"context_line":"using the keymaster provided object key. This makes it safe to then store the"},{"line_number":162,"context_line":"wrapped key alongside object data and metadata."},{"line_number":163,"context_line":""},{"line_number":164,"context_line":"This process of `key wrapping` is performed to enable more efficient re-keying"}],"source_content_type":"text/x-rst","patch_set":5,"id":"bab6814e_7ebaf3c1","line":161,"range":{"start_line":161,"start_character":10,"end_line":161,"end_character":39},"updated":"2016-05-24 15:03:19.000000000","message":"the object key provided by the keymaster.","commit_id":"658fc188b6803ebd990645c4de7ed3610133ee70"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"ceac169c6b395de16790d400b3f68e30852e019e","unresolved":false,"context_lines":[{"line_number":158,"context_line":""},{"line_number":159,"context_line":"The encrypter uses the object key to `wrap` other randomly generated keys that"},{"line_number":160,"context_line":"are used to encrypt object data. A random key is `wrapped` by encrypting it"},{"line_number":161,"context_line":"using the keymaster provided object key. This makes it safe to then store the"},{"line_number":162,"context_line":"wrapped key alongside object data and metadata."},{"line_number":163,"context_line":""},{"line_number":164,"context_line":"This process of `key wrapping` is performed to enable more efficient re-keying"}],"source_content_type":"text/x-rst","patch_set":5,"id":"bab6814e_be0c253c","line":161,"range":{"start_line":161,"start_character":10,"end_line":161,"end_character":39},"in_reply_to":"bab6814e_7ebaf3c1","updated":"2016-05-25 13:17:42.000000000","message":"Done","commit_id":"658fc188b6803ebd990645c4de7ed3610133ee70"},{"author":{"_account_id":11330,"name":"Christian Cachin","email":"cca@zurich.ibm.com","username":"cca88"},"change_message_id":"58330b60dea82bea61f5cce687e288b00dceac42","unresolved":false,"context_lines":[{"line_number":170,"context_line":""},{"line_number":171,"context_line":"For example, as described below, the object body is encrypted using a random"},{"line_number":172,"context_line":"key which is then wrapped using the object key. If re-keying requires the"},{"line_number":173,"context_line":"object key to be replaced the only random key needs to be re-encrypted but not"},{"line_number":174,"context_line":"the object body, which is potentially a large amount of data."},{"line_number":175,"context_line":""},{"line_number":176,"context_line":".. note::"}],"source_content_type":"text/x-rst","patch_set":5,"id":"bab6814e_1ef8077c","line":173,"range":{"start_line":173,"start_character":26,"end_line":173,"end_character":34},"updated":"2016-05-24 15:03:19.000000000","message":"only the","commit_id":"658fc188b6803ebd990645c4de7ed3610133ee70"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"ceac169c6b395de16790d400b3f68e30852e019e","unresolved":false,"context_lines":[{"line_number":170,"context_line":""},{"line_number":171,"context_line":"For example, as described below, the object body is encrypted using a random"},{"line_number":172,"context_line":"key which is then wrapped using the object key. If re-keying requires the"},{"line_number":173,"context_line":"object key to be replaced the only random key needs to be re-encrypted but not"},{"line_number":174,"context_line":"the object body, which is potentially a large amount of data."},{"line_number":175,"context_line":""},{"line_number":176,"context_line":".. note::"}],"source_content_type":"text/x-rst","patch_set":5,"id":"bab6814e_7e9e3db9","line":173,"range":{"start_line":173,"start_character":26,"end_line":173,"end_character":34},"in_reply_to":"bab6814e_1ef8077c","updated":"2016-05-25 13:17:42.000000000","message":"Done","commit_id":"658fc188b6803ebd990645c4de7ed3610133ee70"},{"author":{"_account_id":11330,"name":"Christian Cachin","email":"cca@zurich.ibm.com","username":"cca88"},"change_message_id":"58330b60dea82bea61f5cce687e288b00dceac42","unresolved":false,"context_lines":[{"line_number":194,"context_line":"provided by the keymaster and an IV that is randomly chosen for that metadata"},{"line_number":195,"context_line":"item. For example::"},{"line_number":196,"context_line":""},{"line_number":197,"context_line":"  X-Object-Meta-Private1: value"},{"line_number":198,"context_line":"  X-Object-Meta-Private2: value"},{"line_number":199,"context_line":""},{"line_number":200,"context_line":"are transformed to::"}],"source_content_type":"text/x-rst","patch_set":5,"id":"bab6814e_bea3bb4e","line":197,"range":{"start_line":197,"start_character":26,"end_line":197,"end_character":31},"updated":"2016-05-24 15:03:19.000000000","message":"I would use \"value1\" and \"value2\" here (or \"foo\" and \"bar\") unless you want to make the point that the same value is encrypted twice.","commit_id":"658fc188b6803ebd990645c4de7ed3610133ee70"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"ceac169c6b395de16790d400b3f68e30852e019e","unresolved":false,"context_lines":[{"line_number":194,"context_line":"provided by the keymaster and an IV that is randomly chosen for that metadata"},{"line_number":195,"context_line":"item. For example::"},{"line_number":196,"context_line":""},{"line_number":197,"context_line":"  X-Object-Meta-Private1: value"},{"line_number":198,"context_line":"  X-Object-Meta-Private2: value"},{"line_number":199,"context_line":""},{"line_number":200,"context_line":"are transformed to::"}],"source_content_type":"text/x-rst","patch_set":5,"id":"bab6814e_5e56d9d2","line":197,"range":{"start_line":197,"start_character":26,"end_line":197,"end_character":31},"in_reply_to":"bab6814e_bea3bb4e","updated":"2016-05-25 13:17:42.000000000","message":"Done. I have two example headers to illustrate that different iv\u0027s are chosen for each.","commit_id":"658fc188b6803ebd990645c4de7ed3610133ee70"},{"author":{"_account_id":11330,"name":"Christian Cachin","email":"cca@zurich.ibm.com","username":"cca88"},"change_message_id":"58330b60dea82bea61f5cce687e288b00dceac42","unresolved":false,"context_lines":[{"line_number":203,"context_line":"  X-Object-Meta-Private2: E(value, object_key, header_iv_2)"},{"line_number":204,"context_line":""},{"line_number":205,"context_line":"For each custom user metadata header the encrypter stores the associated"},{"line_number":206,"context_line":"crypto_metadata using an ``X-Object-Transient-Sysmeta-`` header. For the same"},{"line_number":207,"context_line":"example::"},{"line_number":208,"context_line":""},{"line_number":209,"context_line":"  X-Object-Transient-Sysmeta-Crypto-Meta-Private1:{\"iv\": header_iv_1,"}],"source_content_type":"text/x-rst","patch_set":5,"id":"bab6814e_fe7383e5","line":206,"range":{"start_line":206,"start_character":0,"end_line":206,"end_character":15},"updated":"2016-05-24 15:03:19.000000000","message":"crypto-metadata (it was called like this, with -, above)","commit_id":"658fc188b6803ebd990645c4de7ed3610133ee70"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"ceac169c6b395de16790d400b3f68e30852e019e","unresolved":false,"context_lines":[{"line_number":203,"context_line":"  X-Object-Meta-Private2: E(value, object_key, header_iv_2)"},{"line_number":204,"context_line":""},{"line_number":205,"context_line":"For each custom user metadata header the encrypter stores the associated"},{"line_number":206,"context_line":"crypto_metadata using an ``X-Object-Transient-Sysmeta-`` header. For the same"},{"line_number":207,"context_line":"example::"},{"line_number":208,"context_line":""},{"line_number":209,"context_line":"  X-Object-Transient-Sysmeta-Crypto-Meta-Private1:{\"iv\": header_iv_1,"}],"source_content_type":"text/x-rst","patch_set":5,"id":"bab6814e_3eff95ab","line":206,"range":{"start_line":206,"start_character":0,"end_line":206,"end_character":15},"in_reply_to":"bab6814e_fe7383e5","updated":"2016-05-25 13:17:42.000000000","message":"Done, well spotted!","commit_id":"658fc188b6803ebd990645c4de7ed3610133ee70"},{"author":{"_account_id":11330,"name":"Christian Cachin","email":"cca@zurich.ibm.com","username":"cca88"},"change_message_id":"58330b60dea82bea61f5cce687e288b00dceac42","unresolved":false,"context_lines":[{"line_number":219,"context_line":""},{"line_number":220,"context_line":"  body_ciphertext \u003d E(body_plaintext, body_key, body_iv)"},{"line_number":221,"context_line":""},{"line_number":222,"context_line":"The body_key is wrapped using the keymaster provided object key and the same"},{"line_number":223,"context_line":"randomly chosen IV that was used for the object body::"},{"line_number":224,"context_line":""},{"line_number":225,"context_line":"  wrapped_body_key \u003d E(body_key, object_key, body_iv)"},{"line_number":226,"context_line":""}],"source_content_type":"text/x-rst","patch_set":5,"id":"bab6814e_a1a0bea6","line":223,"range":{"start_line":222,"start_character":68,"end_line":223,"end_character":18},"updated":"2016-05-24 15:03:19.000000000","message":"I am not sure if that slipped my attention before, but using the same IV sounds not proper, at first reading. Why is that? If only for simplicity, it would be cleaner to use a value derived from body_iv, such as (body_iv - len(body_key)) because body_iv is simply a counter. Hence this choice will align the body_key encryption to the position of the counter (AES CTR) immediately before the object body.","commit_id":"658fc188b6803ebd990645c4de7ed3610133ee70"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"ceac169c6b395de16790d400b3f68e30852e019e","unresolved":false,"context_lines":[{"line_number":219,"context_line":""},{"line_number":220,"context_line":"  body_ciphertext \u003d E(body_plaintext, body_key, body_iv)"},{"line_number":221,"context_line":""},{"line_number":222,"context_line":"The body_key is wrapped using the keymaster provided object key and the same"},{"line_number":223,"context_line":"randomly chosen IV that was used for the object body::"},{"line_number":224,"context_line":""},{"line_number":225,"context_line":"  wrapped_body_key \u003d E(body_key, object_key, body_iv)"},{"line_number":226,"context_line":""}],"source_content_type":"text/x-rst","patch_set":5,"id":"bab6814e_de992970","line":223,"range":{"start_line":222,"start_character":68,"end_line":223,"end_character":18},"in_reply_to":"bab6814e_a1a0bea6","updated":"2016-05-25 13:17:42.000000000","message":"We will change this. It could be derived as you suggest or randomly chosen and stored separately in the metadata.\n\nhttps://trello.com/c/347pLuzK/112-change-body-key-wrapping-to-use-different-iv-than-body","commit_id":"658fc188b6803ebd990645c4de7ed3610133ee70"},{"author":{"_account_id":11330,"name":"Christian Cachin","email":"cca@zurich.ibm.com","username":"cca88"},"change_message_id":"58330b60dea82bea61f5cce687e288b00dceac42","unresolved":false,"context_lines":[{"line_number":335,"context_line":"Considerations for developers"},{"line_number":336,"context_line":"-----------------------------"},{"line_number":337,"context_line":""},{"line_number":338,"context_line":"Developers should be aware that crypto middlewares rely on the path of an"},{"line_number":339,"context_line":"object remaining unchanged. The keymaster derives keys for containers and"},{"line_number":340,"context_line":"objects based on their paths. The encrypter also uses the object path to derive"},{"line_number":341,"context_line":"an IV for encrypting the ETag. As explained above, this choice of IV is"}],"source_content_type":"text/x-rst","patch_set":5,"id":"bab6814e_e4565405","line":338,"range":{"start_line":338,"start_character":32,"end_line":338,"end_character":50},"updated":"2016-05-24 15:03:19.000000000","message":"better: encryption middlewares","commit_id":"658fc188b6803ebd990645c4de7ed3610133ee70"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"ceac169c6b395de16790d400b3f68e30852e019e","unresolved":false,"context_lines":[{"line_number":335,"context_line":"Considerations for developers"},{"line_number":336,"context_line":"-----------------------------"},{"line_number":337,"context_line":""},{"line_number":338,"context_line":"Developers should be aware that crypto middlewares rely on the path of an"},{"line_number":339,"context_line":"object remaining unchanged. The keymaster derives keys for containers and"},{"line_number":340,"context_line":"objects based on their paths. The encrypter also uses the object path to derive"},{"line_number":341,"context_line":"an IV for encrypting the ETag. As explained above, this choice of IV is"}],"source_content_type":"text/x-rst","patch_set":5,"id":"bab6814e_9e31610e","line":338,"range":{"start_line":338,"start_character":32,"end_line":338,"end_character":50},"in_reply_to":"bab6814e_e4565405","updated":"2016-05-25 13:17:42.000000000","message":"Done","commit_id":"658fc188b6803ebd990645c4de7ed3610133ee70"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"507a34972b345569e47dd3a3cee9909b1b1ad159","unresolved":false,"context_lines":[{"line_number":210,"context_line":"TODO - metadata is now the only place where the object key is used to directly"},{"line_number":211,"context_line":"encrypt user data, rather than a wrapped random key. That is ok because"},{"line_number":212,"context_line":"re-keying metadata is not expensive, but we could equally choose a random key"},{"line_number":213,"context_line":"for metadata."},{"line_number":214,"context_line":""},{"line_number":215,"context_line":"The encrypter encrypts each item of custom user metadata using the object key"},{"line_number":216,"context_line":"provided by the keymaster and an IV that is randomly chosen for that metadata"}],"source_content_type":"text/x-rst","patch_set":8,"id":"9abb7d3a_6c2b36ec","line":213,"updated":"2016-06-03 16:42:56.000000000","message":"removing TODO. doc matches implementation.","commit_id":"8e317eb05c468d60ea0869eae9f35347e3fdb4a8"},{"author":{"_account_id":11330,"name":"Christian Cachin","email":"cca@zurich.ibm.com","username":"cca88"},"change_message_id":"9dbe359409a5af09f62f70c42ff992760fd251f7","unresolved":false,"context_lines":[{"line_number":256,"context_line":"                    \"iv\": body_key_iv}}"},{"line_number":257,"context_line":""},{"line_number":258,"context_line":"Note that in this case there is an extra item of crypto metadata which stores"},{"line_number":259,"context_line":"the wrapped body key and it\u0027s IV."},{"line_number":260,"context_line":""},{"line_number":261,"context_line":"Entity tag"},{"line_number":262,"context_line":"++++++++++"}],"source_content_type":"text/x-rst","patch_set":8,"id":"9abb7d3a_d5cbf170","line":259,"range":{"start_line":259,"start_character":25,"end_line":259,"end_character":29},"updated":"2016-05-29 14:02:09.000000000","message":"it\u0027s \u003d\u003e its","commit_id":"8e317eb05c468d60ea0869eae9f35347e3fdb4a8"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"0c33c1ff4c8abde3b89d409a60550a13db734a99","unresolved":false,"context_lines":[{"line_number":256,"context_line":"                    \"iv\": body_key_iv}}"},{"line_number":257,"context_line":""},{"line_number":258,"context_line":"Note that in this case there is an extra item of crypto metadata which stores"},{"line_number":259,"context_line":"the wrapped body key and it\u0027s IV."},{"line_number":260,"context_line":""},{"line_number":261,"context_line":"Entity tag"},{"line_number":262,"context_line":"++++++++++"}],"source_content_type":"text/x-rst","patch_set":8,"id":"9abb7d3a_b89c395e","line":259,"range":{"start_line":259,"start_character":25,"end_line":259,"end_character":29},"in_reply_to":"9abb7d3a_d5cbf170","updated":"2016-06-03 16:38:05.000000000","message":"Done","commit_id":"8e317eb05c468d60ea0869eae9f35347e3fdb4a8"},{"author":{"_account_id":11330,"name":"Christian Cachin","email":"cca@zurich.ibm.com","username":"cca88"},"change_message_id":"9dbe359409a5af09f62f70c42ff992760fd251f7","unresolved":false,"context_lines":[{"line_number":261,"context_line":"Entity tag"},{"line_number":262,"context_line":"++++++++++"},{"line_number":263,"context_line":""},{"line_number":264,"context_line":"While encrypting the object body the encrypter also calculates the hexdigest of"},{"line_number":265,"context_line":"the md5 sum of the plaintext body. This value is encrypted using a keymaster"},{"line_number":266,"context_line":"provided container key, and an IV that is derived from the object\u0027s path, and"},{"line_number":267,"context_line":"saved as an item of system metadata::"},{"line_number":268,"context_line":""}],"source_content_type":"text/x-rst","patch_set":8,"id":"9abb7d3a_558d4130","line":265,"range":{"start_line":264,"start_character":63,"end_line":265,"end_character":15},"updated":"2016-05-29 14:02:09.000000000","message":"language nit: the md5 sum is already a digest.\nBetter alternatives:\n\n\u003d\u003e the ETag (md5 digest) of\n\u003d\u003e the MD5 checksum of\n\u003d\u003e the md5 digest in hex of","commit_id":"8e317eb05c468d60ea0869eae9f35347e3fdb4a8"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"0c33c1ff4c8abde3b89d409a60550a13db734a99","unresolved":false,"context_lines":[{"line_number":261,"context_line":"Entity tag"},{"line_number":262,"context_line":"++++++++++"},{"line_number":263,"context_line":""},{"line_number":264,"context_line":"While encrypting the object body the encrypter also calculates the hexdigest of"},{"line_number":265,"context_line":"the md5 sum of the plaintext body. This value is encrypted using a keymaster"},{"line_number":266,"context_line":"provided container key, and an IV that is derived from the object\u0027s path, and"},{"line_number":267,"context_line":"saved as an item of system metadata::"},{"line_number":268,"context_line":""}],"source_content_type":"text/x-rst","patch_set":8,"id":"9abb7d3a_8ce01afc","line":265,"range":{"start_line":264,"start_character":63,"end_line":265,"end_character":15},"in_reply_to":"9abb7d3a_558d4130","updated":"2016-06-03 16:38:05.000000000","message":"Done","commit_id":"8e317eb05c468d60ea0869eae9f35347e3fdb4a8"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"507a34972b345569e47dd3a3cee9909b1b1ad159","unresolved":false,"context_lines":[{"line_number":308,"context_line":"Decrypter operation"},{"line_number":309,"context_line":"-------------------"},{"line_number":310,"context_line":""},{"line_number":311,"context_line":"TODO - add mention of crypto-override flag in environ."},{"line_number":312,"context_line":""},{"line_number":313,"context_line":"For each GET or HEAD request to an object, the decrypter inspects the response"},{"line_number":314,"context_line":"for encrypted items (revealed by crypto metadata headers), and if any are"}],"source_content_type":"text/x-rst","patch_set":8,"id":"9abb7d3a_8c0f7a87","line":311,"updated":"2016-06-03 16:42:56.000000000","message":"remove TODO. The crypto-override flag is no longer used between keymaster and decrypter (or encrypter)","commit_id":"8e317eb05c468d60ea0869eae9f35347e3fdb4a8"}]}