)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":5293,"name":"Nikhil","email":"SlickNik@gmail.com","username":"slicknik"},"change_message_id":"c89f6c02f9656981bb4a237b76f3f042a364d519","unresolved":false,"context_lines":[{"line_number":7,"context_line":"Backup to swift using wrong large object type"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"Trove backups are stored in Swift as Dynamic Large Objects. This is not"},{"line_number":10,"context_line":"ideal, and could cause restore to fail as DLOs may not return all of"},{"line_number":11,"context_line":"their parts. Static Large Objects should be used instead. SLOs perform"},{"line_number":12,"context_line":"checksum and object size validation and can reduce object download times"},{"line_number":13,"context_line":"significantly."}],"source_content_type":"text/x-gerrit-commit-message","patch_set":11,"id":"bab6814e_6c7cc272","line":10,"updated":"2016-06-01 09:27:12.000000000","message":"This would only happen if a user deleted one of the download segments. In the case of SLO, if the user did the same thing, you\u0027d end up with an incomplete download -- also resulting in an unusable backup.","commit_id":"6672fe868a5e98de308ec884b753878afb4a475b"}],"releasenotes/notes/slo-backups-3c35135316f837e1.yaml":[{"author":{"_account_id":9664,"name":"amrith","email":"amrith.kumar@gmail.com","username":"amrith"},"change_message_id":"acb81fed91b9152c5b9871b4397d70177f285ccc","unresolved":false,"context_lines":[{"line_number":6,"context_line":"    segment size of the SLO. Backups that are smaller"},{"line_number":7,"context_line":"    than the segment size will be uploaded as regular"},{"line_number":8,"context_line":"    objects. This is an improvement over old Dynamic"},{"line_number":9,"context_line":"    Large Object implementation."}],"source_content_type":"text/x-yaml","patch_set":9,"id":"1a122d0e_e5b43d36","line":9,"updated":"2016-04-19 14:25:53.000000000","message":"please put a bug number in the release note (Bug NNNNN) at the end as I described in my email :)\n\nHeck, release notes are the new \u0027typo in your commit message\u0027 review comment.","commit_id":"58c2ed319503d2bb12198e660b1082dceb48a7ce"},{"author":{"_account_id":15321,"name":"Matthew Van Dijk","email":"mvandijk@tesora.com","username":"mvandijk"},"change_message_id":"c9eae703aa0a19d175e107c12126a6be11e3d048","unresolved":false,"context_lines":[{"line_number":6,"context_line":"    segment size of the SLO. Backups that are smaller"},{"line_number":7,"context_line":"    than the segment size will be uploaded as regular"},{"line_number":8,"context_line":"    objects. This is an improvement over old Dynamic"},{"line_number":9,"context_line":"    Large Object implementation."}],"source_content_type":"text/x-yaml","patch_set":9,"id":"1a122d0e_85e629ae","line":9,"in_reply_to":"1a122d0e_e5b43d36","updated":"2016-04-19 14:31:14.000000000","message":"Done.","commit_id":"58c2ed319503d2bb12198e660b1082dceb48a7ce"}],"trove/common/strategies/storage/swift.py":[{"author":{"_account_id":6159,"name":"Flavio Percoco Premoli","display_name":"flaper87","email":"flavio.percoco@flyrlabs.com","username":"flaper87"},"change_message_id":"2894fb114ac1daf2474e1f7c066d9c39e6face9d","unresolved":false,"context_lines":[{"line_number":123,"context_line":""},{"line_number":124,"context_line":"        # Wrap the output of the backup process to segment it for swift"},{"line_number":125,"context_line":"        stream_reader \u003d StreamReader(stream, filename, MAX_FILE_SIZE)"},{"line_number":126,"context_line":"        LOG.debug(\u0027Using segment size %s\u0027 % stream_reader.max_file_size)"},{"line_number":127,"context_line":""},{"line_number":128,"context_line":"        url \u003d self.connection.url"},{"line_number":129,"context_line":"        # Full location where the backup manifest is stored"}],"source_content_type":"text/x-python","patch_set":11,"id":"dab17558_860f732d","line":126,"updated":"2016-05-08 20:49:25.000000000","message":"Do we need a debug line for this? Can `max_file_size` be different from MAX_FILE_SIZE?","commit_id":"6672fe868a5e98de308ec884b753878afb4a475b"},{"author":{"_account_id":6159,"name":"Flavio Percoco Premoli","display_name":"flaper87","email":"flavio.percoco@flyrlabs.com","username":"flaper87"},"change_message_id":"2894fb114ac1daf2474e1f7c066d9c39e6face9d","unresolved":false,"context_lines":[{"line_number":135,"context_line":"        # Read from the stream and write to the container in swift"},{"line_number":136,"context_line":"        while not stream_reader.end_of_file:"},{"line_number":137,"context_line":"            LOG.debug(\u0027Saving segment %s.\u0027 % stream_reader.segment)"},{"line_number":138,"context_line":"            path \u003d stream_reader.segment_path"},{"line_number":139,"context_line":"            etag \u003d self.connection.put_object(BACKUP_CONTAINER,"},{"line_number":140,"context_line":"                                              stream_reader.segment,"},{"line_number":141,"context_line":"                                              stream_reader)"}],"source_content_type":"text/x-python","patch_set":11,"id":"dab17558_e61e3f62","line":138,"updated":"2016-05-08 20:49:25.000000000","message":"nit: We should be able to do away with the `path` var by using `stream_reader.segment_path` in L154","commit_id":"6672fe868a5e98de308ec884b753878afb4a475b"},{"author":{"_account_id":6159,"name":"Flavio Percoco Premoli","display_name":"flaper87","email":"flavio.percoco@flyrlabs.com","username":"flaper87"},"change_message_id":"2894fb114ac1daf2474e1f7c066d9c39e6face9d","unresolved":false,"context_lines":[{"line_number":176,"context_line":""},{"line_number":177,"context_line":"        LOG.debug(\u0027Metadata headers: %s\u0027 % str(headers))"},{"line_number":178,"context_line":"        if large_object:"},{"line_number":179,"context_line":"            LOG.info(_(\u0027Creating the manifest file.\u0027))"},{"line_number":180,"context_line":"            manifest_data \u003d json.dumps(segment_results)"},{"line_number":181,"context_line":"            LOG.debug(\u0027Manifest contents: %s\u0027 % manifest_data)"},{"line_number":182,"context_line":"            # The etag returned from the manifest PUT is the checksum of the"}],"source_content_type":"text/x-python","patch_set":11,"id":"dab17558_c6197b7b","line":179,"updated":"2016-05-08 20:49:25.000000000","message":"This could probably be `debug`","commit_id":"6672fe868a5e98de308ec884b753878afb4a475b"}],"trove/guestagent/strategies/storage/swift.py":[{"author":{"_account_id":14576,"name":"Petr Malik","email":"pmalik@tesora.com","username":"pmalik"},"change_message_id":"a6bd5afc8837ddd4f8a7e23644fb0f180c818b47","unresolved":false,"context_lines":[{"line_number":163,"context_line":"        # All segments uploaded."},{"line_number":164,"context_line":"        # An SLO will be generated if the backup was more than one segment in"},{"line_number":165,"context_line":"        # length."},{"line_number":166,"context_line":"        large_object \u003d True if num_segments \u003e 1 else False"},{"line_number":167,"context_line":"        if large_object:"},{"line_number":168,"context_line":"            LOG.info(_(\u0027Creating the manifest file.\u0027))"},{"line_number":169,"context_line":"            manifest_data \u003d json.dumps(segment_results)"}],"source_content_type":"text/x-python","patch_set":1,"id":"ba15a1d1_fbed2b26","line":166,"updated":"2015-09-12 00:56:34.000000000","message":"why not just\n\nlarge_object \u003d num_segments \u003e 1","commit_id":"b24d94bcaa5b6afe35901b7e0706362f3fbb9e15"},{"author":{"_account_id":15321,"name":"Matthew Van Dijk","email":"mvandijk@tesora.com","username":"mvandijk"},"change_message_id":"51a7932f71b612706541317cc84a8cfd1b2a355d","unresolved":false,"context_lines":[{"line_number":163,"context_line":"        # All segments uploaded."},{"line_number":164,"context_line":"        # An SLO will be generated if the backup was more than one segment in"},{"line_number":165,"context_line":"        # length."},{"line_number":166,"context_line":"        large_object \u003d True if num_segments \u003e 1 else False"},{"line_number":167,"context_line":"        if large_object:"},{"line_number":168,"context_line":"            LOG.info(_(\u0027Creating the manifest file.\u0027))"},{"line_number":169,"context_line":"            manifest_data \u003d json.dumps(segment_results)"}],"source_content_type":"text/x-python","patch_set":1,"id":"ba15a1d1_dec61516","line":166,"in_reply_to":"ba15a1d1_fbed2b26","updated":"2015-09-12 03:38:43.000000000","message":"Done","commit_id":"b24d94bcaa5b6afe35901b7e0706362f3fbb9e15"},{"author":{"_account_id":14576,"name":"Petr Malik","email":"pmalik@tesora.com","username":"pmalik"},"change_message_id":"a6bd5afc8837ddd4f8a7e23644fb0f180c818b47","unresolved":false,"context_lines":[{"line_number":208,"context_line":"                {\u0027tag\u0027: etag, \u0027checksum\u0027: final_swift_checksum})"},{"line_number":209,"context_line":"            return False, \"Error saving data to Swift!\", None, location"},{"line_number":210,"context_line":""},{"line_number":211,"context_line":"        if not large_object:"},{"line_number":212,"context_line":"            # Delete the old segment file that was copied"},{"line_number":213,"context_line":"            LOG.debug(\u0027Deleting the old segment file %s.\u0027"},{"line_number":214,"context_line":"                      % stream_reader.first_segment)"}],"source_content_type":"text/x-python","patch_set":1,"id":"ba15a1d1_3bbba310","line":211,"updated":"2015-09-12 00:56:34.000000000","message":"Could this possibly run in a finally clause? I guess we want to do the cleanup even if something fails.","commit_id":"b24d94bcaa5b6afe35901b7e0706362f3fbb9e15"},{"author":{"_account_id":15321,"name":"Matthew Van Dijk","email":"mvandijk@tesora.com","username":"mvandijk"},"change_message_id":"51a7932f71b612706541317cc84a8cfd1b2a355d","unresolved":false,"context_lines":[{"line_number":208,"context_line":"                {\u0027tag\u0027: etag, \u0027checksum\u0027: final_swift_checksum})"},{"line_number":209,"context_line":"            return False, \"Error saving data to Swift!\", None, location"},{"line_number":210,"context_line":""},{"line_number":211,"context_line":"        if not large_object:"},{"line_number":212,"context_line":"            # Delete the old segment file that was copied"},{"line_number":213,"context_line":"            LOG.debug(\u0027Deleting the old segment file %s.\u0027"},{"line_number":214,"context_line":"                      % stream_reader.first_segment)"}],"source_content_type":"text/x-python","patch_set":1,"id":"ba15a1d1_7ead8930","line":211,"in_reply_to":"ba15a1d1_3bbba310","updated":"2015-09-12 03:38:43.000000000","message":"Not sure if that is the correct approach. I\u0027m actually going to move this up to right after we do the copy, I don\u0027t see a reason to keep it this long.","commit_id":"b24d94bcaa5b6afe35901b7e0706362f3fbb9e15"},{"author":{"_account_id":330,"name":"John Dickinson","email":"me@not.mn","username":"notmyname"},"change_message_id":"9f0e6aae8ea89f1ba10190e67e71d1fd1eb22a3d","unresolved":false,"context_lines":[{"line_number":284,"context_line":"            headers[self._set_attr(key)] \u003d value"},{"line_number":285,"context_line":""},{"line_number":286,"context_line":"        LOG.info(_(\"Writing metadata: %s\"), str(headers))"},{"line_number":287,"context_line":"        self.connection.post_object(container, filename, headers\u003dheaders)"}],"source_content_type":"text/x-python","patch_set":1,"id":"da20952f_27de7dda","line":287,"updated":"2015-09-11 20:27:02.000000000","message":"Generally using POST with DLO or SLO is a bad idea (with swift today). If you know the metadata up front, you should set it during the original PUT.","commit_id":"b24d94bcaa5b6afe35901b7e0706362f3fbb9e15"},{"author":{"_account_id":15321,"name":"Matthew Van Dijk","email":"mvandijk@tesora.com","username":"mvandijk"},"change_message_id":"f4d053efd008fa733160b9bfb2ede0b32a682faa","unresolved":false,"context_lines":[{"line_number":284,"context_line":"            headers[self._set_attr(key)] \u003d value"},{"line_number":285,"context_line":""},{"line_number":286,"context_line":"        LOG.info(_(\"Writing metadata: %s\"), str(headers))"},{"line_number":287,"context_line":"        self.connection.post_object(container, filename, headers\u003dheaders)"}],"source_content_type":"text/x-python","patch_set":1,"id":"da20952f_a204bb13","line":287,"in_reply_to":"da20952f_27de7dda","updated":"2015-09-11 20:33:32.000000000","message":"I\u0027ll look into it.","commit_id":"b24d94bcaa5b6afe35901b7e0706362f3fbb9e15"},{"author":{"_account_id":10440,"name":"Alex Tomic","email":"atomic777@gmail.com","username":"atomic77"},"change_message_id":"adb1d26c4692e3819547dde26c7b1c9fed3d13c3","unresolved":false,"context_lines":[{"line_number":189,"context_line":""},{"line_number":190,"context_line":"            # Validation checksum is the Swift Checksum"},{"line_number":191,"context_line":"            final_swift_checksum \u003d swift_checksum.hexdigest()"},{"line_number":192,"context_line":"        else:"},{"line_number":193,"context_line":"            LOG.info(_(\u0027Backup fits in a single segment. Moving segment \u0027"},{"line_number":194,"context_line":"                       \u0027%(segment)s to %(filename)s.\u0027)"},{"line_number":195,"context_line":"                     % {\u0027segment\u0027: stream_reader.first_segment,"}],"source_content_type":"text/x-python","patch_set":5,"id":"fa80f949_050da78f","line":192,"updated":"2015-11-05 22:00:09.000000000","message":"Is this whole block really necessary? \n\nI understand it may seem weird to store large objects with 1 segment, but if you\u0027ve already built the logic to do the segmentation, and there\u0027s little practical difference from swift\u0027s perspective, why maintain another code path for this special case? Notice that you have even more code to handle this \u0027simple\u0027 case than the block above","commit_id":"583b86e7ae2233ab4930268dbb4ef9f19a44b3eb"},{"author":{"_account_id":10440,"name":"Alex Tomic","email":"atomic777@gmail.com","username":"atomic77"},"change_message_id":"234a6a078bcbd67fcee6635031aea2c43f2c8acd","unresolved":false,"context_lines":[{"line_number":189,"context_line":""},{"line_number":190,"context_line":"            # Validation checksum is the Swift Checksum"},{"line_number":191,"context_line":"            final_swift_checksum \u003d swift_checksum.hexdigest()"},{"line_number":192,"context_line":"        else:"},{"line_number":193,"context_line":"            LOG.info(_(\u0027Backup fits in a single segment. Moving segment \u0027"},{"line_number":194,"context_line":"                       \u0027%(segment)s to %(filename)s.\u0027)"},{"line_number":195,"context_line":"                     % {\u0027segment\u0027: stream_reader.first_segment,"}],"source_content_type":"text/x-python","patch_set":5,"id":"da85f559_5dee9eb5","line":192,"in_reply_to":"da85f559_0678c88e","updated":"2015-11-12 22:59:09.000000000","message":"Interesting, seems like the default minimum segment size is 1mb. TIL \n\n[1] http://docs.openstack.org/developer/swift/overview_large_objects.html","commit_id":"583b86e7ae2233ab4930268dbb4ef9f19a44b3eb"},{"author":{"_account_id":15321,"name":"Matthew Van Dijk","email":"mvandijk@tesora.com","username":"mvandijk"},"change_message_id":"21882723190e3bcb4578bceda0a6017b9e0c1744","unresolved":false,"context_lines":[{"line_number":189,"context_line":""},{"line_number":190,"context_line":"            # Validation checksum is the Swift Checksum"},{"line_number":191,"context_line":"            final_swift_checksum \u003d swift_checksum.hexdigest()"},{"line_number":192,"context_line":"        else:"},{"line_number":193,"context_line":"            LOG.info(_(\u0027Backup fits in a single segment. Moving segment \u0027"},{"line_number":194,"context_line":"                       \u0027%(segment)s to %(filename)s.\u0027)"},{"line_number":195,"context_line":"                     % {\u0027segment\u0027: stream_reader.first_segment,"}],"source_content_type":"text/x-python","patch_set":5,"id":"da85f559_0678c88e","line":192,"in_reply_to":"fa80f949_050da78f","updated":"2015-11-12 22:01:55.000000000","message":"It is necessary as there is a minumum size for SLO, and if the total size is less than that the manifest create will fail. This code block is needed in that case. The segment size could be way larger than that minimum SLO size, but seeing as we have no idea of the size of the object until the stream is done, we cannot make the decision beforehand.","commit_id":"583b86e7ae2233ab4930268dbb4ef9f19a44b3eb"},{"author":{"_account_id":14576,"name":"Petr Malik","email":"pmalik@tesora.com","username":"pmalik"},"change_message_id":"86d82a0d3ca0ca1362ac320c5218ec41fba5a3cd","unresolved":false,"context_lines":[{"line_number":203,"context_line":"            # Delete the old segment file that was copied"},{"line_number":204,"context_line":"            LOG.debug(\u0027Deleting the old segment file %s.\u0027"},{"line_number":205,"context_line":"                      % stream_reader.first_segment)"},{"line_number":206,"context_line":"            self.connection.delete_object(BACKUP_CONTAINER,"},{"line_number":207,"context_line":"                                          stream_reader.first_segment)"},{"line_number":208,"context_line":"            final_swift_checksum \u003d segment_result[\u0027etag\u0027]"},{"line_number":209,"context_line":""}],"source_content_type":"text/x-python","patch_set":5,"id":"da85f559_fa73a64b","line":206,"updated":"2015-11-09 15:32:36.000000000","message":"Should not this execute in a finally block to always perform the cleanup, just in case the put copy above fails?","commit_id":"583b86e7ae2233ab4930268dbb4ef9f19a44b3eb"},{"author":{"_account_id":15321,"name":"Matthew Van Dijk","email":"mvandijk@tesora.com","username":"mvandijk"},"change_message_id":"21882723190e3bcb4578bceda0a6017b9e0c1744","unresolved":false,"context_lines":[{"line_number":203,"context_line":"            # Delete the old segment file that was copied"},{"line_number":204,"context_line":"            LOG.debug(\u0027Deleting the old segment file %s.\u0027"},{"line_number":205,"context_line":"                      % stream_reader.first_segment)"},{"line_number":206,"context_line":"            self.connection.delete_object(BACKUP_CONTAINER,"},{"line_number":207,"context_line":"                                          stream_reader.first_segment)"},{"line_number":208,"context_line":"            final_swift_checksum \u003d segment_result[\u0027etag\u0027]"},{"line_number":209,"context_line":""}],"source_content_type":"text/x-python","patch_set":5,"id":"da85f559_465ed013","line":206,"in_reply_to":"da85f559_fa73a64b","updated":"2015-11-12 22:01:55.000000000","message":"That would be a new feature and OOS of this work. That should probably be done below when the etag is checked and we raise an exception.","commit_id":"583b86e7ae2233ab4930268dbb4ef9f19a44b3eb"}],"trove/taskmanager/models.py":[{"author":{"_account_id":330,"name":"John Dickinson","email":"me@not.mn","username":"notmyname"},"change_message_id":"9f0e6aae8ea89f1ba10190e67e71d1fd1eb22a3d","unresolved":false,"context_lines":[{"line_number":1303,"context_line":"            LOG.debug(\"Deleting large object file: %(cont)s/%(filename)s\" %"},{"line_number":1304,"context_line":"                      {\u0027cont\u0027: container, \u0027filename\u0027: filename})"},{"line_number":1305,"context_line":"            client.delete_object(container, filename,"},{"line_number":1306,"context_line":"                                 query_string\u003d\u0027multipart-manifest\u003ddelete\u0027)"},{"line_number":1307,"context_line":"        else:"},{"line_number":1308,"context_line":"            # Single object"},{"line_number":1309,"context_line":"            LOG.debug(\"Deleting object file: %(cont)s/%(filename)s\" %"}],"source_content_type":"text/x-python","patch_set":1,"id":"da20952f_87550953","line":1306,"updated":"2015-09-11 20:27:02.000000000","message":"why did you stop deleting the segments here?","commit_id":"b24d94bcaa5b6afe35901b7e0706362f3fbb9e15"},{"author":{"_account_id":15321,"name":"Matthew Van Dijk","email":"mvandijk@tesora.com","username":"mvandijk"},"change_message_id":"f4d053efd008fa733160b9bfb2ede0b32a682faa","unresolved":false,"context_lines":[{"line_number":1303,"context_line":"            LOG.debug(\"Deleting large object file: %(cont)s/%(filename)s\" %"},{"line_number":1304,"context_line":"                      {\u0027cont\u0027: container, \u0027filename\u0027: filename})"},{"line_number":1305,"context_line":"            client.delete_object(container, filename,"},{"line_number":1306,"context_line":"                                 query_string\u003d\u0027multipart-manifest\u003ddelete\u0027)"},{"line_number":1307,"context_line":"        else:"},{"line_number":1308,"context_line":"            # Single object"},{"line_number":1309,"context_line":"            LOG.debug(\"Deleting object file: %(cont)s/%(filename)s\" %"}],"source_content_type":"text/x-python","patch_set":1,"id":"da20952f_6709d530","line":1306,"in_reply_to":"da20952f_87550953","updated":"2015-09-11 20:33:32.000000000","message":"SLO does it for you with that query parameter. http://docs.openstack.org/developer/swift/middleware.html#deleting-a-large-object","commit_id":"b24d94bcaa5b6afe35901b7e0706362f3fbb9e15"},{"author":{"_account_id":10440,"name":"Alex Tomic","email":"atomic777@gmail.com","username":"atomic77"},"change_message_id":"adb1d26c4692e3819547dde26c7b1c9fed3d13c3","unresolved":false,"context_lines":[{"line_number":1304,"context_line":"                      {\u0027cont\u0027: container, \u0027filename\u0027: filename})"},{"line_number":1305,"context_line":"            client.delete_object(container, filename,"},{"line_number":1306,"context_line":"                                 query_string\u003d\u0027multipart-manifest\u003ddelete\u0027)"},{"line_number":1307,"context_line":"        else:"},{"line_number":1308,"context_line":"            # Single object"},{"line_number":1309,"context_line":"            LOG.debug(\"Deleting object file: %(cont)s/%(filename)s\" %"},{"line_number":1310,"context_line":"                      {\u0027cont\u0027: container, \u0027filename\u0027: filename})"}],"source_content_type":"text/x-python","patch_set":5,"id":"fa80f949_e522e300","line":1307,"updated":"2015-11-05 22:00:09.000000000","message":"Again, another code path to handle the simple case","commit_id":"583b86e7ae2233ab4930268dbb4ef9f19a44b3eb"},{"author":{"_account_id":15321,"name":"Matthew Van Dijk","email":"mvandijk@tesora.com","username":"mvandijk"},"change_message_id":"21882723190e3bcb4578bceda0a6017b9e0c1744","unresolved":false,"context_lines":[{"line_number":1304,"context_line":"                      {\u0027cont\u0027: container, \u0027filename\u0027: filename})"},{"line_number":1305,"context_line":"            client.delete_object(container, filename,"},{"line_number":1306,"context_line":"                                 query_string\u003d\u0027multipart-manifest\u003ddelete\u0027)"},{"line_number":1307,"context_line":"        else:"},{"line_number":1308,"context_line":"            # Single object"},{"line_number":1309,"context_line":"            LOG.debug(\"Deleting object file: %(cont)s/%(filename)s\" %"},{"line_number":1310,"context_line":"                      {\u0027cont\u0027: container, \u0027filename\u0027: filename})"}],"source_content_type":"text/x-python","patch_set":5,"id":"da85f559_26b86c2a","line":1307,"in_reply_to":"fa80f949_e522e300","updated":"2015-11-12 22:01:55.000000000","message":"It is necessary as we have 2 types of backup objects - large or singular.","commit_id":"583b86e7ae2233ab4930268dbb4ef9f19a44b3eb"}]}
