)]}'
{"releasenotes/notes/add-tool-for-migrating-data-to-gnocchi-cea8d4db68ce03d0.yaml":[{"author":{"_account_id":1669,"name":"Julien Danjou","display_name":"jd","email":"julien@danjou.info","username":"jdanjou"},"change_message_id":"8f9ccdaa028711e167cd822e3843d8eab7dbdbf2","unresolved":false,"context_lines":[{"line_number":3,"context_line":"  - \u003e"},{"line_number":4,"context_line":"    Add a tool for migratinig metrics data from Ceilometer\u0027s native storage"},{"line_number":5,"context_line":"    to Gnocchi. Since we have deprecated Ceilometer API and the Gnocchi will"},{"line_number":6,"context_line":"    be the recommended metrics data storage backend."},{"line_number":7,"context_line":""}],"source_content_type":"text/x-yaml","patch_set":8,"id":"ff0f0b1f_09de596c","line":6,"updated":"2017-05-23 07:03:44.000000000","message":"it\u0027d be cool to say where the tool actually is","commit_id":"1a2b90ffd624c3f555de08baa14a444a25ba3432"}],"tools/migrate_data_to_gnocchi.py":[{"author":{"_account_id":1669,"name":"Julien Danjou","display_name":"jd","email":"julien@danjou.info","username":"jdanjou"},"change_message_id":"d714fff882f2414e692e0241929870a3c7b736fe","unresolved":false,"context_lines":[{"line_number":25,"context_line":"from ceilometer.storage import impl_sqlalchemy"},{"line_number":26,"context_line":""},{"line_number":27,"context_line":""},{"line_number":28,"context_line":"class ProgressBar(object):"},{"line_number":29,"context_line":"    def __init__(self, count\u003d0, total\u003d0, width\u003d70):"},{"line_number":30,"context_line":"        self.count \u003d count"},{"line_number":31,"context_line":"        self.total \u003d total"}],"source_content_type":"text/x-python","patch_set":2,"id":"1a430d35_8665d2e2","line":28,"updated":"2017-02-03 11:01:55.000000000","message":"pretty sure there\u0027s some python module out there you could use :)","commit_id":"a2e17f873a8c3e8bb24db6309354cb0603f3ee34"},{"author":{"_account_id":1669,"name":"Julien Danjou","display_name":"jd","email":"julien@danjou.info","username":"jdanjou"},"change_message_id":"d714fff882f2414e692e0241929870a3c7b736fe","unresolved":false,"context_lines":[{"line_number":94,"context_line":""},{"line_number":95,"context_line":""},{"line_number":96,"context_line":"def count_samples(storage_conn):"},{"line_number":97,"context_line":"    if isinstance(storage_conn, impl_sqlalchemy.Connection):"},{"line_number":98,"context_line":"        from ceilometer.storage.sqlalchemy import models"},{"line_number":99,"context_line":"        session \u003d storage_conn._engine_facade.get_session()"},{"line_number":100,"context_line":"        query \u003d session.query(models.Sample.id)"}],"source_content_type":"text/x-python","patch_set":2,"id":"1a430d35_a6b9ee30","line":97,"updated":"2017-02-03 11:01:55.000000000","message":"you may want to raise something if it\u0027s false otherwise you\u0027ll return None which will fail later I guess","commit_id":"a2e17f873a8c3e8bb24db6309354cb0603f3ee34"},{"author":{"_account_id":11564,"name":"Chris Dent","email":"cdent@anticdent.org","username":"chdent"},"change_message_id":"6756f3e0f1c6a496b2bca00965b2b28fc0919217","unresolved":false,"context_lines":[{"line_number":52,"context_line":"    parser.add_argument("},{"line_number":53,"context_line":"        \u0027--native-metering-connection\u0027,"},{"line_number":54,"context_line":"        required\u003dTrue,"},{"line_number":55,"context_line":"        help\u003d\u0027The database connection url of native storage backend. \u0027"},{"line_number":56,"context_line":"             \u0027e.g. mysql+pymysql://root:password@127.0.0.1/ceilometer?charset\u0027"},{"line_number":57,"context_line":"             \u0027\u003dutf8\u0027,"},{"line_number":58,"context_line":"    )"}],"source_content_type":"text/x-python","patch_set":5,"id":"ba2be162_c21a787c","line":55,"updated":"2017-03-06 12:22:17.000000000","message":"What does \"native\" mean here? It seems confusing to me. Do you mean \"built in\"?","commit_id":"ad7856ec992e871dc7a265154e6a669d89f4ad98"},{"author":{"_account_id":8290,"name":"liusheng","email":"liusheng2048@gmail.com","username":"LiuSheng"},"change_message_id":"25fe016e6eeccce1b5fd0ff043ed011f0875acbf","unresolved":false,"context_lines":[{"line_number":52,"context_line":"    parser.add_argument("},{"line_number":53,"context_line":"        \u0027--native-metering-connection\u0027,"},{"line_number":54,"context_line":"        required\u003dTrue,"},{"line_number":55,"context_line":"        help\u003d\u0027The database connection url of native storage backend. \u0027"},{"line_number":56,"context_line":"             \u0027e.g. mysql+pymysql://root:password@127.0.0.1/ceilometer?charset\u0027"},{"line_number":57,"context_line":"             \u0027\u003dutf8\u0027,"},{"line_number":58,"context_line":"    )"}],"source_content_type":"text/x-python","patch_set":5,"id":"ba2be162_5d3d8929","line":55,"in_reply_to":"ba2be162_c21a787c","updated":"2017-03-06 12:28:29.000000000","message":"that means the storage backends except gnocchi, \u0027built in\u0027 is more explicit ?","commit_id":"ad7856ec992e871dc7a265154e6a669d89f4ad98"},{"author":{"_account_id":11564,"name":"Chris Dent","email":"cdent@anticdent.org","username":"chdent"},"change_message_id":"6756f3e0f1c6a496b2bca00965b2b28fc0919217","unresolved":false,"context_lines":[{"line_number":57,"context_line":"             \u0027\u003dutf8\u0027,"},{"line_number":58,"context_line":"    )"},{"line_number":59,"context_line":"    parser.add_argument("},{"line_number":60,"context_line":"        \u0027--ceilometer-config-file\u0027,"},{"line_number":61,"context_line":"        help\u003d\"The config file of ceilometer, it is main used for gnocchi \""},{"line_number":62,"context_line":"             \"dispatcher to init gnocchiclient with the service credentials \""},{"line_number":63,"context_line":"             \"defined in the ceilometer config file. Default as \""}],"source_content_type":"text/x-python","patch_set":5,"id":"ba2be162_fd563dcc","line":60,"updated":"2017-03-06 12:22:17.000000000","message":"It\u0027s not clear where this is being used?","commit_id":"ad7856ec992e871dc7a265154e6a669d89f4ad98"},{"author":{"_account_id":8290,"name":"liusheng","email":"liusheng2048@gmail.com","username":"LiuSheng"},"change_message_id":"25fe016e6eeccce1b5fd0ff043ed011f0875acbf","unresolved":false,"context_lines":[{"line_number":57,"context_line":"             \u0027\u003dutf8\u0027,"},{"line_number":58,"context_line":"    )"},{"line_number":59,"context_line":"    parser.add_argument("},{"line_number":60,"context_line":"        \u0027--ceilometer-config-file\u0027,"},{"line_number":61,"context_line":"        help\u003d\"The config file of ceilometer, it is main used for gnocchi \""},{"line_number":62,"context_line":"             \"dispatcher to init gnocchiclient with the service credentials \""},{"line_number":63,"context_line":"             \"defined in the ceilometer config file. Default as \""}],"source_content_type":"text/x-python","patch_set":5,"id":"ba2be162_3dde15dd","line":60,"in_reply_to":"ba2be162_fd563dcc","updated":"2017-03-06 12:28:29.000000000","message":"it is used in gnocchi_conf \u003d service.prepare_service([]) and provide the gnocchi credentials.","commit_id":"ad7856ec992e871dc7a265154e6a669d89f4ad98"},{"author":{"_account_id":11564,"name":"Chris Dent","email":"cdent@anticdent.org","username":"chdent"},"change_message_id":"6756f3e0f1c6a496b2bca00965b2b28fc0919217","unresolved":false,"context_lines":[{"line_number":68,"context_line":"        default\u003d\u0027gnocchi-data-migration.log\u0027,"},{"line_number":69,"context_line":"        help\u003d\"The log file to record messages during migration.\", )"},{"line_number":70,"context_line":"    parser.add_argument("},{"line_number":71,"context_line":"        \u0027--file-missing-messages\u0027,"},{"line_number":72,"context_line":"        default\u003d\u0027gnocchi-data-migration-missing.msg\u0027,"},{"line_number":73,"context_line":"        help\u003d\"A file specified to store the \u0027message_id\u0027 of sample which was \""},{"line_number":74,"context_line":"             \"failed to be migrated.\", )"}],"source_content_type":"text/x-python","patch_set":5,"id":"ba2be162_7dc7ad63","line":71,"updated":"2017-03-06 12:22:17.000000000","message":"or this","commit_id":"ad7856ec992e871dc7a265154e6a669d89f4ad98"},{"author":{"_account_id":8290,"name":"liusheng","email":"liusheng2048@gmail.com","username":"LiuSheng"},"change_message_id":"25fe016e6eeccce1b5fd0ff043ed011f0875acbf","unresolved":false,"context_lines":[{"line_number":68,"context_line":"        default\u003d\u0027gnocchi-data-migration.log\u0027,"},{"line_number":69,"context_line":"        help\u003d\"The log file to record messages during migration.\", )"},{"line_number":70,"context_line":"    parser.add_argument("},{"line_number":71,"context_line":"        \u0027--file-missing-messages\u0027,"},{"line_number":72,"context_line":"        default\u003d\u0027gnocchi-data-migration-missing.msg\u0027,"},{"line_number":73,"context_line":"        help\u003d\"A file specified to store the \u0027message_id\u0027 of sample which was \""},{"line_number":74,"context_line":"             \"failed to be migrated.\", )"}],"source_content_type":"text/x-python","patch_set":5,"id":"ba2be162_fd58fd8b","line":71,"in_reply_to":"ba2be162_7dc7ad63","updated":"2017-03-06 12:28:29.000000000","message":"oh, this is unused, thanks for pointing :)","commit_id":"ad7856ec992e871dc7a265154e6a669d89f4ad98"},{"author":{"_account_id":11564,"name":"Chris Dent","email":"cdent@anticdent.org","username":"chdent"},"change_message_id":"6756f3e0f1c6a496b2bca00965b2b28fc0919217","unresolved":false,"context_lines":[{"line_number":76,"context_line":"        \u0027--batch-migration-size\u0027,"},{"line_number":77,"context_line":"        default\u003d300,"},{"line_number":78,"context_line":"        help\u003d\"The amount of samples will be posted to gnocchi every \""},{"line_number":79,"context_line":"             \"migrating.\", )"},{"line_number":80,"context_line":"    parser.add_argument("},{"line_number":81,"context_line":"        \u0027--start-timestamp\u0027,"},{"line_number":82,"context_line":"        default\u003dNone,"}],"source_content_type":"text/x-python","patch_set":5,"id":"ba2be162_1d595135","line":79,"updated":"2017-03-06 12:22:17.000000000","message":"This sentence is wrong.\n\nThe amount of samples that will be posted to gnocchi per batch.\n\nmight be a better choice","commit_id":"ad7856ec992e871dc7a265154e6a669d89f4ad98"},{"author":{"_account_id":8290,"name":"liusheng","email":"liusheng2048@gmail.com","username":"LiuSheng"},"change_message_id":"25fe016e6eeccce1b5fd0ff043ed011f0875acbf","unresolved":false,"context_lines":[{"line_number":76,"context_line":"        \u0027--batch-migration-size\u0027,"},{"line_number":77,"context_line":"        default\u003d300,"},{"line_number":78,"context_line":"        help\u003d\"The amount of samples will be posted to gnocchi every \""},{"line_number":79,"context_line":"             \"migrating.\", )"},{"line_number":80,"context_line":"    parser.add_argument("},{"line_number":81,"context_line":"        \u0027--start-timestamp\u0027,"},{"line_number":82,"context_line":"        default\u003dNone,"}],"source_content_type":"text/x-python","patch_set":5,"id":"ba2be162_ddf33971","line":79,"in_reply_to":"ba2be162_1d595135","updated":"2017-03-06 12:28:29.000000000","message":"thanks for pointing :)","commit_id":"ad7856ec992e871dc7a265154e6a669d89f4ad98"},{"author":{"_account_id":11564,"name":"Chris Dent","email":"cdent@anticdent.org","username":"chdent"},"change_message_id":"4435fed963bf1115af1c8dd56ded88be55f3bb80","unresolved":false,"context_lines":[{"line_number":57,"context_line":"             \u0027\u003dutf8\u0027,"},{"line_number":58,"context_line":"    )"},{"line_number":59,"context_line":"    parser.add_argument("},{"line_number":60,"context_line":"        \u0027--ceilometer-config-file\u0027,"},{"line_number":61,"context_line":"        help\u003d\"The config file of ceilometer, it is main used for gnocchi \""},{"line_number":62,"context_line":"             \"dispatcher to init gnocchiclient with the service credentials \""},{"line_number":63,"context_line":"             \"defined in the ceilometer config file. Default as \""}],"source_content_type":"text/x-python","patch_set":6,"id":"9a30ddce_30ab3cd8","line":60,"updated":"2017-03-08 12:09:54.000000000","message":"So it\u0027s still not clear where this is being used. There\u0027s no args.ceilometer_config_file being used anywhere so it will always be the default config file that is used in prepare_service, yeah?","commit_id":"87d156b0907ce643114a7b9a6cf92010689e3a9f"},{"author":{"_account_id":8290,"name":"liusheng","email":"liusheng2048@gmail.com","username":"LiuSheng"},"change_message_id":"c1013aac330426e153d1ab4b635e6d0df0dc037c","unresolved":false,"context_lines":[{"line_number":57,"context_line":"             \u0027\u003dutf8\u0027,"},{"line_number":58,"context_line":"    )"},{"line_number":59,"context_line":"    parser.add_argument("},{"line_number":60,"context_line":"        \u0027--ceilometer-config-file\u0027,"},{"line_number":61,"context_line":"        help\u003d\"The config file of ceilometer, it is main used for gnocchi \""},{"line_number":62,"context_line":"             \"dispatcher to init gnocchiclient with the service credentials \""},{"line_number":63,"context_line":"             \"defined in the ceilometer config file. Default as \""}],"source_content_type":"text/x-python","patch_set":6,"id":"dfeb2761_61017b05","line":60,"in_reply_to":"9a30ddce_30ab3cd8","updated":"2017-04-05 01:16:52.000000000","message":"sorry, my mistake, I have fixed that in ps7","commit_id":"87d156b0907ce643114a7b9a6cf92010689e3a9f"},{"author":{"_account_id":6537,"name":"gordon chung","email":"gord@live.ca","username":"chungg"},"change_message_id":"3422c93554c637f6a0dba39a3f4e53036b93f8d4","unresolved":false,"context_lines":[{"line_number":119,"context_line":""},{"line_number":120,"context_line":"    storage_conn \u003d get_native_storage_conn(args.native_metering_connection)"},{"line_number":121,"context_line":"    total_amount \u003d count_samples(storage_conn)"},{"line_number":122,"context_line":"    print(\u0027%s samples will be migrated to Gnocchi.\u0027 % total_amount)"},{"line_number":123,"context_line":""},{"line_number":124,"context_line":"    # NOTE: we need service credentials to init gnocchiclient"},{"line_number":125,"context_line":"    gnocchi_conf \u003d service.prepare_service([])"}],"source_content_type":"text/x-python","patch_set":6,"id":"9a30ddce_08f3889f","line":122,"updated":"2017-03-07 23:09:28.000000000","message":"this count is probably wrong since it\u0027s before filter.","commit_id":"87d156b0907ce643114a7b9a6cf92010689e3a9f"},{"author":{"_account_id":8290,"name":"liusheng","email":"liusheng2048@gmail.com","username":"LiuSheng"},"change_message_id":"c1013aac330426e153d1ab4b635e6d0df0dc037c","unresolved":false,"context_lines":[{"line_number":119,"context_line":""},{"line_number":120,"context_line":"    storage_conn \u003d get_native_storage_conn(args.native_metering_connection)"},{"line_number":121,"context_line":"    total_amount \u003d count_samples(storage_conn)"},{"line_number":122,"context_line":"    print(\u0027%s samples will be migrated to Gnocchi.\u0027 % total_amount)"},{"line_number":123,"context_line":""},{"line_number":124,"context_line":"    # NOTE: we need service credentials to init gnocchiclient"},{"line_number":125,"context_line":"    gnocchi_conf \u003d service.prepare_service([])"}],"source_content_type":"text/x-python","patch_set":6,"id":"dfeb2761_c1116754","line":122,"in_reply_to":"9a30ddce_08f3889f","updated":"2017-04-05 01:16:52.000000000","message":"thanks for pointing out that, fixed in ps7","commit_id":"87d156b0907ce643114a7b9a6cf92010689e3a9f"},{"author":{"_account_id":6537,"name":"gordon chung","email":"gord@live.ca","username":"chungg"},"change_message_id":"3422c93554c637f6a0dba39a3f4e53036b93f8d4","unresolved":false,"context_lines":[{"line_number":135,"context_line":"        start_timestamp_op\u003d\u0027ge\u0027,"},{"line_number":136,"context_line":"        end_timestamp\u003dargs.end_timestamp,"},{"line_number":137,"context_line":"        end_timestamp_op\u003d\u0027lt\u0027)"},{"line_number":138,"context_line":"    samples \u003d storage_conn.get_samples(sample_filter)"},{"line_number":139,"context_line":""},{"line_number":140,"context_line":"    gnocchi_dispatcher \u003d gnocchi.GnocchiDispatcher(gnocchi_conf)"},{"line_number":141,"context_line":""}],"source_content_type":"text/x-python","patch_set":6,"id":"9a30ddce_e88634b8","line":138,"updated":"2017-03-07 23:09:28.000000000","message":"... i\u0027m kind of worried that this doesn\u0027t work in real life and we\u0027re pretending like it does. running this query alone would probably fail if it\u0027s anything \u003e1hr because of 1) query timeout and 2) memory required to store result.\n\ni think this is better if we add warning that this probably doesn\u0027t work.lol or we should at minimum warn that query times should be kept very, very low.","commit_id":"87d156b0907ce643114a7b9a6cf92010689e3a9f"},{"author":{"_account_id":11564,"name":"Chris Dent","email":"cdent@anticdent.org","username":"chdent"},"change_message_id":"4435fed963bf1115af1c8dd56ded88be55f3bb80","unresolved":false,"context_lines":[{"line_number":135,"context_line":"        start_timestamp_op\u003d\u0027ge\u0027,"},{"line_number":136,"context_line":"        end_timestamp\u003dargs.end_timestamp,"},{"line_number":137,"context_line":"        end_timestamp_op\u003d\u0027lt\u0027)"},{"line_number":138,"context_line":"    samples \u003d storage_conn.get_samples(sample_filter)"},{"line_number":139,"context_line":""},{"line_number":140,"context_line":"    gnocchi_dispatcher \u003d gnocchi.GnocchiDispatcher(gnocchi_conf)"},{"line_number":141,"context_line":""}],"source_content_type":"text/x-python","patch_set":6,"id":"9a30ddce_f0f604f8","line":138,"in_reply_to":"9a30ddce_e88634b8","updated":"2017-03-08 12:09:54.000000000","message":"so you\u0027re saying: people need to be sure start_timestamp and end_timestamp are very small windows?","commit_id":"87d156b0907ce643114a7b9a6cf92010689e3a9f"},{"author":{"_account_id":8290,"name":"liusheng","email":"liusheng2048@gmail.com","username":"LiuSheng"},"change_message_id":"c1013aac330426e153d1ab4b635e6d0df0dc037c","unresolved":false,"context_lines":[{"line_number":135,"context_line":"        start_timestamp_op\u003d\u0027ge\u0027,"},{"line_number":136,"context_line":"        end_timestamp\u003dargs.end_timestamp,"},{"line_number":137,"context_line":"        end_timestamp_op\u003d\u0027lt\u0027)"},{"line_number":138,"context_line":"    samples \u003d storage_conn.get_samples(sample_filter)"},{"line_number":139,"context_line":""},{"line_number":140,"context_line":"    gnocchi_dispatcher \u003d gnocchi.GnocchiDispatcher(gnocchi_conf)"},{"line_number":141,"context_line":""}],"source_content_type":"text/x-python","patch_set":6,"id":"dfeb2761_01a86fd8","line":138,"in_reply_to":"9a30ddce_f0f604f8","updated":"2017-04-05 01:16:52.000000000","message":"yes, I have profiled the memory consumption, it will cost \u003e 900M memory when there are \u003e40K samples.\nTo resolve that, I have changed this tool to query only the \"batch_size\" amount of samples every time which will be a sliding window to migrate samples every cycle.\n\nSorry for reply lately.","commit_id":"87d156b0907ce643114a7b9a6cf92010689e3a9f"}]}