)]}'
{"manila/data/utils.py":[{"author":{"_account_id":8851,"name":"Valeriy Ponomaryov","email":"kiparis.kh@gmail.com","username":"vponomaryov"},"change_message_id":"e6d6040593a93dfc670bdb347aa0d55854ab050e","unresolved":false,"context_lines":[{"line_number":137,"context_line":"                                       \u0027attr\u0027: size})"},{"line_number":138,"context_line":"                    self.total_size +\u003d size"},{"line_number":139,"context_line":""},{"line_number":140,"context_line":"    def _start_copy_and_wait(self, src_file, dest_file):"},{"line_number":141,"context_line":"        process \u003d multiprocessing.Process(target\u003dshutil.copy2,"},{"line_number":142,"context_line":"                                          args\u003d(src_file, dest_file))"},{"line_number":143,"context_line":"        process.start()"},{"line_number":144,"context_line":""},{"line_number":145,"context_line":"        previous_bytes \u003d 0"},{"line_number":146,"context_line":"        retries \u003d 0"},{"line_number":147,"context_line":"        while process.is_alive():"},{"line_number":148,"context_line":"            current_bytes \u003d self.get_progress()[\u0027bytes_copied\u0027]"},{"line_number":149,"context_line":""},{"line_number":150,"context_line":"            if current_bytes \u003d\u003d previous_bytes:"},{"line_number":151,"context_line":"                if retries \u003e 3:"},{"line_number":152,"context_line":"                    process.terminate()"},{"line_number":153,"context_line":"                else:"},{"line_number":154,"context_line":"                    retries +\u003d 1"},{"line_number":155,"context_line":"                    time.sleep(0.5)"},{"line_number":156,"context_line":"            elif self.cancelled:"},{"line_number":157,"context_line":"                process.terminate()"},{"line_number":158,"context_line":"            else:"},{"line_number":159,"context_line":"                retries \u003d 0"},{"line_number":160,"context_line":"                previous_bytes \u003d current_bytes"},{"line_number":161,"context_line":"                time.sleep(0.5)"}],"source_content_type":"text/x-python","patch_set":2,"id":"3a57f1b5_c9f54aed","line":161,"range":{"start_line":140,"start_character":4,"end_line":161,"end_character":31},"updated":"2016-02-12 12:51:18.000000000","message":"Here is what will happen using current code:\n\nMain process \"manila-data\" is \"parent\".\n\n- Comes request to migrate share\n- New thread in \"parent\" is spawned\n- You create all directories first - serially.\n- Then you iterate all files and call this func \"_start_copy_and_wait\" that spawns separate process AND\n- AND you wait until it finishes either by termination or end of copy\n- only after that you reach next iteration of a loop with spawn of processes.\n\nSo, as a result:\n\n- you spawn processes one by one and don\u0027t have parallelism indeed.\n\n\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\n\nWhat should be done:\n\nThis method \"_start_copy_and_wait\", should spawn process, then add this spawned object to some container-list of \"parent\" and then exit.\n\nYou have in mind running N amount of children, you keep links to them in some list of \"parent\".\n\nNow, you just need to implement periodic func in \"parent\" that will go over list with links to processes and perform second part of your current \"_start_copy_and_wait\" - check results and remove links when it is terminated and save its results like \"success\" or \"error\".\n\nIn this case you will have parallelism.\n\nAlso, implementing it, you must setup some threshould for maximum amount of simultaneously working \"child\"s. So, should be some pool of copy operations. Else you risk to end-up killing host machine with spawn-bomb.","commit_id":"2ecd34b9388d815f68e9dedfa2ceb4cdce02a8f3"},{"author":{"_account_id":17097,"name":"Tiago Pasqualini da Silva","email":"t.pasqualini@luizalabs.com","username":"tiago.pasqualini"},"change_message_id":"7235d9c2baeb64689cd8be982fd4e119a8452d41","unresolved":false,"context_lines":[{"line_number":137,"context_line":"                                       \u0027attr\u0027: size})"},{"line_number":138,"context_line":"                    self.total_size +\u003d size"},{"line_number":139,"context_line":""},{"line_number":140,"context_line":"    def _start_copy_and_wait(self, src_file, dest_file):"},{"line_number":141,"context_line":"        process \u003d multiprocessing.Process(target\u003dshutil.copy2,"},{"line_number":142,"context_line":"                                          args\u003d(src_file, dest_file))"},{"line_number":143,"context_line":"        process.start()"},{"line_number":144,"context_line":""},{"line_number":145,"context_line":"        previous_bytes \u003d 0"},{"line_number":146,"context_line":"        retries \u003d 0"},{"line_number":147,"context_line":"        while process.is_alive():"},{"line_number":148,"context_line":"            current_bytes \u003d self.get_progress()[\u0027bytes_copied\u0027]"},{"line_number":149,"context_line":""},{"line_number":150,"context_line":"            if current_bytes \u003d\u003d previous_bytes:"},{"line_number":151,"context_line":"                if retries \u003e 3:"},{"line_number":152,"context_line":"                    process.terminate()"},{"line_number":153,"context_line":"                else:"},{"line_number":154,"context_line":"                    retries +\u003d 1"},{"line_number":155,"context_line":"                    time.sleep(0.5)"},{"line_number":156,"context_line":"            elif self.cancelled:"},{"line_number":157,"context_line":"                process.terminate()"},{"line_number":158,"context_line":"            else:"},{"line_number":159,"context_line":"                retries \u003d 0"},{"line_number":160,"context_line":"                previous_bytes \u003d current_bytes"},{"line_number":161,"context_line":"                time.sleep(0.5)"}],"source_content_type":"text/x-python","patch_set":2,"id":"dae33548_3323b6be","line":161,"range":{"start_line":140,"start_character":4,"end_line":161,"end_character":31},"in_reply_to":"3a57f1b5_c9f54aed","updated":"2016-02-18 17:06:53.000000000","message":"Actually the idea is to have a single process copying all the files, leaving data service able to respond to any other requests. This implementation is wrong, since it starts a different process for each file being copied. I changed it to copy all the files in a separate process. Basically, I moved the whole copy method to run in a separate process.","commit_id":"2ecd34b9388d815f68e9dedfa2ceb4cdce02a8f3"},{"author":{"_account_id":14567,"name":"Rodrigo Barbieri","email":"rodrigo.barbieri2010@gmail.com","username":"ganso"},"change_message_id":"3303849c0eb2621e5b554761f6cc4017c917972d","unresolved":false,"context_lines":[{"line_number":51,"context_line":"        if self.total_size \u003e 0:"},{"line_number":52,"context_line":"            total_progress \u003d (self.current_total_size.value * 100 /"},{"line_number":53,"context_line":"                              self.total_size)"},{"line_number":54,"context_line":"        else:"},{"line_number":55,"context_line":"            total_progress \u003d 100"},{"line_number":56,"context_line":""},{"line_number":57,"context_line":"        current_file_progress \u003d 0"},{"line_number":58,"context_line":"        if self.current_copy_size.value \u003e 0:"}],"source_content_type":"text/x-python","patch_set":6,"id":"bae84128_526d7203","line":55,"range":{"start_line":54,"start_character":8,"end_line":55,"end_character":32},"updated":"2016-02-19 13:46:00.000000000","message":"I see a situation where there are files to be copied but it is still exploring the files and directories while a get_progress request is received, and would show progress \u003d 100\n\nI believe a possible approach to avoid this situation is to add a flag that checks if it has already explored or not.","commit_id":"ee3b2d52a5e280757e8a03ca192d2875b2f3a22c"},{"author":{"_account_id":17097,"name":"Tiago Pasqualini da Silva","email":"t.pasqualini@luizalabs.com","username":"tiago.pasqualini"},"change_message_id":"2ff93a0037a9c026e4d8e7621dcb30c69d6e5583","unresolved":false,"context_lines":[{"line_number":51,"context_line":"        if self.total_size \u003e 0:"},{"line_number":52,"context_line":"            total_progress \u003d (self.current_total_size.value * 100 /"},{"line_number":53,"context_line":"                              self.total_size)"},{"line_number":54,"context_line":"        else:"},{"line_number":55,"context_line":"            total_progress \u003d 100"},{"line_number":56,"context_line":""},{"line_number":57,"context_line":"        current_file_progress \u003d 0"},{"line_number":58,"context_line":"        if self.current_copy_size.value \u003e 0:"}],"source_content_type":"text/x-python","patch_set":6,"id":"bae84128_eb95a140","line":55,"range":{"start_line":54,"start_character":8,"end_line":55,"end_character":32},"in_reply_to":"bae84128_526d7203","updated":"2016-02-19 19:23:21.000000000","message":"Done","commit_id":"ee3b2d52a5e280757e8a03ca192d2875b2f3a22c"},{"author":{"_account_id":14567,"name":"Rodrigo Barbieri","email":"rodrigo.barbieri2010@gmail.com","username":"ganso"},"change_message_id":"3303849c0eb2621e5b554761f6cc4017c917972d","unresolved":false,"context_lines":[{"line_number":60,"context_line":"        current_file_path \u003d six.text_type(self.current_copy_path.value)"},{"line_number":61,"context_line":""},{"line_number":62,"context_line":"        progress \u003d {"},{"line_number":63,"context_line":"            \u0027bytes_copied\u0027: self.current_total_size.value + size,"},{"line_number":64,"context_line":"            \u0027total_progress\u0027: total_progress,"},{"line_number":65,"context_line":"            \u0027current_file_path\u0027: current_file_path,"},{"line_number":66,"context_line":"            \u0027current_file_progress\u0027: current_file_progress"}],"source_content_type":"text/x-python","patch_set":6,"id":"bae84128_1574749a","line":63,"range":{"start_line":63,"start_character":12,"end_line":63,"end_character":25},"updated":"2016-02-19 13:46:00.000000000","message":"this is current_file_bytes_copied, it can easily be mistaken by a total amount of bytes copied.","commit_id":"ee3b2d52a5e280757e8a03ca192d2875b2f3a22c"},{"author":{"_account_id":17097,"name":"Tiago Pasqualini da Silva","email":"t.pasqualini@luizalabs.com","username":"tiago.pasqualini"},"change_message_id":"2ff93a0037a9c026e4d8e7621dcb30c69d6e5583","unresolved":false,"context_lines":[{"line_number":60,"context_line":"        current_file_path \u003d six.text_type(self.current_copy_path.value)"},{"line_number":61,"context_line":""},{"line_number":62,"context_line":"        progress \u003d {"},{"line_number":63,"context_line":"            \u0027bytes_copied\u0027: self.current_total_size.value + size,"},{"line_number":64,"context_line":"            \u0027total_progress\u0027: total_progress,"},{"line_number":65,"context_line":"            \u0027current_file_path\u0027: current_file_path,"},{"line_number":66,"context_line":"            \u0027current_file_progress\u0027: current_file_progress"}],"source_content_type":"text/x-python","patch_set":6,"id":"bae84128_b00ca495","line":63,"range":{"start_line":63,"start_character":12,"end_line":63,"end_character":25},"in_reply_to":"bae84128_1574749a","updated":"2016-02-19 19:23:21.000000000","message":"This is actually the total amount of bytes copied. Changed to total_bytes_copied.","commit_id":"ee3b2d52a5e280757e8a03ca192d2875b2f3a22c"},{"author":{"_account_id":14567,"name":"Rodrigo Barbieri","email":"rodrigo.barbieri2010@gmail.com","username":"ganso"},"change_message_id":"3303849c0eb2621e5b554761f6cc4017c917972d","unresolved":false,"context_lines":[{"line_number":118,"context_line":"                if retries \u003e\u003d 3:"},{"line_number":119,"context_line":"                    LOG.warning(_LW(\"Copy is stalled. Terminating process.\"))"},{"line_number":120,"context_line":"                    process.terminate()"},{"line_number":121,"context_line":"                    self.cancel()"},{"line_number":122,"context_line":"                    terminated \u003d True"},{"line_number":123,"context_line":"                else:"},{"line_number":124,"context_line":"                    retries +\u003d 1"}],"source_content_type":"text/x-python","patch_set":6,"id":"bae84128_f530f0a0","line":121,"range":{"start_line":121,"start_character":20,"end_line":121,"end_character":33},"updated":"2016-02-19 13:46:00.000000000","message":"this is error, it is different than \"cancelled\" scenario, cancelled scenario is when user cancels.\n\nThen if terminated \u003d True but cancelled \u003d False, throw an exception.","commit_id":"ee3b2d52a5e280757e8a03ca192d2875b2f3a22c"},{"author":{"_account_id":17097,"name":"Tiago Pasqualini da Silva","email":"t.pasqualini@luizalabs.com","username":"tiago.pasqualini"},"change_message_id":"2ff93a0037a9c026e4d8e7621dcb30c69d6e5583","unresolved":false,"context_lines":[{"line_number":118,"context_line":"                if retries \u003e\u003d 3:"},{"line_number":119,"context_line":"                    LOG.warning(_LW(\"Copy is stalled. Terminating process.\"))"},{"line_number":120,"context_line":"                    process.terminate()"},{"line_number":121,"context_line":"                    self.cancel()"},{"line_number":122,"context_line":"                    terminated \u003d True"},{"line_number":123,"context_line":"                else:"},{"line_number":124,"context_line":"                    retries +\u003d 1"}],"source_content_type":"text/x-python","patch_set":6,"id":"bae84128_8bb055f6","line":121,"range":{"start_line":121,"start_character":20,"end_line":121,"end_character":33},"in_reply_to":"bae84128_f530f0a0","updated":"2016-02-19 19:23:21.000000000","message":"Done","commit_id":"ee3b2d52a5e280757e8a03ca192d2875b2f3a22c"}]}