)]}'
{"swift/obj/replicator.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"0d47813c796f5dcc88a6dd251ce1b065d1e53e44","unresolved":false,"context_lines":[{"line_number":580,"context_line":"        while True:"},{"line_number":581,"context_line":"            eventlet.sleep(self.lockup_timeout)"},{"line_number":582,"context_line":"            if self.replication_count \u003d\u003d self.last_replication_count:"},{"line_number":583,"context_line":"                self.logger.error(_(\"Lockup detected.. killing live coros.\"))"},{"line_number":584,"context_line":"                self.kill_coros()"},{"line_number":585,"context_line":"            self.last_replication_count \u003d self.replication_count"},{"line_number":586,"context_line":""}],"source_content_type":"text/x-python","patch_set":1,"id":"df7087c5_6885af59","side":"PARENT","line":583,"updated":"2018-03-13 21:48:39.000000000","message":"somehow I think we loose a little bit just letting go of this ... but IME it wasn\u0027t reliable in the main pthread where stuff like rehashing could make all of eventlet lockup.","commit_id":"de38b727b475876b4a3e4dcf7dca21de8454722e"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"0d47813c796f5dcc88a6dd251ce1b065d1e53e44","unresolved":false,"context_lines":[{"line_number":161,"context_line":"                try:"},{"line_number":162,"context_line":"                    # this will reap the process if it has exited, but"},{"line_number":163,"context_line":"                    # otherwise will not wait"},{"line_number":164,"context_line":"                    proc.wait(timeout\u003d0)"},{"line_number":165,"context_line":"                    reaped_procs.add(proc)"},{"line_number":166,"context_line":"                except subprocess.TimeoutExpired:"},{"line_number":167,"context_line":"                    pass"}],"source_content_type":"text/x-python","patch_set":1,"id":"df7087c5_a8f0a792","line":164,"updated":"2018-03-13 21:48:39.000000000","message":"do want to put the proc objects into the queue - or use a green\u0027d version of os.waitpid?\n\n    object-6016: STDERR: Traceback (most recent call last):\n    object-6016: STDERR: File \"/usr/local/lib/python2.7/dist-packages/eventlet/queue.py\", line 118, in switch\n    object-6016: STDERR: self.greenlet.switch(value)\n    object-6016: STDERR: File \"/usr/local/lib/python2.7/dist-packages/eventlet/greenthread.py\", line 218, in main\n    object-6016: STDERR: result \u003d function(*args, **kwargs)\n    object-6016: STDERR: File \"/vagrant/swift/swift/obj/replicator.py\", line 164, in _child_process_reaper\n    object-6016: STDERR: proc.wait(timeout\u003d0)\n    object-6016: STDERR: AttributeError: \u0027int\u0027 object has no attribute \u0027wait\u0027","commit_id":"5e8f1b3dc98e983c0bbd46a044c33da40304d482"},{"author":{"_account_id":2622,"name":"Samuel Merritt","email":"spam+launchpad@andcheese.org","username":"torgomatic"},"change_message_id":"004a07f80be7dd9a97c8eefb68e429b11f21f22f","unresolved":false,"context_lines":[{"line_number":161,"context_line":"                try:"},{"line_number":162,"context_line":"                    # this will reap the process if it has exited, but"},{"line_number":163,"context_line":"                    # otherwise will not wait"},{"line_number":164,"context_line":"                    proc.wait(timeout\u003d0)"},{"line_number":165,"context_line":"                    reaped_procs.add(proc)"},{"line_number":166,"context_line":"                except subprocess.TimeoutExpired:"},{"line_number":167,"context_line":"                    pass"}],"source_content_type":"text/x-python","patch_set":1,"id":"df7087c5_196b271e","line":164,"in_reply_to":"df7087c5_a8f0a792","updated":"2018-03-14 23:15:35.000000000","message":"Doh! I was going to send proc objects. They\u0027ve got a green version of os.waitpid() in their wait() methods already, so that saves me from writing one, but then I managed to commit \"put(proc.pid)\" down below and apparently didn\u0027t write any tests to catch it.","commit_id":"5e8f1b3dc98e983c0bbd46a044c33da40304d482"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"0d47813c796f5dcc88a6dd251ce1b065d1e53e44","unresolved":false,"context_lines":[{"line_number":791,"context_line":"                if job[\u0027delete\u0027]:"},{"line_number":792,"context_line":"                    self.run_pool.spawn(self.update_deleted, job)"},{"line_number":793,"context_line":"                else:"},{"line_number":794,"context_line":"                    self.run_pool.spawn(self.update, job)"},{"line_number":795,"context_line":"            current_nodes \u003d None"},{"line_number":796,"context_line":"            self.run_pool.waitall()"},{"line_number":797,"context_line":"        except (Exception, Timeout):"}],"source_content_type":"text/x-python","patch_set":1,"id":"df7087c5_48430b14","line":794,"updated":"2018-03-13 21:48:39.000000000","message":"Tim has pointed out how weird it was to wrap the waitall in a timeout, but not these spawn\u0027s which could just as easily block for \u003e lockup_timeout","commit_id":"5e8f1b3dc98e983c0bbd46a044c33da40304d482"},{"author":{"_account_id":2622,"name":"Samuel Merritt","email":"spam+launchpad@andcheese.org","username":"torgomatic"},"change_message_id":"004a07f80be7dd9a97c8eefb68e429b11f21f22f","unresolved":false,"context_lines":[{"line_number":791,"context_line":"                if job[\u0027delete\u0027]:"},{"line_number":792,"context_line":"                    self.run_pool.spawn(self.update_deleted, job)"},{"line_number":793,"context_line":"                else:"},{"line_number":794,"context_line":"                    self.run_pool.spawn(self.update, job)"},{"line_number":795,"context_line":"            current_nodes \u003d None"},{"line_number":796,"context_line":"            self.run_pool.waitall()"},{"line_number":797,"context_line":"        except (Exception, Timeout):"}],"source_content_type":"text/x-python","patch_set":1,"id":"df7087c5_f998b3c0","line":794,"in_reply_to":"df7087c5_48430b14","updated":"2018-03-14 23:15:35.000000000","message":"Yep, that\u0027s true. If you only get a couple immortal rsyncs then you won\u0027t block until waitall(), but if you get a lot then it\u0027ll happen here at spawn time.","commit_id":"5e8f1b3dc98e983c0bbd46a044c33da40304d482"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"0d47813c796f5dcc88a6dd251ce1b065d1e53e44","unresolved":false,"context_lines":[{"line_number":846,"context_line":""},{"line_number":847,"context_line":"    def run_forever(self, *args, **kwargs):"},{"line_number":848,"context_line":"        self.logger.info(_(\"Starting object replicator in daemon mode.\"))"},{"line_number":849,"context_line":"        eventlet.spawn_n(self._child_process_reaper)"},{"line_number":850,"context_line":"        # Run the replicator continually"},{"line_number":851,"context_line":"        while True:"},{"line_number":852,"context_line":"            self._zero_stats()"}],"source_content_type":"text/x-python","patch_set":1,"id":"df7087c5_c84e3b37","line":849,"updated":"2018-03-13 21:48:39.000000000","message":"hrmm... I guess SIGTERM doesn\u0027t really have a graceful exit path","commit_id":"5e8f1b3dc98e983c0bbd46a044c33da40304d482"},{"author":{"_account_id":2622,"name":"Samuel Merritt","email":"spam+launchpad@andcheese.org","username":"torgomatic"},"change_message_id":"004a07f80be7dd9a97c8eefb68e429b11f21f22f","unresolved":false,"context_lines":[{"line_number":846,"context_line":""},{"line_number":847,"context_line":"    def run_forever(self, *args, **kwargs):"},{"line_number":848,"context_line":"        self.logger.info(_(\"Starting object replicator in daemon mode.\"))"},{"line_number":849,"context_line":"        eventlet.spawn_n(self._child_process_reaper)"},{"line_number":850,"context_line":"        # Run the replicator continually"},{"line_number":851,"context_line":"        while True:"},{"line_number":852,"context_line":"            self._zero_stats()"}],"source_content_type":"text/x-python","patch_set":1,"id":"df7087c5_ae566f29","line":849,"in_reply_to":"df7087c5_c84e3b37","updated":"2018-03-14 23:15:35.000000000","message":"Nope, not really. If we have any unreaped children, they\u0027ll be  PID 1\u0027s problem.","commit_id":"5e8f1b3dc98e983c0bbd46a044c33da40304d482"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"9b38d7499ffc72bebdeea320c8948fda2213badb","unresolved":false,"context_lines":[{"line_number":90,"context_line":"        if not self.rsync_module:"},{"line_number":91,"context_line":"            self.rsync_module \u003d \u0027{replication_ip}::object\u0027"},{"line_number":92,"context_line":"        self.http_timeout \u003d int(conf.get(\u0027http_timeout\u0027, 60))"},{"line_number":93,"context_line":"        self.lockup_timeout \u003d int(conf.get(\u0027lockup_timeout\u0027, 1800))"},{"line_number":94,"context_line":"        self.recon_cache_path \u003d conf.get(\u0027recon_cache_path\u0027,"},{"line_number":95,"context_line":"                                         \u0027/var/cache/swift\u0027)"},{"line_number":96,"context_line":"        self.rcache \u003d os.path.join(self.recon_cache_path, \"object.recon\")"}],"source_content_type":"text/x-python","patch_set":3,"id":"bf659307_88d93cf4","side":"PARENT","line":93,"updated":"2018-03-29 04:43:52.000000000","message":"Is there anything approaching an equivalent of this? Or is this just gone? The idea being -- it was a bad idea before, and we\u0027ve removed this particular foot-gun. You\u0027re welcome.","commit_id":"9aca9ad7802a9843876f1280b591d99c4212d0c8"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"9b38d7499ffc72bebdeea320c8948fda2213badb","unresolved":false,"context_lines":[{"line_number":178,"context_line":"                                        stderr\u003dsubprocess.STDOUT)"},{"line_number":179,"context_line":"                results \u003d proc.stdout.read()"},{"line_number":180,"context_line":"                ret_val \u003d proc.wait()"},{"line_number":181,"context_line":"        except GreenletExit:"},{"line_number":182,"context_line":"            self.logger.error(_(\"Killing by lockup detector\"))"},{"line_number":183,"context_line":"            if proc:"},{"line_number":184,"context_line":"                # Assume rsync is still responsive and give it a chance"}],"source_content_type":"text/x-python","patch_set":3,"id":"bf659307_6805c845","side":"PARENT","line":181,"updated":"2018-03-29 04:43:52.000000000","message":"Any time we don\u0027t have to catch then reraise some internal-to-eventlet exception that *intentionally* inherits from BaseException and *not* Exception specifically so you *don\u0027t* go trying to catch and handle it -- that\u0027s a good day.","commit_id":"9aca9ad7802a9843876f1280b591d99c4212d0c8"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"93d5fd6a0292e4b70d3e0bf0d95f83d7394d3a00","unresolved":false,"context_lines":[{"line_number":560,"context_line":"            try:"},{"line_number":561,"context_line":"                coro.kill(GreenletExit)"},{"line_number":562,"context_line":"            except GreenletExit:"},{"line_number":563,"context_line":"                pass"},{"line_number":564,"context_line":""},{"line_number":565,"context_line":"    def heartbeat(self):"},{"line_number":566,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":3,"id":"bf659307_47bcf868","side":"PARENT","line":563,"updated":"2018-03-23 19:29:09.000000000","message":"this was such a terrible implementationin practice - so glad to see it gone.","commit_id":"9aca9ad7802a9843876f1280b591d99c4212d0c8"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"93d5fd6a0292e4b70d3e0bf0d95f83d7394d3a00","unresolved":false,"context_lines":[{"line_number":580,"context_line":"        while True:"},{"line_number":581,"context_line":"            eventlet.sleep(self.lockup_timeout)"},{"line_number":582,"context_line":"            if self.replication_count \u003d\u003d self.last_replication_count:"},{"line_number":583,"context_line":"                self.logger.error(_(\"Lockup detected.. killing live coros.\"))"},{"line_number":584,"context_line":"                self.kill_coros()"},{"line_number":585,"context_line":"            self.last_replication_count \u003d self.replication_count"},{"line_number":586,"context_line":""}],"source_content_type":"text/x-python","patch_set":3,"id":"bf659307_67625c19","side":"PARENT","line":583,"updated":"2018-03-23 19:29:09.000000000","message":"this check, this count, was not useless - but it wasn\u0027t reliable either because it ran in the same os thread and the blocking disk io.\n\nEverything that happened after this log line was just terrible in practice tho.","commit_id":"9aca9ad7802a9843876f1280b591d99c4212d0c8"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"9b38d7499ffc72bebdeea320c8948fda2213badb","unresolved":false,"context_lines":[{"line_number":149,"context_line":"            try:"},{"line_number":150,"context_line":"                new_proc \u003d self._child_process_reaper_queue.get("},{"line_number":151,"context_line":"                    timeout\u003dtimeout)"},{"line_number":152,"context_line":"                if new_proc is not None:"},{"line_number":153,"context_line":"                    procs.add(new_proc)"},{"line_number":154,"context_line":"                else:"},{"line_number":155,"context_line":"                    done \u003d True"}],"source_content_type":"text/x-python","patch_set":3,"id":"bf659307_2306111f","line":152,"range":{"start_line":152,"start_character":28,"end_line":152,"end_character":34},"updated":"2018-03-29 04:43:52.000000000","message":"I might\u0027ve swapped these to avoid the not... w/e","commit_id":"ecf8ae50e14db146f9368cda73a265a92663ef03"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"9b38d7499ffc72bebdeea320c8948fda2213badb","unresolved":false,"context_lines":[{"line_number":152,"context_line":"                if new_proc is not None:"},{"line_number":153,"context_line":"                    procs.add(new_proc)"},{"line_number":154,"context_line":"                else:"},{"line_number":155,"context_line":"                    done \u003d True"},{"line_number":156,"context_line":"            except queue.Empty:"},{"line_number":157,"context_line":"                pass"},{"line_number":158,"context_line":""}],"source_content_type":"text/x-python","patch_set":3,"id":"bf659307_a311c154","line":155,"updated":"2018-03-29 04:43:52.000000000","message":"We don\u0027t just break? I guess it gives every proc *one last chance* to finish... but I\u0027m not sure there\u0027ll actually be much difference in practice.","commit_id":"ecf8ae50e14db146f9368cda73a265a92663ef03"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"9b38d7499ffc72bebdeea320c8948fda2213badb","unresolved":false,"context_lines":[{"line_number":161,"context_line":"                try:"},{"line_number":162,"context_line":"                    # this will reap the process if it has exited, but"},{"line_number":163,"context_line":"                    # otherwise will not wait"},{"line_number":164,"context_line":"                    proc.wait(timeout\u003d0)"},{"line_number":165,"context_line":"                    reaped_procs.add(proc)"},{"line_number":166,"context_line":"                except subprocess.TimeoutExpired:"},{"line_number":167,"context_line":"                    pass"}],"source_content_type":"text/x-python","patch_set":3,"id":"bf659307_c31f1559","line":164,"range":{"start_line":164,"start_character":30,"end_line":164,"end_character":39},"updated":"2018-03-29 04:43:52.000000000","message":"All right... so most/all of the waiting is done on the queue... I think I like it.","commit_id":"ecf8ae50e14db146f9368cda73a265a92663ef03"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"864f169317689eaac7fdef70c64ef99b325e79d6","unresolved":false,"context_lines":[{"line_number":161,"context_line":"                try:"},{"line_number":162,"context_line":"                    # this will reap the process if it has exited, but"},{"line_number":163,"context_line":"                    # otherwise will not wait"},{"line_number":164,"context_line":"                    proc.wait(timeout\u003d0)"},{"line_number":165,"context_line":"                    reaped_procs.add(proc)"},{"line_number":166,"context_line":"                except subprocess.TimeoutExpired:"},{"line_number":167,"context_line":"                    pass"}],"source_content_type":"text/x-python","patch_set":3,"id":"bf659307_ed3c71c9","line":164,"range":{"start_line":164,"start_character":30,"end_line":164,"end_character":39},"in_reply_to":"bf659307_c31f1559","updated":"2018-04-02 20:42:41.000000000","message":"Cool -- so eventlet\u0027s supported a timeout since at least our lower-bound on eventlet: https://github.com/eventlet/eventlet/blob/v0.17.4/eventlet/green/subprocess.py#L63\n\nLooks like it goes back nearly five years -- https://github.com/eventlet/eventlet/commit/3c84402\n\nI wonder if it\u0027d be worth doing something like\n\n status \u003d proc.poll()\n if status is not None:\n     reaped_procs.add(proc)\n\nand dropping the try/except/pass, just to be a little more explicit.","commit_id":"ecf8ae50e14db146f9368cda73a265a92663ef03"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"93d5fd6a0292e4b70d3e0bf0d95f83d7394d3a00","unresolved":false,"context_lines":[{"line_number":165,"context_line":"                    reaped_procs.add(proc)"},{"line_number":166,"context_line":"                except subprocess.TimeoutExpired:"},{"line_number":167,"context_line":"                    pass"},{"line_number":168,"context_line":"            procs -\u003d reaped_procs"},{"line_number":169,"context_line":""},{"line_number":170,"context_line":"    # Just exists for doc anchor point"},{"line_number":171,"context_line":"    def sync(self, node, job, suffixes, *args, **kwargs):"}],"source_content_type":"text/x-python","patch_set":3,"id":"bf659307_276c6407","line":168,"updated":"2018-03-23 19:29:09.000000000","message":"this is working well for me - no zombies","commit_id":"ecf8ae50e14db146f9368cda73a265a92663ef03"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e8bbf3dce78e5d8ff355b88e3812ffca6290702b","unresolved":false,"context_lines":[{"line_number":201,"context_line":"        proc \u003d None"},{"line_number":202,"context_line":""},{"line_number":203,"context_line":"        try:"},{"line_number":204,"context_line":"            with Timeout(self.rsync_timeout):"},{"line_number":205,"context_line":"                proc \u003d subprocess.Popen(args,"},{"line_number":206,"context_line":"                                        stdout\u003dsubprocess.PIPE,"},{"line_number":207,"context_line":"                                        stderr\u003dsubprocess.STDOUT)"}],"source_content_type":"text/x-python","patch_set":3,"id":"bf659307_9c8cdad7","line":204,"range":{"start_line":204,"start_character":25,"end_line":204,"end_character":43},"updated":"2018-04-02 21:21:03.000000000","message":"I wonder if we could just pass this to proc.wait()...","commit_id":"ecf8ae50e14db146f9368cda73a265a92663ef03"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"9b38d7499ffc72bebdeea320c8948fda2213badb","unresolved":false,"context_lines":[{"line_number":214,"context_line":"                try:"},{"line_number":215,"context_line":"                    # Note: Python 2.7\u0027s subprocess.Popen class doesn\u0027t take"},{"line_number":216,"context_line":"                    # any arguments for wait(), but Python 3\u0027s does."},{"line_number":217,"context_line":"                    # However, Eventlet\u0027s replacement Popen takes a timeout"},{"line_number":218,"context_line":"                    # argument regardless of Python version, so we don\u0027t"},{"line_number":219,"context_line":"                    # need any conditional code here."},{"line_number":220,"context_line":"                    proc.wait(timeout\u003d1.0)"},{"line_number":221,"context_line":"                except subprocess.TimeoutExpired:"}],"source_content_type":"text/x-python","patch_set":3,"id":"bf659307_48f80448","line":218,"range":{"start_line":217,"start_character":31,"end_line":218,"end_character":59},"updated":"2018-03-29 04:43:52.000000000","message":"Nice. Which versions of eventlet? Or was it *always* so forward thinking?","commit_id":"ecf8ae50e14db146f9368cda73a265a92663ef03"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"93d5fd6a0292e4b70d3e0bf0d95f83d7394d3a00","unresolved":false,"context_lines":[{"line_number":226,"context_line":"                    # we send it over to another greenthread, not part of"},{"line_number":227,"context_line":"                    # our pool, whose sole duty is to wait for child"},{"line_number":228,"context_line":"                    # processes to exit."},{"line_number":229,"context_line":"                    self._child_process_reaper_queue.put(proc)"},{"line_number":230,"context_line":"            return 1  # failure response code"},{"line_number":231,"context_line":""},{"line_number":232,"context_line":"        total_time \u003d time.time() - start_time"}],"source_content_type":"text/x-python","patch_set":3,"id":"bf659307_a77f14aa","line":229,"updated":"2018-03-23 19:29:09.000000000","message":"this seems to be working well for me OMM","commit_id":"ecf8ae50e14db146f9368cda73a265a92663ef03"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"9b38d7499ffc72bebdeea320c8948fda2213badb","unresolved":false,"context_lines":[{"line_number":793,"context_line":"                else:"},{"line_number":794,"context_line":"                    self.run_pool.spawn(self.update, job)"},{"line_number":795,"context_line":"            current_nodes \u003d None"},{"line_number":796,"context_line":"            self.run_pool.waitall()"},{"line_number":797,"context_line":"        except (Exception, Timeout):"},{"line_number":798,"context_line":"            if current_nodes:"},{"line_number":799,"context_line":"                self._add_failure_stats([(failure_dev[\u0027replication_ip\u0027],"}],"source_content_type":"text/x-python","patch_set":3,"id":"bf659307_487b24bd","line":796,"updated":"2018-03-29 04:43:52.000000000","message":"Right, so now we *absolutely require* that no greenthread waits indefinitely. And if we ever accidentally reintroduce a\n\n proc.wait()\n\nin the main thread, we\u0027re back where we started :-/\n\nMaybe we should keep a Timeout? Something like 2 * pool size, maybe?","commit_id":"ecf8ae50e14db146f9368cda73a265a92663ef03"},{"author":{"_account_id":2622,"name":"Samuel Merritt","email":"spam+launchpad@andcheese.org","username":"torgomatic"},"change_message_id":"c379611a2072b90c73bf959210298b40e1617f2e","unresolved":false,"context_lines":[{"line_number":793,"context_line":"                else:"},{"line_number":794,"context_line":"                    self.run_pool.spawn(self.update, job)"},{"line_number":795,"context_line":"            current_nodes \u003d None"},{"line_number":796,"context_line":"            self.run_pool.waitall()"},{"line_number":797,"context_line":"        except (Exception, Timeout):"},{"line_number":798,"context_line":"            if current_nodes:"},{"line_number":799,"context_line":"                self._add_failure_stats([(failure_dev[\u0027replication_ip\u0027],"}],"source_content_type":"text/x-python","patch_set":3,"id":"bf659307_fc8156c4","line":796,"in_reply_to":"bf659307_487b24bd","updated":"2018-04-02 21:12:10.000000000","message":"The old timeout didn\u0027t really work, though. Sometimes it did, and sometimes it didn\u0027t.\n\nEither of the calls to self.run_pool.spawn() could wedge; all it takes is concurrency\u003dC and the presence of C wedged greenthreads. That wasn\u0027t protected by the old timeout.\n\nIf we want to have a timeout on the greenthreads, there\u0027s some work to be done. First, we\u0027d make sure all rsync processes are tracked in some shared data structure so that killing a greenthread doesn\u0027t lose an rsync reference. Second, we\u0027d need some way of allowing the pool\u0027s greenthreads to report liveness so that a long-running cross-region rsync doesn\u0027t get killed just because it\u0027s slow. Once we had those, I\u0027d feel okay about putting a timeout around the work jobs.\n\nIt ends up looking a lot like Pistil, but for greenthreads.","commit_id":"ecf8ae50e14db146f9368cda73a265a92663ef03"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"e8bbf3dce78e5d8ff355b88e3812ffca6290702b","unresolved":false,"context_lines":[{"line_number":793,"context_line":"                else:"},{"line_number":794,"context_line":"                    self.run_pool.spawn(self.update, job)"},{"line_number":795,"context_line":"            current_nodes \u003d None"},{"line_number":796,"context_line":"            self.run_pool.waitall()"},{"line_number":797,"context_line":"        except (Exception, Timeout):"},{"line_number":798,"context_line":"            if current_nodes:"},{"line_number":799,"context_line":"                self._add_failure_stats([(failure_dev[\u0027replication_ip\u0027],"}],"source_content_type":"text/x-python","patch_set":3,"id":"bf659307_bc8bdec0","line":796,"in_reply_to":"bf659307_fc8156c4","updated":"2018-04-02 21:21:03.000000000","message":"\u003e Either of the calls to self.run_pool.spawn() could wedge\n\nYeah, so we\u0027d want timeouts around those, *too*, if we bring it back here. Fair point on long-running cross-region rsyncs, though.","commit_id":"ecf8ae50e14db146f9368cda73a265a92663ef03"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"864f169317689eaac7fdef70c64ef99b325e79d6","unresolved":false,"context_lines":[{"line_number":794,"context_line":"                    self.run_pool.spawn(self.update, job)"},{"line_number":795,"context_line":"            current_nodes \u003d None"},{"line_number":796,"context_line":"            self.run_pool.waitall()"},{"line_number":797,"context_line":"        except (Exception, Timeout):"},{"line_number":798,"context_line":"            if current_nodes:"},{"line_number":799,"context_line":"                self._add_failure_stats([(failure_dev[\u0027replication_ip\u0027],"},{"line_number":800,"context_line":"                                          failure_dev[\u0027device\u0027])"}],"source_content_type":"text/x-python","patch_set":3,"id":"bf659307_7cb7863b","line":797,"range":{"start_line":797,"start_character":27,"end_line":797,"end_character":34},"updated":"2018-04-02 20:42:41.000000000","message":"Can anything trip this now? Something down in collect_jobs, maybe, I guess?","commit_id":"ecf8ae50e14db146f9368cda73a265a92663ef03"},{"author":{"_account_id":2622,"name":"Samuel Merritt","email":"spam+launchpad@andcheese.org","username":"torgomatic"},"change_message_id":"c379611a2072b90c73bf959210298b40e1617f2e","unresolved":false,"context_lines":[{"line_number":794,"context_line":"                    self.run_pool.spawn(self.update, job)"},{"line_number":795,"context_line":"            current_nodes \u003d None"},{"line_number":796,"context_line":"            self.run_pool.waitall()"},{"line_number":797,"context_line":"        except (Exception, Timeout):"},{"line_number":798,"context_line":"            if current_nodes:"},{"line_number":799,"context_line":"                self._add_failure_stats([(failure_dev[\u0027replication_ip\u0027],"},{"line_number":800,"context_line":"                                          failure_dev[\u0027device\u0027])"}],"source_content_type":"text/x-python","patch_set":3,"id":"bf659307_3c658e46","line":797,"range":{"start_line":797,"start_character":27,"end_line":797,"end_character":34},"in_reply_to":"bf659307_7cb7863b","updated":"2018-04-02 21:12:10.000000000","message":"I can take that out. I don\u0027t think there\u0027s any timeouts in here any more.","commit_id":"ecf8ae50e14db146f9368cda73a265a92663ef03"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"9b38d7499ffc72bebdeea320c8948fda2213badb","unresolved":false,"context_lines":[{"line_number":846,"context_line":""},{"line_number":847,"context_line":"    def run_forever(self, *args, **kwargs):"},{"line_number":848,"context_line":"        self.logger.info(_(\"Starting object replicator in daemon mode.\"))"},{"line_number":849,"context_line":"        eventlet.spawn_n(self._child_process_reaper)"},{"line_number":850,"context_line":"        # Run the replicator continually"},{"line_number":851,"context_line":"        while True:"},{"line_number":852,"context_line":"            self._zero_stats()"}],"source_content_type":"text/x-python","patch_set":3,"id":"bf659307_0838ccec","line":849,"updated":"2018-03-29 04:43:52.000000000","message":"Any downside to never having a wait() on this?\n\nDo/should we have any bound on how big can the queue get?","commit_id":"ecf8ae50e14db146f9368cda73a265a92663ef03"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"864f169317689eaac7fdef70c64ef99b325e79d6","unresolved":false,"context_lines":[{"line_number":846,"context_line":""},{"line_number":847,"context_line":"    def run_forever(self, *args, **kwargs):"},{"line_number":848,"context_line":"        self.logger.info(_(\"Starting object replicator in daemon mode.\"))"},{"line_number":849,"context_line":"        eventlet.spawn_n(self._child_process_reaper)"},{"line_number":850,"context_line":"        # Run the replicator continually"},{"line_number":851,"context_line":"        while True:"},{"line_number":852,"context_line":"            self._zero_stats()"}],"source_content_type":"text/x-python","patch_set":3,"id":"bf659307_7c3466de","line":849,"in_reply_to":"bf659307_0838ccec","updated":"2018-04-02 20:42:41.000000000","message":"\u003e Do/should we have any bound on how big can the queue get?\n\nThe more I think about this, the more it seems like a memory leak waiting to happen...","commit_id":"ecf8ae50e14db146f9368cda73a265a92663ef03"},{"author":{"_account_id":2622,"name":"Samuel Merritt","email":"spam+launchpad@andcheese.org","username":"torgomatic"},"change_message_id":"c379611a2072b90c73bf959210298b40e1617f2e","unresolved":false,"context_lines":[{"line_number":846,"context_line":""},{"line_number":847,"context_line":"    def run_forever(self, *args, **kwargs):"},{"line_number":848,"context_line":"        self.logger.info(_(\"Starting object replicator in daemon mode.\"))"},{"line_number":849,"context_line":"        eventlet.spawn_n(self._child_process_reaper)"},{"line_number":850,"context_line":"        # Run the replicator continually"},{"line_number":851,"context_line":"        while True:"},{"line_number":852,"context_line":"            self._zero_stats()"}],"source_content_type":"text/x-python","patch_set":3,"id":"bf659307_bcce7e60","line":849,"in_reply_to":"bf659307_0838ccec","updated":"2018-04-02 21:12:10.000000000","message":"Not really. spawn_n starts the greenthread up but doesn\u0027t return a reference or anything. In run_forever mode, we don\u0027t exit, so there\u0027s no spot at which we\u0027d wait.","commit_id":"ecf8ae50e14db146f9368cda73a265a92663ef03"},{"author":{"_account_id":2622,"name":"Samuel Merritt","email":"spam+launchpad@andcheese.org","username":"torgomatic"},"change_message_id":"c379611a2072b90c73bf959210298b40e1617f2e","unresolved":false,"context_lines":[{"line_number":846,"context_line":""},{"line_number":847,"context_line":"    def run_forever(self, *args, **kwargs):"},{"line_number":848,"context_line":"        self.logger.info(_(\"Starting object replicator in daemon mode.\"))"},{"line_number":849,"context_line":"        eventlet.spawn_n(self._child_process_reaper)"},{"line_number":850,"context_line":"        # Run the replicator continually"},{"line_number":851,"context_line":"        while True:"},{"line_number":852,"context_line":"            self._zero_stats()"}],"source_content_type":"text/x-python","patch_set":3,"id":"bf659307_5c68821b","line":849,"in_reply_to":"bf659307_7c3466de","updated":"2018-04-02 21:12:10.000000000","message":"There\u0027s an implicit bound on there of 2^16 or so. If your process table gets that full, your system is properly hosed and you won\u0027t be able to fork() any more.","commit_id":"ecf8ae50e14db146f9368cda73a265a92663ef03"}],"test/unit/obj/test_replicator.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"0d47813c796f5dcc88a6dd251ce1b065d1e53e44","unresolved":false,"context_lines":[{"line_number":2034,"context_line":"        warnings \u003d self.logger.get_lines_for_level(\u0027warning\u0027)"},{"line_number":2035,"context_line":"        self.assertIn("},{"line_number":2036,"context_line":"            \"next_part_power set in policy \u0027one\u0027. Skipping\", warnings)"},{"line_number":2037,"context_line":""},{"line_number":2038,"context_line":"    def test_replicate_rsync_timeout(self):"},{"line_number":2039,"context_line":"        cur_part \u003d \u00270\u0027"},{"line_number":2040,"context_line":"        df \u003d self.df_mgr.get_diskfile(\u0027sda\u0027, cur_part, \u0027a\u0027, \u0027c\u0027, \u0027o\u0027,"}],"source_content_type":"text/x-python","patch_set":1,"id":"df7087c5_e828dfe2","line":2037,"updated":"2018-03-13 21:48:39.000000000","message":"this test is maybe not relevant anymore - but the new code/interfaces need some unittests","commit_id":"5e8f1b3dc98e983c0bbd46a044c33da40304d482"},{"author":{"_account_id":2622,"name":"Samuel Merritt","email":"spam+launchpad@andcheese.org","username":"torgomatic"},"change_message_id":"004a07f80be7dd9a97c8eefb68e429b11f21f22f","unresolved":false,"context_lines":[{"line_number":2034,"context_line":"        warnings \u003d self.logger.get_lines_for_level(\u0027warning\u0027)"},{"line_number":2035,"context_line":"        self.assertIn("},{"line_number":2036,"context_line":"            \"next_part_power set in policy \u0027one\u0027. Skipping\", warnings)"},{"line_number":2037,"context_line":""},{"line_number":2038,"context_line":"    def test_replicate_rsync_timeout(self):"},{"line_number":2039,"context_line":"        cur_part \u003d \u00270\u0027"},{"line_number":2040,"context_line":"        df \u003d self.df_mgr.get_diskfile(\u0027sda\u0027, cur_part, \u0027a\u0027, \u0027c\u0027, \u0027o\u0027,"}],"source_content_type":"text/x-python","patch_set":1,"id":"df7087c5_174f5493","line":2037,"in_reply_to":"df7087c5_e828dfe2","updated":"2018-03-14 23:15:35.000000000","message":"Done","commit_id":"5e8f1b3dc98e983c0bbd46a044c33da40304d482"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"93d5fd6a0292e4b70d3e0bf0d95f83d7394d3a00","unresolved":false,"context_lines":[{"line_number":2095,"context_line":"        with mock.patch(\u0027swift.obj.replicator.http_connect\u0027,"},{"line_number":2096,"context_line":"                        mock_http_connect(200)), \\"},{"line_number":2097,"context_line":"                mock.patch.object(self.replicator, \u0027rsync_timeout\u0027, 0.01), \\"},{"line_number":2098,"context_line":"                mock.patch(\u0027eventlet.green.subprocess.Popen\u0027, new_mock):"},{"line_number":2099,"context_line":"            self.replicator.run_once()"},{"line_number":2100,"context_line":"        for proc in mock_procs:"},{"line_number":2101,"context_line":"            self.assertEqual(proc._calls, ["}],"source_content_type":"text/x-python","patch_set":3,"id":"bf659307_c74488ba","line":2098,"updated":"2018-03-23 19:29:09.000000000","message":"i like this!","commit_id":"ecf8ae50e14db146f9368cda73a265a92663ef03"}]}
