)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":6926,"name":"Bogdan Dobrelya","email":"bdobreli@redhat.com","username":"bogdando"},"change_message_id":"730ca11923778497441aad3365508e2753e068ec","unresolved":false,"context_lines":[{"line_number":11,"context_line":"being processed, or another (unlocked) images."},{"line_number":12,"context_line":""},{"line_number":13,"context_line":"This reimplements I477219b7dca1e6cfa02a278c55a0cc1a9833d007 in the"},{"line_number":14,"context_line":"locks-free and non-blocking way. Locking both images and its layers"},{"line_number":15,"context_line":"together with emoving exp-backoff sleeping of workers, when there is"},{"line_number":16,"context_line":"collisions happen on layers allows running more concurrent workers"},{"line_number":17,"context_line":"in the more efficient way (each worker always does some job and never"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":11,"id":"3fa7e38b_05846c0b","line":14,"range":{"start_line":14,"start_character":0,"end_line":14,"end_character":5},"updated":"2019-10-08 08:23:57.000000000","message":"nit: actually I was trying to say \"collisions-free\" as we still do use locking here","commit_id":"41f910f31d39862bf0a98fee65293a27597571b4"}],"tripleo_common/image/image_uploader.py":[{"author":{"_account_id":14985,"name":"Alex Schultz","email":"aschultz@next-development.com","username":"mwhahaha"},"change_message_id":"efa8ca7868aa3d9fe36890987e4195a5112cd19a","unresolved":false,"context_lines":[{"line_number":1546,"context_line":"        try:"},{"line_number":1547,"context_line":"            cls._fetch_lock(layer)"},{"line_number":1548,"context_line":"            if cls._target_layer_exists_registry("},{"line_number":1549,"context_line":"                    target_url, layer_entry, [layer_entry], target_session):"},{"line_number":1550,"context_line":"                cls._fetch_unlock(layer)"},{"line_number":1551,"context_line":"                return"},{"line_number":1552,"context_line":"        except (ImageUploaderThreadException, ImageUploaderSkippedException):"}],"source_content_type":"text/x-python","patch_set":14,"id":"3fa7e38b_5cf0f6c9","line":1549,"updated":"2019-10-08 14:57:08.000000000","message":"This is the quick check where we don\u0027t actually refetch the layer because it already exists. But at you can see it is in lock section.","commit_id":"fdf6995152580d6730160d8c62f81e00d71d762c"},{"author":{"_account_id":14985,"name":"Alex Schultz","email":"aschultz@next-development.com","username":"mwhahaha"},"change_message_id":"4fec860c7e06a08a6401cd6c63f2d2ed06e62c33","unresolved":false,"context_lines":[{"line_number":1929,"context_line":""},{"line_number":1930,"context_line":"        export \u003d target_url.netloc in cls.export_registries"},{"line_number":1931,"context_line":"        if export:"},{"line_number":1932,"context_line":"            return image_export.export_stream("},{"line_number":1933,"context_line":"                target_url, layer, layer_stream, verify_digest\u003dverify_digest)"},{"line_number":1934,"context_line":""},{"line_number":1935,"context_line":"        for chunk in layer_stream:"},{"line_number":1936,"context_line":"            if not chunk:"}],"source_content_type":"text/x-python","patch_set":14,"id":"3fa7e38b_61daed17","line":1933,"range":{"start_line":1932,"start_character":12,"end_line":1933,"end_character":77},"updated":"2019-10-08 14:32:04.000000000","message":"This is the function under threading/python2 that basically turns it single threaded.","commit_id":"fdf6995152580d6730160d8c62f81e00d71d762c"},{"author":{"_account_id":14985,"name":"Alex Schultz","email":"aschultz@next-development.com","username":"mwhahaha"},"change_message_id":"4fec860c7e06a08a6401cd6c63f2d2ed06e62c33","unresolved":false,"context_lines":[{"line_number":2170,"context_line":""},{"line_number":2171,"context_line":"        # workers will be half the CPU, with a minimum of 4"},{"line_number":2172,"context_line":"        workers \u003d max(4, processutils.get_worker_count() // 2)"},{"line_number":2173,"context_line":"        with futures.ThreadPoolExecutor(max_workers\u003dworkers) as p:"},{"line_number":2174,"context_line":"            for task in tasks:"},{"line_number":2175,"context_line":"                jobs.append(p.submit(self.upload_task, task))"},{"line_number":2176,"context_line":""}],"source_content_type":"text/x-python","patch_set":14,"id":"3fa7e38b_61c50d3c","line":2173,"range":{"start_line":2173,"start_character":21,"end_line":2173,"end_character":39},"updated":"2019-10-08 14:32:04.000000000","message":"So the issues that switching the executor from thread to process addresses is still not solved by this patch. The problem is the concurrent IO done when fetching the layers when run under python2 effectively makes this single threaded.","commit_id":"fdf6995152580d6730160d8c62f81e00d71d762c"},{"author":{"_account_id":6926,"name":"Bogdan Dobrelya","email":"bdobreli@redhat.com","username":"bogdando"},"change_message_id":"6ae7e78cbc7656bf98ee43317fbcafd4db0ce2f8","unresolved":false,"context_lines":[{"line_number":2170,"context_line":""},{"line_number":2171,"context_line":"        # workers will be half the CPU, with a minimum of 4"},{"line_number":2172,"context_line":"        workers \u003d max(4, processutils.get_worker_count() // 2)"},{"line_number":2173,"context_line":"        with futures.ThreadPoolExecutor(max_workers\u003dworkers) as p:"},{"line_number":2174,"context_line":"            for task in tasks:"},{"line_number":2175,"context_line":"                jobs.append(p.submit(self.upload_task, task))"},{"line_number":2176,"context_line":""}],"source_content_type":"text/x-python","patch_set":14,"id":"3fa7e38b_627bccbd","line":2173,"range":{"start_line":2173,"start_character":21,"end_line":2173,"end_character":39},"in_reply_to":"3fa7e38b_3c317aa3","updated":"2019-10-08 15:52:54.000000000","message":"Please see https://bugs.launchpad.net/tripleo/+bug/1847225\nThe snippets I placed there illustrate that real multiple fetching happens (to my understanding)?","commit_id":"fdf6995152580d6730160d8c62f81e00d71d762c"},{"author":{"_account_id":14985,"name":"Alex Schultz","email":"aschultz@next-development.com","username":"mwhahaha"},"change_message_id":"efa8ca7868aa3d9fe36890987e4195a5112cd19a","unresolved":false,"context_lines":[{"line_number":2170,"context_line":""},{"line_number":2171,"context_line":"        # workers will be half the CPU, with a minimum of 4"},{"line_number":2172,"context_line":"        workers \u003d max(4, processutils.get_worker_count() // 2)"},{"line_number":2173,"context_line":"        with futures.ThreadPoolExecutor(max_workers\u003dworkers) as p:"},{"line_number":2174,"context_line":"            for task in tasks:"},{"line_number":2175,"context_line":"                jobs.append(p.submit(self.upload_task, task))"},{"line_number":2176,"context_line":""}],"source_content_type":"text/x-python","patch_set":14,"id":"3fa7e38b_3c317aa3","line":2173,"range":{"start_line":2173,"start_character":21,"end_line":2173,"end_character":39},"in_reply_to":"3fa7e38b_3c873a64","updated":"2019-10-08 14:57:08.000000000","message":"I believe it\u0027s a py2 problem when dealing with the IO. The rhel8 jobs don\u0027t exhibit the same problems.  This is likely a moot issue once we can finally get centos8 out the door.  The locking prevents the actual refetching of the layers because we check if the layer already exists on the disk. While we end up \"reprocessing\" the layers over and over (thanks containers), we don\u0027t actually go to the source to refetch it once I added locking.  Previously we were actually concurrently fetching the same layers. We reduced this impact by 1) shuffling the containers so that we don\u0027t concurrently process the same service containers (e.g. cinder-api, cinder-volume, cinder-backup, etc) and 2) adding locking to prevent the same layer from being worked on if it\u0027s already in the process of being fetched.  Once the layer is on disk and been pushed to the registry, we unlock it which allows the locked containers to start the processing. One of the actions is to check if the layer is already available and if so then do the filesystem hard link. Thats why we end up with log messages indicating that the layers are being locked/unlocked.","commit_id":"fdf6995152580d6730160d8c62f81e00d71d762c"},{"author":{"_account_id":14985,"name":"Alex Schultz","email":"aschultz@next-development.com","username":"mwhahaha"},"change_message_id":"eca4a523d04a4a978f53536f5f65254f7bec379f","unresolved":false,"context_lines":[{"line_number":2170,"context_line":""},{"line_number":2171,"context_line":"        # workers will be half the CPU, with a minimum of 4"},{"line_number":2172,"context_line":"        workers \u003d max(4, processutils.get_worker_count() // 2)"},{"line_number":2173,"context_line":"        with futures.ThreadPoolExecutor(max_workers\u003dworkers) as p:"},{"line_number":2174,"context_line":"            for task in tasks:"},{"line_number":2175,"context_line":"                jobs.append(p.submit(self.upload_task, task))"},{"line_number":2176,"context_line":""}],"source_content_type":"text/x-python","patch_set":14,"id":"3fa7e38b_45f33a9f","line":2173,"range":{"start_line":2173,"start_character":21,"end_line":2173,"end_character":39},"in_reply_to":"3fa7e38b_422ed0a5","updated":"2019-10-08 16:22:42.000000000","message":"So there\u0027s a race condition somewhere, likely because these layers are small layers. We aren\u0027t duplicating the fetch for all of them. For example in my log, I had the following for a bunch of containers:\n\n\n2019-10-04 16:44:15 334467 DEBUG tripleo_common.image.image_uploader [  ] [/tripleomaster/centos-binary-nova-conductor] Layer already exists: sha256:d8d02d45731499028db01b6fa35475f91d230628b4e25fab8e3c015594dc3261\n\n[centos@centos7-a code]$ grep sha256:d8d02d45731499028db01b6fa35475f91d230628b4e25fab8e3c015594dc3261 prepare.log | grep Read\n2019-10-04 16:41:58 329843 DEBUG tripleo_common.image.image_uploader [  ] [/tripleomaster/centos-binary-nova-compute-ironic] Read 75412258 bytes for sha256:d8d02d45731499028db01b6fa35475f91d230628b4e25fab8e3c015594dc3261\n\n\nIt was only fetched once but shared for 87 containers.  However I do have some layers that were fetched 4 or 5 times.\n\n\n[centos@centos7-a code]$ grep sha256:69f57df07f28946b1585acd76e4d65cccf536c573d5af7ba8371488d9c7a96b0 prepare.log  | grep Read\n2019-10-04 16:44:24 334458 DEBUG tripleo_common.image.image_uploader [  ] [/tripleomaster/centos-binary-neutron-metadata-agent] Read 3384 bytes for sha256:69f57df07f28946b1585acd76e4d65cccf536c573d5af7ba8371488d9c7a96b0\n2019-10-04 16:45:40 334462 DEBUG tripleo_common.image.image_uploader [  ] [/tripleomaster/centos-binary-neutron-l3-agent] Read 3384 bytes for sha256:69f57df07f28946b1585acd76e4d65cccf536c573d5af7ba8371488d9c7a96b0\n2019-10-04 16:45:47 334458 DEBUG tripleo_common.image.image_uploader [  ] [/tripleomaster/centos-binary-neutron-metadata-agent-ovn] Read 3384 bytes for sha256:69f57df07f28946b1585acd76e4d65cccf536c573d5af7ba8371488d9c7a96b0\n2019-10-04 16:47:50 334458 DEBUG tripleo_common.image.image_uploader [  ] [/tripleomaster/centos-binary-neutron-openvswitch-agent] Read 3384 bytes for sha256:69f57df07f28946b1585acd76e4d65cccf536c573d5af7ba8371488d9c7a96b0\n2019-10-04 16:51:00 334458 DEBUG tripleo_common.image.image_uploader [  ] [/tripleomaster/centos-binary-neutron-dhcp-agent] Read 3384 bytes for sha256:69f57df07f28946b1585acd76e4d65cccf536c573d5af7ba8371488d9c7a96b0\n2019-10-04 16:52:01 334458 DEBUG tripleo_common.image.image_uploader [  ] [/tripleomaster/centos-binary-ironic-neutron-agent] Read 3384 bytes for sha256:69f57df07f28946b1585acd76e4d65cccf536c573d5af7ba8371488d9c7a96b0\n2019-10-04 16:52:15 334462 DEBUG tripleo_common.image.image_uploader [  ] [/tripleomaster/centos-binary-neutron-server-ovn] Read 3384 bytes for sha256:69f57df07f28946b1585acd76e4d65cccf536c573d5af7ba8371488d9c7a96b0","commit_id":"fdf6995152580d6730160d8c62f81e00d71d762c"},{"author":{"_account_id":14985,"name":"Alex Schultz","email":"aschultz@next-development.com","username":"mwhahaha"},"change_message_id":"c2d424dabc0b895688ad46227c931be2da29195d","unresolved":false,"context_lines":[{"line_number":2170,"context_line":""},{"line_number":2171,"context_line":"        # workers will be half the CPU, with a minimum of 4"},{"line_number":2172,"context_line":"        workers \u003d max(4, processutils.get_worker_count() // 2)"},{"line_number":2173,"context_line":"        with futures.ThreadPoolExecutor(max_workers\u003dworkers) as p:"},{"line_number":2174,"context_line":"            for task in tasks:"},{"line_number":2175,"context_line":"                jobs.append(p.submit(self.upload_task, task))"},{"line_number":2176,"context_line":""}],"source_content_type":"text/x-python","patch_set":14,"id":"3fa7e38b_68ee4db5","line":2173,"range":{"start_line":2173,"start_character":21,"end_line":2173,"end_character":39},"in_reply_to":"3fa7e38b_45f33a9f","updated":"2019-10-08 17:05:50.000000000","message":"Interesting enough is that the layers that get pulled multiple times do not actually get linked together like the layers that are properly identified as already existing. For example,\n\n\n[centos@centos7-a code]$ stat /var/lib/image-serve/v2/tripleomaster/centos-binary-neutron-server-ovn/blobs/sha256:d8d02d45731499028db01b6fa35475f91d230628b4e25fab8e3c015594dc3261.gz\n  File: ‘/var/lib/image-serve/v2/tripleomaster/centos-binary-neutron-server-ovn/blobs/sha256:d8d02d45731499028db01b6fa35475f91d230628b4e25fab8e3c015594dc3261.gz’\n  Size: 75412258  \tBlocks: 147296     IO Block: 4096   regular file\nDevice: fd01h/64769d\tInode: 234897793   Links: 88\nAccess: (0644/-rw-r--r--)  Uid: (    0/    root)   Gid: (    0/    root)\nContext: unconfined_u:object_r:httpd_sys_content_t:s0\nAccess: 2019-10-08 16:47:44.486378901 +0000\nModify: 2019-10-04 16:41:59.167831617 +0000\nChange: 2019-10-04 16:52:14.881935927 +0000\n Birth: -\n\n\nvs \n\n\n[centos@centos7-a code]$ stat /var/lib/image-serve/v2/tripleomaster/centos-binary-neutron-server-ovn/blobs/sha256:69f57df07f28946b1585acd76e4d65cccf536c573d5af7ba8371488d9c7a96b0.gz\n  File: ‘/var/lib/image-serve/v2/tripleomaster/centos-binary-neutron-server-ovn/blobs/sha256:69f57df07f28946b1585acd76e4d65cccf536c573d5af7ba8371488d9c7a96b0.gz’\n  Size: 3384      \tBlocks: 8          IO Block: 4096   regular file\nDevice: fd01h/64769d\tInode: 12673224    Links: 1\nAccess: (0644/-rw-r--r--)  Uid: (    0/    root)   Gid: (    0/    root)\nContext: unconfined_u:object_r:httpd_sys_content_t:s0\nAccess: 2019-10-04 16:52:15.033936199 +0000\nModify: 2019-10-04 16:52:15.146936402 +0000\nChange: 2019-10-04 16:52:15.146936402 +0000\n Birth: -\n\n\nThere\u0027s also no \"Linking layers:\" log item for the non-deduplicated layer.\n\n\n[centos@centos7-a code]$ grep sha256:69f57df07f28946b1585acd76e4d65cccf536c573d5af7ba8371488d9c7a96b0 prepare.log | grep -i link\n[centos@centos7-a code]$ grep sha256:d8d02d45731499028db01b6fa35475f91d230628b4e25fab8e3c015594dc3261 prepare.log | grep -i link\n2019-10-04 16:44:15 334467 DEBUG tripleo_common.image.image_export [  ] [tripleomaster/centos-binary-nova-compute-ironic] Linking layers: /var/lib/image-serve/v2/tripleomaster/centos-binary-nova-compute-ironic/blobs/sha256:d8d02d45731499028db01b6fa35475f91d230628b4e25fab8e3c015594dc3261.gz -\u003e /var/lib/image-serve/v2/tripleomaster/centos-binary-nova-conductor/blobs/sha256:d8d02d45731499028db01b6fa35475f91d230628b4e25fab8e3c015594dc3261.gz\n2019-10-04 16:44:15 334456 DEBUG tripleo_common.image.image_export [  ] [tripleomaster/centos-binary-nova-compute-ironic] Linking layers: /var/lib/image-serve/v2/tripleomaster/centos-binary-nova-compute-ironic/blobs/sha256:d8d02d45731499028db01b6fa35475f91d230628b4e25fab8e3c015594dc3261.gz -\u003e /var/lib/image-serve/v2/tripleomaster/centos-binary-designate-central/blobs/sha256:d8d02d45731499028db01b6fa35475f91d230628b4e25fab8e3c015594dc3261.gz\n... snip ...\n2019-10-04 16:52:14 334462 DEBUG tripleo_common.image.image_export [  ] [tripleomaster/centos-binary-nova-compute-ironic] Linking layers: /var/lib/image-serve/v2/tripleomaster/centos-binary-nova-compute-ironic/blobs/sha256:d8d02d45731499028db01b6fa35475f91d230628b4e25fab8e3c015594dc3261.gz -\u003e /var/lib/image-serve/v2/tripleomaster/centos-binary-neutron-server-ovn/blobs/sha256:d8d02d45731499028db01b6fa35475f91d230628b4e25fab8e3c015594dc3261.gz","commit_id":"fdf6995152580d6730160d8c62f81e00d71d762c"},{"author":{"_account_id":6926,"name":"Bogdan Dobrelya","email":"bdobreli@redhat.com","username":"bogdando"},"change_message_id":"17ef9efacff068000eb25ec0ccf0761d779b3622","unresolved":false,"context_lines":[{"line_number":2170,"context_line":""},{"line_number":2171,"context_line":"        # workers will be half the CPU, with a minimum of 4"},{"line_number":2172,"context_line":"        workers \u003d max(4, processutils.get_worker_count() // 2)"},{"line_number":2173,"context_line":"        with futures.ThreadPoolExecutor(max_workers\u003dworkers) as p:"},{"line_number":2174,"context_line":"            for task in tasks:"},{"line_number":2175,"context_line":"                jobs.append(p.submit(self.upload_task, task))"},{"line_number":2176,"context_line":""}],"source_content_type":"text/x-python","patch_set":14,"id":"3fa7e38b_3c873a64","line":2173,"range":{"start_line":2173,"start_character":21,"end_line":2173,"end_character":39},"in_reply_to":"3fa7e38b_61c50d3c","updated":"2019-10-08 14:50:50.000000000","message":"I wonder if this critical section is really impacts anything as those 60M layers are streamed quite fast? I see the issue here is \"low quality\" (sleeping/blocking each over) workers repeating itself, like fetching same layers all the time","commit_id":"fdf6995152580d6730160d8c62f81e00d71d762c"},{"author":{"_account_id":14985,"name":"Alex Schultz","email":"aschultz@next-development.com","username":"mwhahaha"},"change_message_id":"8697e6658b1f0e4a7e71313b1e782aca811f074a","unresolved":false,"context_lines":[{"line_number":2170,"context_line":""},{"line_number":2171,"context_line":"        # workers will be half the CPU, with a minimum of 4"},{"line_number":2172,"context_line":"        workers \u003d max(4, processutils.get_worker_count() // 2)"},{"line_number":2173,"context_line":"        with futures.ThreadPoolExecutor(max_workers\u003dworkers) as p:"},{"line_number":2174,"context_line":"            for task in tasks:"},{"line_number":2175,"context_line":"                jobs.append(p.submit(self.upload_task, task))"},{"line_number":2176,"context_line":""}],"source_content_type":"text/x-python","patch_set":14,"id":"3fa7e38b_422ed0a5","line":2173,"range":{"start_line":2173,"start_character":21,"end_line":2173,"end_character":39},"in_reply_to":"3fa7e38b_627bccbd","updated":"2019-10-08 15:58:41.000000000","message":"\u003e Please see https://bugs.launchpad.net/tripleo/+bug/1847225\n \u003e The snippets I placed there illustrate that real multiple fetching\n \u003e happens (to my understanding)?\n\nIt shouldn\u0027t be. i need to look at the logs further. If it\u0027s being fetched again then the existing check is failing.","commit_id":"fdf6995152580d6730160d8c62f81e00d71d762c"},{"author":{"_account_id":6926,"name":"Bogdan Dobrelya","email":"bdobreli@redhat.com","username":"bogdando"},"change_message_id":"d392db8a32eba349b9668c6dea49ffd2ef0e578d","unresolved":false,"context_lines":[{"line_number":2170,"context_line":""},{"line_number":2171,"context_line":"        # workers will be half the CPU, with a minimum of 4"},{"line_number":2172,"context_line":"        workers \u003d max(4, processutils.get_worker_count() // 2)"},{"line_number":2173,"context_line":"        with futures.ThreadPoolExecutor(max_workers\u003dworkers) as p:"},{"line_number":2174,"context_line":"            for task in tasks:"},{"line_number":2175,"context_line":"                jobs.append(p.submit(self.upload_task, task))"},{"line_number":2176,"context_line":""}],"source_content_type":"text/x-python","patch_set":14,"id":"3fa7e38b_0231b80b","line":2173,"range":{"start_line":2173,"start_character":21,"end_line":2173,"end_character":39},"in_reply_to":"3fa7e38b_627bccbd","updated":"2019-10-08 15:53:28.000000000","message":"I\u0027m investigating the latter in https://review.opendev.org/#/c/687288/4 as well","commit_id":"fdf6995152580d6730160d8c62f81e00d71d762c"},{"author":{"_account_id":6926,"name":"Bogdan Dobrelya","email":"bdobreli@redhat.com","username":"bogdando"},"change_message_id":"cd653628a9e3d2de0d592c4c8c69af13f7e756f2","unresolved":false,"context_lines":[{"line_number":2170,"context_line":""},{"line_number":2171,"context_line":"        # workers will be half the CPU, with a minimum of 4"},{"line_number":2172,"context_line":"        workers \u003d max(4, processutils.get_worker_count() // 2)"},{"line_number":2173,"context_line":"        with futures.ThreadPoolExecutor(max_workers\u003dworkers) as p:"},{"line_number":2174,"context_line":"            for task in tasks:"},{"line_number":2175,"context_line":"                jobs.append(p.submit(self.upload_task, task))"},{"line_number":2176,"context_line":""}],"source_content_type":"text/x-python","patch_set":14,"id":"3fa7e38b_b638a99f","line":2173,"range":{"start_line":2173,"start_character":21,"end_line":2173,"end_character":39},"in_reply_to":"3fa7e38b_68ee4db5","updated":"2019-10-16 12:45:32.000000000","message":"this is addressed by https://review.opendev.org/#/c/686756/","commit_id":"fdf6995152580d6730160d8c62f81e00d71d762c"}]}