)]}'
{"tripleo_common/image/image_export.py":[{"author":{"_account_id":6926,"name":"Bogdan Dobrelya","email":"bdobreli@redhat.com","username":"bogdando"},"change_message_id":"2fbdb09de2c4c367c2f27187792c31e3e8b940b4","unresolved":false,"context_lines":[{"line_number":178,"context_line":"        if layer not in image_layers and not known_path:"},{"line_number":179,"context_line":"            continue"},{"line_number":180,"context_line":""},{"line_number":181,"context_line":"        if not image and layer in image_layers:"},{"line_number":182,"context_line":"            image_url \u003d image_layers[layer]"},{"line_number":183,"context_line":"            image, _ \u003d image_tag_from_url(image_url)"},{"line_number":184,"context_line":""}],"source_content_type":"text/x-python","patch_set":19,"id":"3fa7e38b_916fde26","line":181,"range":{"start_line":181,"start_character":20,"end_line":181,"end_character":46},"updated":"2019-10-14 15:14:34.000000000","message":"this is probably redundant?","commit_id":"9f3cf0614638df715221d8a105bc0816c493ffa2"}],"tripleo_common/image/image_uploader.py":[{"author":{"_account_id":6926,"name":"Bogdan Dobrelya","email":"bdobreli@redhat.com","username":"bogdando"},"change_message_id":"b5805c5963999ff69d2a11db2bd8e074839f5f87","unresolved":false,"context_lines":[{"line_number":1587,"context_line":"            for job in futures.as_completed(copy_jobs):"},{"line_number":1588,"context_line":"                e \u003d job.exception()"},{"line_number":1589,"context_line":"                if e:"},{"line_number":1590,"context_line":"                    raise e"},{"line_number":1591,"context_line":"                layer \u003d job.result()"},{"line_number":1592,"context_line":"                if layer:"},{"line_number":1593,"context_line":"                    LOG.debug(\u0027[%s] Upload complete for layer %s\u0027 %"}],"source_content_type":"text/x-python","patch_set":4,"id":"3fa7e38b_fc37624c","line":1590,"updated":"2019-10-08 15:06:33.000000000","message":"the failure http://logs.rdoproject.org/88/687288/4/openstack-check/tripleo-ci-centos-7-ovb-3ctlr_1comp-featureset001/d046ccb/logs/undercloud/var/log/tripleo-container-image-prepare.log.txt.gz is real, but I cannot trace it properly :/","commit_id":"7ce703ae97921bc2fc0394f018601863e760a5d7"},{"author":{"_account_id":6926,"name":"Bogdan Dobrelya","email":"bdobreli@redhat.com","username":"bogdando"},"change_message_id":"1ea23a32b2b1a69bf6faedc69bebc1257a5d677c","unresolved":false,"context_lines":[{"line_number":1185,"context_line":""},{"line_number":1186,"context_line":"    @classmethod"},{"line_number":1187,"context_line":"    def _check_uploaded_layers(cls, layer):"},{"line_number":1188,"context_line":"        if layer in cls.processed_objects_info:"},{"line_number":1189,"context_line":"            LOG.debug(\u0027[%s] Layer is recognized as already uploaded\u0027"},{"line_number":1190,"context_line":"                      % layer)"},{"line_number":1191,"context_line":"            return True"}],"source_content_type":"text/x-python","patch_set":8,"id":"3fa7e38b_0e530f19","line":1188,"updated":"2019-10-09 15:51:52.000000000","message":"in the logs of my testing env I can see this code never runs yet, that\u0027s why layers keep being re-fetched after the lock lifts away","commit_id":"9f0681e08d30bbe7838930068f4fc2f114dbdde0"},{"author":{"_account_id":6926,"name":"Bogdan Dobrelya","email":"bdobreli@redhat.com","username":"bogdando"},"change_message_id":"ed874908568e956a4c5e35c5e3f67a5c3d936a66","unresolved":false,"context_lines":[{"line_number":1561,"context_line":"        else:"},{"line_number":1562,"context_line":"            # Track it for the global view of all workers"},{"line_number":1563,"context_line":"            # and for the *_layer_exists fast path using it as a cache"},{"line_number":1564,"context_line":"            cls._track_uploaded_layers(layer_val, lock)"},{"line_number":1565,"context_line":"            cls._track_uploaded_layers(layer, lock)"},{"line_number":1566,"context_line":"            return layer_val"},{"line_number":1567,"context_line":"        finally:"}],"source_content_type":"text/x-python","patch_set":8,"id":"3fa7e38b_43d480e9","line":1564,"updated":"2019-10-09 15:33:15.000000000","message":"looking into the streams exporting code, it seems we normally raise an error if layer_val !\u003d layer digests. So I\u0027m not sure if we need to track both here. Should be not a big deal if those are equal as we add it upon a set(), not a list. But still... not sure in the final form of this code","commit_id":"9f0681e08d30bbe7838930068f4fc2f114dbdde0"},{"author":{"_account_id":6926,"name":"Bogdan Dobrelya","email":"bdobreli@redhat.com","username":"bogdando"},"change_message_id":"0eb3cab89d5797ac1eb523226ff9605bc351bd2b","unresolved":false,"context_lines":[{"line_number":1783,"context_line":"                                      session):"},{"line_number":1784,"context_line":"        # Check, if the layer had been already uploaded into the"},{"line_number":1785,"context_line":"        # image-serve registry and is tracked in the global view"},{"line_number":1786,"context_line":"        # (it also acts here as a fast in-memory cache)"},{"line_number":1787,"context_line":"        if cls._check_uploaded_layers(layer.get(\u0027digest\u0027, None)):"},{"line_number":1788,"context_line":"            return True"},{"line_number":1789,"context_line":"        for l in check_layers:"}],"source_content_type":"text/x-python","patch_set":8,"id":"3fa7e38b_a353f415","line":1786,"updated":"2019-10-09 15:16:35.000000000","message":"I expect that provides a great shortcut as saves a few of a HTTP queries per ... quite often?","commit_id":"9f0681e08d30bbe7838930068f4fc2f114dbdde0"},{"author":{"_account_id":6926,"name":"Bogdan Dobrelya","email":"bdobreli@redhat.com","username":"bogdando"},"change_message_id":"ed874908568e956a4c5e35c5e3f67a5c3d936a66","unresolved":false,"context_lines":[{"line_number":1891,"context_line":"            layer_val \u003d cls._copy_stream_to_registry(target_url, layer,"},{"line_number":1892,"context_line":"                                                     calc_digest,"},{"line_number":1893,"context_line":"                                                     layer_stream, session,"},{"line_number":1894,"context_line":"                                                     verify_digest\u003dFalse)"},{"line_number":1895,"context_line":"        except Exception:"},{"line_number":1896,"context_line":"            # Untrack the bad layer from the global view of uploaded layers"},{"line_number":1897,"context_line":"            cls._track_uploaded_layers(layer_val, lock, forget\u003dTrue)"}],"source_content_type":"text/x-python","patch_set":8,"id":"3fa7e38b_63cc1c6a","line":1894,"range":{"start_line":1894,"start_character":53,"end_line":1894,"end_character":72},"updated":"2019-10-09 15:33:15.000000000","message":"I\u0027m not sure why we omit verifying digest for the local upload? Shall we treat the situation when layer_val\u0027d digest !\u003d layer digest as a problem and never track such layers in the global view cache?..","commit_id":"9f0681e08d30bbe7838930068f4fc2f114dbdde0"},{"author":{"_account_id":6926,"name":"Bogdan Dobrelya","email":"bdobreli@redhat.com","username":"bogdando"},"change_message_id":"6ae74ce839a3ca21a38c1c77f4ccf85ee4a8bd0c","unresolved":false,"context_lines":[{"line_number":1155,"context_line":"            LOG.debug(\u0027[%s] Layer recognized as already \u0027"},{"line_number":1156,"context_line":"                      \u0027uploaded for image %s to %s\u0027"},{"line_number":1157,"context_line":"                      % (layer, image, known_path))"},{"line_number":1158,"context_line":"            raise ImageUploaderLayerException(\u0027layer is already aploaded\u0027)"},{"line_number":1159,"context_line":"        LOG.debug(\u0027Locking layer %s\u0027 % layer)"},{"line_number":1160,"context_line":"        LOG.debug(\u0027Starting acquire for lock %s\u0027 % layer)"},{"line_number":1161,"context_line":"        with lock.get_lock():"}],"source_content_type":"text/x-python","patch_set":40,"id":"3fa7e38b_c12e5e8b","line":1158,"updated":"2019-10-23 10:41:46.000000000","message":"I think that was the issue (see my comment below). We should just return w/o a lock, when a layer was uploaded earlier","commit_id":"3e58fa8258b74a7cdd83b4dd374d86da24da9cb9"},{"author":{"_account_id":6926,"name":"Bogdan Dobrelya","email":"bdobreli@redhat.com","username":"bogdando"},"change_message_id":"d72327c3e8ebc853e5b08a692fb368b03e5ab464","unresolved":false,"context_lines":[{"line_number":1678,"context_line":"                if e:"},{"line_number":1679,"context_line":"                    raise e"},{"line_number":1680,"context_line":"                layer \u003d job.result()"},{"line_number":1681,"context_line":"                LOG.debug(\u0027[%s] Upload complete for layer %s\u0027 %"},{"line_number":1682,"context_line":"                          (image, layer))"},{"line_number":1683,"context_line":"                jobs_finished +\u003d 1"},{"line_number":1684,"context_line":"                LOG.debug(\u0027[%s] Waiting for next job: %i of %i complete\u0027 %"}],"source_content_type":"text/x-python","patch_set":40,"id":"3fa7e38b_8cd0af55","line":1681,"range":{"start_line":1681,"start_character":32,"end_line":1681,"end_character":47},"updated":"2019-10-22 15:03:50.000000000","message":"I can see in the logs that sometimes this message is logged for a layer that was taken by a worker for image X but it was waiting for another worker to finish it for another image Y referencing that layer as well. Then, we end up having the layer linked for Y(/Z/T...) but NOT for X! It seems the global view should somehow help us here to complete the missing cross-linking paths after each of the places like this one","commit_id":"3e58fa8258b74a7cdd83b4dd374d86da24da9cb9"},{"author":{"_account_id":6926,"name":"Bogdan Dobrelya","email":"bdobreli@redhat.com","username":"bogdando"},"change_message_id":"6ae74ce839a3ca21a38c1c77f4ccf85ee4a8bd0c","unresolved":false,"context_lines":[{"line_number":1678,"context_line":"                if e:"},{"line_number":1679,"context_line":"                    raise e"},{"line_number":1680,"context_line":"                layer \u003d job.result()"},{"line_number":1681,"context_line":"                LOG.debug(\u0027[%s] Upload complete for layer %s\u0027 %"},{"line_number":1682,"context_line":"                          (image, layer))"},{"line_number":1683,"context_line":"                jobs_finished +\u003d 1"},{"line_number":1684,"context_line":"                LOG.debug(\u0027[%s] Waiting for next job: %i of %i complete\u0027 %"}],"source_content_type":"text/x-python","patch_set":40,"id":"3fa7e38b_e1391ad8","line":1681,"range":{"start_line":1681,"start_character":32,"end_line":1681,"end_character":47},"in_reply_to":"3fa7e38b_8cd0af55","updated":"2019-10-23 10:41:46.000000000","message":"so the rev #41 seems resolved that issue.","commit_id":"3e58fa8258b74a7cdd83b4dd374d86da24da9cb9"},{"author":{"_account_id":6926,"name":"Bogdan Dobrelya","email":"bdobreli@redhat.com","username":"bogdando"},"change_message_id":"7f524207a41f77acdd5a333b57a9f73d54b959ef","unresolved":false,"context_lines":[{"line_number":1840,"context_line":"                LOG.debug(\u0027[%s] Layer %s already exists at %s\u0027 %"},{"line_number":1841,"context_line":"                          (image, l[\u0027digest\u0027], known_path))"},{"line_number":1842,"context_line":"                layer_found \u003d l"},{"line_number":1843,"context_line":"                break"},{"line_number":1844,"context_line":"            else:"},{"line_number":1845,"context_line":"                parts[\u0027digest\u0027] \u003d l[\u0027digest\u0027]"},{"line_number":1846,"context_line":"                blob_url \u003d cls._build_url("}],"source_content_type":"text/x-python","patch_set":62,"id":"3fa7e38b_4c1bba1b","line":1843,"updated":"2019-11-07 11:53:22.000000000","message":"I think that was the reason this was returning wrong l[digest] et al info in the end of the loop. The original code was returning True right here w/o further iterating.","commit_id":"33431d033070dffe6f847d572668456e9b15b4f1"},{"author":{"_account_id":14985,"name":"Alex Schultz","email":"aschultz@next-development.com","username":"mwhahaha"},"change_message_id":"0e2be3e02380d86c068f2857e6b065db6f70b7f6","unresolved":false,"context_lines":[{"line_number":2089,"context_line":"            return"},{"line_number":2090,"context_line":"        local_images \u003d []"},{"line_number":2091,"context_line":""},{"line_number":2092,"context_line":"        # Pull a single image first, to avoid duplicate pulls of the"},{"line_number":2093,"context_line":"        # same base layers"},{"line_number":2094,"context_line":"        local_images.extend(upload_task(args\u003dself.upload_tasks.pop()))"},{"line_number":2095,"context_line":""},{"line_number":2096,"context_line":"        with self._get_executor() as p:"},{"line_number":2097,"context_line":"            for result in p.map(upload_task, self.upload_tasks):"}],"source_content_type":"text/x-python","patch_set":65,"id":"3fa7e38b_79abc41d","side":"PARENT","line":2094,"range":{"start_line":2092,"start_character":0,"end_line":2094,"end_character":70},"updated":"2019-11-08 15:13:36.000000000","message":"much of the time improvements are probably from this since we don\u0027t have the delay of the initial fetch. I wonder how much improvement we would get just from doing this","commit_id":"26bd0efd264a2bcbcc1f4033ea23ec4a851d29bd"},{"author":{"_account_id":6926,"name":"Bogdan Dobrelya","email":"bdobreli@redhat.com","username":"bogdando"},"change_message_id":"cdc5221fbff21132d0aa1a865355ef4bf3b69c7a","unresolved":false,"context_lines":[{"line_number":2089,"context_line":"            return"},{"line_number":2090,"context_line":"        local_images \u003d []"},{"line_number":2091,"context_line":""},{"line_number":2092,"context_line":"        # Pull a single image first, to avoid duplicate pulls of the"},{"line_number":2093,"context_line":"        # same base layers"},{"line_number":2094,"context_line":"        local_images.extend(upload_task(args\u003dself.upload_tasks.pop()))"},{"line_number":2095,"context_line":""},{"line_number":2096,"context_line":"        with self._get_executor() as p:"},{"line_number":2097,"context_line":"            for result in p.map(upload_task, self.upload_tasks):"}],"source_content_type":"text/x-python","patch_set":65,"id":"3fa7e38b_141e15bb","side":"PARENT","line":2094,"range":{"start_line":2092,"start_character":0,"end_line":2094,"end_character":70},"in_reply_to":"3fa7e38b_79598459","updated":"2019-11-08 16:10:50.000000000","message":"just checked that locally, removing only this part makes things only terrible slow and re-fetching much more layers all the time","commit_id":"26bd0efd264a2bcbcc1f4033ea23ec4a851d29bd"},{"author":{"_account_id":6926,"name":"Bogdan Dobrelya","email":"bdobreli@redhat.com","username":"bogdando"},"change_message_id":"75ff7b81fb00796abba199b7b0e334fedb62e6f5","unresolved":false,"context_lines":[{"line_number":2089,"context_line":"            return"},{"line_number":2090,"context_line":"        local_images \u003d []"},{"line_number":2091,"context_line":""},{"line_number":2092,"context_line":"        # Pull a single image first, to avoid duplicate pulls of the"},{"line_number":2093,"context_line":"        # same base layers"},{"line_number":2094,"context_line":"        local_images.extend(upload_task(args\u003dself.upload_tasks.pop()))"},{"line_number":2095,"context_line":""},{"line_number":2096,"context_line":"        with self._get_executor() as p:"},{"line_number":2097,"context_line":"            for result in p.map(upload_task, self.upload_tasks):"}],"source_content_type":"text/x-python","patch_set":65,"id":"3fa7e38b_79598459","side":"PARENT","line":2094,"range":{"start_line":2092,"start_character":0,"end_line":2094,"end_character":70},"in_reply_to":"3fa7e38b_79abc41d","updated":"2019-11-08 15:16:37.000000000","message":"it\u0027s hard to say.\nthere is more than only time improvements, please see for the analysys have made for the banchmarking document I posted earlier","commit_id":"26bd0efd264a2bcbcc1f4033ea23ec4a851d29bd"},{"author":{"_account_id":8449,"name":"Marios Andreou","email":"marios.andreou@gmail.com","username":"marios"},"change_message_id":"27d960ae677097b7fd907880bbcb0372e46fd099","unresolved":false,"context_lines":[{"line_number":1183,"context_line":"        LOG.debug(\u0027Released lock on layer %s\u0027 % layer)"},{"line_number":1184,"context_line":""},{"line_number":1185,"context_line":"    @classmethod"},{"line_number":1186,"context_line":"    def _global_view_proxy(cls, value\u003dNone, forget\u003dFalse):"},{"line_number":1187,"context_line":"        if not cls.lock:"},{"line_number":1188,"context_line":"            LOG.warning(\u0027No lock information provided for value %s\u0027 % value)"},{"line_number":1189,"context_line":"            return"}],"source_content_type":"text/x-python","patch_set":65,"id":"3fa7e38b_e281f14e","line":1186,"updated":"2019-11-15 08:03:28.000000000","message":"perhaps adding some comments on each function with docstring will help reviewers (+ anyone else that comes across it in the future)","commit_id":"46f81298948865a2c15c4a5035e95a0c77adb5d5"},{"author":{"_account_id":6926,"name":"Bogdan Dobrelya","email":"bdobreli@redhat.com","username":"bogdando"},"change_message_id":"602fd12856780da8599403c7f3f5ff1ca4986f05","unresolved":false,"context_lines":[{"line_number":1183,"context_line":"        LOG.debug(\u0027Released lock on layer %s\u0027 % layer)"},{"line_number":1184,"context_line":""},{"line_number":1185,"context_line":"    @classmethod"},{"line_number":1186,"context_line":"    def _global_view_proxy(cls, value\u003dNone, forget\u003dFalse):"},{"line_number":1187,"context_line":"        if not cls.lock:"},{"line_number":1188,"context_line":"            LOG.warning(\u0027No lock information provided for value %s\u0027 % value)"},{"line_number":1189,"context_line":"            return"}],"source_content_type":"text/x-python","patch_set":65,"id":"3fa7e38b_2f035b78","line":1186,"in_reply_to":"3fa7e38b_3a0877d7","updated":"2020-01-02 08:19:58.000000000","message":"here it is https://review.opendev.org/700873\n\nand just the in-line comments I posted all around should also help to get it hopefully","commit_id":"46f81298948865a2c15c4a5035e95a0c77adb5d5"},{"author":{"_account_id":9712,"name":"Dougal Matthews","email":"dougal.matthews@canonical.com","username":"dougal"},"change_message_id":"729fd1ddc2b9c3af8ec9a7a2b9a372d9355a2437","unresolved":false,"context_lines":[{"line_number":1183,"context_line":"        LOG.debug(\u0027Released lock on layer %s\u0027 % layer)"},{"line_number":1184,"context_line":""},{"line_number":1185,"context_line":"    @classmethod"},{"line_number":1186,"context_line":"    def _global_view_proxy(cls, value\u003dNone, forget\u003dFalse):"},{"line_number":1187,"context_line":"        if not cls.lock:"},{"line_number":1188,"context_line":"            LOG.warning(\u0027No lock information provided for value %s\u0027 % value)"},{"line_number":1189,"context_line":"            return"}],"source_content_type":"text/x-python","patch_set":65,"id":"3fa7e38b_3a0877d7","line":1186,"in_reply_to":"3fa7e38b_a2e779ac","updated":"2019-12-06 11:03:01.000000000","message":"Did this follow up happen? Just reading and trying to understand this code now.","commit_id":"46f81298948865a2c15c4a5035e95a0c77adb5d5"},{"author":{"_account_id":6926,"name":"Bogdan Dobrelya","email":"bdobreli@redhat.com","username":"bogdando"},"change_message_id":"ed3192f485a48f403439ba6e0000a87c20b9d5ab","unresolved":false,"context_lines":[{"line_number":1183,"context_line":"        LOG.debug(\u0027Released lock on layer %s\u0027 % layer)"},{"line_number":1184,"context_line":""},{"line_number":1185,"context_line":"    @classmethod"},{"line_number":1186,"context_line":"    def _global_view_proxy(cls, value\u003dNone, forget\u003dFalse):"},{"line_number":1187,"context_line":"        if not cls.lock:"},{"line_number":1188,"context_line":"            LOG.warning(\u0027No lock information provided for value %s\u0027 % value)"},{"line_number":1189,"context_line":"            return"}],"source_content_type":"text/x-python","patch_set":65,"id":"3fa7e38b_a2e779ac","line":1186,"in_reply_to":"3fa7e38b_e281f14e","updated":"2019-11-15 09:05:43.000000000","message":"if you don\u0027t mind, I would do that via a follow-up?","commit_id":"46f81298948865a2c15c4a5035e95a0c77adb5d5"},{"author":{"_account_id":14985,"name":"Alex Schultz","email":"aschultz@next-development.com","username":"mwhahaha"},"change_message_id":"0e2be3e02380d86c068f2857e6b065db6f70b7f6","unresolved":false,"context_lines":[{"line_number":1814,"context_line":"    def _target_layer_exists_registry(cls, target_url, layer, check_layers,"},{"line_number":1815,"context_line":"                                      session):"},{"line_number":1816,"context_line":"        image, tag \u003d cls._image_tag_from_url(target_url)"},{"line_number":1817,"context_line":"        norm_image \u003d (image[1:] if image.startswith(\u0027/\u0027) else image)"},{"line_number":1818,"context_line":"        parts \u003d {"},{"line_number":1819,"context_line":"            \u0027image\u0027: image,"},{"line_number":1820,"context_line":"            \u0027tag\u0027: tag"}],"source_content_type":"text/x-python","patch_set":65,"id":"3fa7e38b_f9bfb45d","line":1817,"range":{"start_line":1817,"start_character":8,"end_line":1817,"end_character":68},"updated":"2019-11-08 15:13:36.000000000","message":"image \u003d image.lstrip(\u0027/\u0027) ?","commit_id":"46f81298948865a2c15c4a5035e95a0c77adb5d5"}]}