)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a67e4edc1d81e399e722082dbaa3c9bb5065dede","unresolved":false,"context_lines":[{"line_number":11,"context_line":"loss of data availability. Flag such configurations early and prevent"},{"line_number":12,"context_line":"the proxy-server from starting."},{"line_number":13,"context_line":""},{"line_number":14,"context_line":"Note that memcache gat added to a bunch of after_fn return values"},{"line_number":15,"context_line":"because we would mis-place listing_formats and dlo in the"},{"line_number":16,"context_line":"widely-used-in-tests pipeline:"},{"line_number":17,"context_line":""}],"source_content_type":"text/x-gerrit-commit-message","patch_set":1,"id":"9fdfeff1_7f9ac46e","line":14,"range":{"start_line":14,"start_character":19,"end_line":14,"end_character":22},"updated":"2019-02-11 23:41:30.000000000","message":"got","commit_id":"d22208e4cd345dec480a788c244277b84de0164e"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"68bbe16ce2d9bafbdebdc4f9084f886e670acc43","unresolved":false,"context_lines":[{"line_number":9,"context_line":"There are a variety of known-bad middleware configurations which can"},{"line_number":10,"context_line":"lead to non-functioning features, broken container listings, or even the"},{"line_number":11,"context_line":"loss of data availability. Flag such configurations early and prevent"},{"line_number":12,"context_line":"the proxy-server from starting."},{"line_number":13,"context_line":""},{"line_number":14,"context_line":"Note that memcache got added to a bunch of after_fn return values"},{"line_number":15,"context_line":"because we would mis-place listing_formats and dlo in the"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":5,"id":"7faddb67_e16a2f18","line":12,"updated":"2019-07-31 14:50:16.000000000","message":"I think preventing the proxy from starting is a somewhat hostile position for what is ostensibly a new feature to assist operators.\n\nI think something more like a \"config check\" tool/function would be a lot more useful and operator friendly.","commit_id":"65e846c9a7ea104a834879cf9eb669050f22c6ee"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"38627b0a5e25b36338ed686413100b933853016f","unresolved":true,"context_lines":[{"line_number":19,"context_line":""},{"line_number":20,"context_line":"Change-Id: Ib47ecc945b5a8960b5d74ce165f0d5f9a131ed56"},{"line_number":21,"context_line":"Partial-Bug: 1813725"},{"line_number":22,"context_line":"Closes-Bug: 1758710"},{"line_number":23,"context_line":"Closes-Bug: 1806468"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":5,"id":"29a2eb50_45b99682","line":23,"range":{"start_line":22,"start_character":0,"end_line":23,"end_character":19},"updated":"2021-07-15 20:21:47.000000000","message":"Another: https://bugs.launchpad.net/swift/+bug/1909213","commit_id":"65e846c9a7ea104a834879cf9eb669050f22c6ee"}],"swift/proxy/server.py":[{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"048d64d096146da76242a92e16f43de9265089a0","unresolved":false,"context_lines":[{"line_number":86,"context_line":"# If this is *not* true, an InvalidPipelineError will be raised, preventing"},{"line_number":87,"context_line":"# the proxy-server from starting up. Note that this is separate from the"},{"line_number":88,"context_line":"# required filters above -- if a filter is not present, it is simply ignored."},{"line_number":89,"context_line":"filter_soft_deps \u003d {"},{"line_number":90,"context_line":"    \u0027slo\u0027: ["},{"line_number":91,"context_line":"        \u0027encryption\u0027,  # ensure manifest is encrypted and seg etags are sane"},{"line_number":92,"context_line":"        \u0027symlink\u0027,  # so we can follow symlinks to get to segments"}],"source_content_type":"text/x-python","patch_set":1,"id":"9fdfeff1_e2bc98dd","line":89,"updated":"2019-02-11 22:54:50.000000000","message":"The dependency is called \"soft\" yet the comment says \"must\". This is incongruent.","commit_id":"d22208e4cd345dec480a788c244277b84de0164e"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"e398b795df7c060a87ccd8dea4d8e9defd0749be","unresolved":false,"context_lines":[{"line_number":86,"context_line":"# If this is *not* true, an InvalidPipelineError will be raised, preventing"},{"line_number":87,"context_line":"# the proxy-server from starting up. Note that this is separate from the"},{"line_number":88,"context_line":"# required filters above -- if a filter is not present, it is simply ignored."},{"line_number":89,"context_line":"filter_soft_deps \u003d {"},{"line_number":90,"context_line":"    \u0027slo\u0027: ["},{"line_number":91,"context_line":"        \u0027encryption\u0027,  # ensure manifest is encrypted and seg etags are sane"},{"line_number":92,"context_line":"        \u0027symlink\u0027,  # so we can follow symlinks to get to segments"}],"source_content_type":"text/x-python","patch_set":1,"id":"9fdfeff1_a59bfa14","line":89,"in_reply_to":"9fdfeff1_65d192fb","updated":"2019-02-12 00:09:22.000000000","message":"I see. Well, can they be something like opt_filter_deps, perhaps? Just cutting the gordean knot by omitting the softness.","commit_id":"d22208e4cd345dec480a788c244277b84de0164e"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"3a98c9782d74658fb9d65c78e415190efbc2a9f6","unresolved":false,"context_lines":[{"line_number":86,"context_line":"# If this is *not* true, an InvalidPipelineError will be raised, preventing"},{"line_number":87,"context_line":"# the proxy-server from starting up. Note that this is separate from the"},{"line_number":88,"context_line":"# required filters above -- if a filter is not present, it is simply ignored."},{"line_number":89,"context_line":"filter_soft_deps \u003d {"},{"line_number":90,"context_line":"    \u0027slo\u0027: ["},{"line_number":91,"context_line":"        \u0027encryption\u0027,  # ensure manifest is encrypted and seg etags are sane"},{"line_number":92,"context_line":"        \u0027symlink\u0027,  # so we can follow symlinks to get to segments"}],"source_content_type":"text/x-python","patch_set":1,"id":"5fc1f717_fcb44cd3","line":89,"in_reply_to":"9fdfeff1_a59bfa14","updated":"2019-03-23 06:25:35.000000000","message":"Done","commit_id":"d22208e4cd345dec480a788c244277b84de0164e"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"a67e4edc1d81e399e722082dbaa3c9bb5065dede","unresolved":false,"context_lines":[{"line_number":86,"context_line":"# If this is *not* true, an InvalidPipelineError will be raised, preventing"},{"line_number":87,"context_line":"# the proxy-server from starting up. Note that this is separate from the"},{"line_number":88,"context_line":"# required filters above -- if a filter is not present, it is simply ignored."},{"line_number":89,"context_line":"filter_soft_deps \u003d {"},{"line_number":90,"context_line":"    \u0027slo\u0027: ["},{"line_number":91,"context_line":"        \u0027encryption\u0027,  # ensure manifest is encrypted and seg etags are sane"},{"line_number":92,"context_line":"        \u0027symlink\u0027,  # so we can follow symlinks to get to segments"}],"source_content_type":"text/x-python","patch_set":1,"id":"9fdfeff1_65d192fb","line":89,"in_reply_to":"9fdfeff1_e2bc98dd","updated":"2019-02-11 23:41:30.000000000","message":"...immediately prefaced by \"if present\". Do you have a better name in mind?\n\nIn general, we don\u0027t have hard middleware dependencies -- the closest we get is\n\n- s3api, where deploying slo or encryption enables some features but you can still deploy without them, or\n- slo, which imports some bulk-deleter code but doesn\u0027t actually issue a request for the bulk middleware to process.\n\nThis is different; slo doesn\u0027t require symlinks -- each feature can be toggled independently. But if symlink is left of slo, we know we aren\u0027t going to be able to pass func tests, because we can\u0027t replace a segment with a symlink that points to the same data.\n\nSo what\u0027s a better name for \"if I am enabled *and* this thing is enabled, it needs to be down stream of me\"? Or, what\u0027s a better wording for documenting it?","commit_id":"d22208e4cd345dec480a788c244277b84de0164e"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"bf681a8d9d884c0a60d31dd4bb697486531a8955","unresolved":false,"context_lines":[{"line_number":768,"context_line":"                if pipe.index(mware) \u003e pipe.index(dep):"},{"line_number":769,"context_line":"                    problems.append(\u0027%s must be right of %s\u0027 % (dep, mware))"},{"line_number":770,"context_line":"        if problems:"},{"line_number":771,"context_line":"            raise InvalidPipelineError(problems)"},{"line_number":772,"context_line":""},{"line_number":773,"context_line":""},{"line_number":774,"context_line":"def parse_per_policy_config(conf):"}],"source_content_type":"text/x-python","patch_set":4,"id":"5fc1f717_40a98521","line":771,"updated":"2019-03-27 21:58:51.000000000","message":"this would be great if it was separate function\n\nthis would be better if there was a \"unsafe_skip_pipeline_validation\" option since it could easily cause a failure to start proxies after reboot\n\n... give effected users a chance to file a bug a cycle to get a fix or figure out a migration plan for their pipeline","commit_id":"cb1ef3d551673cba1a72300f6e72731fc86e9ad5"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"68bbe16ce2d9bafbdebdc4f9084f886e670acc43","unresolved":false,"context_lines":[{"line_number":768,"context_line":"                if pipe.index(mware) \u003e pipe.index(dep):"},{"line_number":769,"context_line":"                    problems.append(\u0027%s must be right of %s\u0027 % (dep, mware))"},{"line_number":770,"context_line":"        if problems:"},{"line_number":771,"context_line":"            raise InvalidPipelineError(problems)"},{"line_number":772,"context_line":""},{"line_number":773,"context_line":""},{"line_number":774,"context_line":"def parse_per_policy_config(conf):"}],"source_content_type":"text/x-python","patch_set":4,"id":"7faddb67_e471dd77","line":771,"in_reply_to":"3fce034c_66032b44","updated":"2019-07-31 14:50:16.000000000","message":"I think designing it to be consumed by tools that are generating configs (e.g. someone\u0027s chef/ansible or the SwiftStack Controller) is absolutely useful - perhaps even more useful than running it here inline when it\u0027s kind of too late.\n\nI can\u0027t say when/if YOU personally would want to allow bypassing this check - since you can always just write a patch - but any operator that was running with a misordered pipeline that we didn\u0027t previously disallow might appreciate an obvious way to keep doing what they were doing knowing they won\u0027t make it *worse* while they can meanwhile file a bug or get in #openstack-swift and try and validate if all the data in their cluster is still going to \"work\" after the recommended pipeline reordering, or if they need a lengthy migration before they can switch over.","commit_id":"cb1ef3d551673cba1a72300f6e72731fc86e9ad5"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"da0ea10ab4bab00c6f2b82a0507f3bc6660af3a9","unresolved":false,"context_lines":[{"line_number":768,"context_line":"                if pipe.index(mware) \u003e pipe.index(dep):"},{"line_number":769,"context_line":"                    problems.append(\u0027%s must be right of %s\u0027 % (dep, mware))"},{"line_number":770,"context_line":"        if problems:"},{"line_number":771,"context_line":"            raise InvalidPipelineError(problems)"},{"line_number":772,"context_line":""},{"line_number":773,"context_line":""},{"line_number":774,"context_line":"def parse_per_policy_config(conf):"}],"source_content_type":"text/x-python","patch_set":4,"id":"3fce034c_66032b44","line":771,"in_reply_to":"5fc1f717_40a98521","updated":"2019-04-12 21:00:48.000000000","message":"IDK how useful this would be as a separate function... we\u0027re checking a PipelineWrapper, which seems like a bit of a heavy lift for most callers...\n\nMaybe it would mostly still work with a list of names? Provided, of course, that whoever calls it uses the same entry point names...\n\nAs far as a config option to override... I kinda wonder if we should expose allow_modify_pipeline in proxy-server.conf... I kinda remember Sam wanting to be able to say, \"No, seriously, I *don\u0027t want* a gatekeeper!\"\n\nIDK how much I really want to enable bypassing this though -- I\u0027ve tried to be pretty conservative in terms of making sure that these are known-bad configs with clear repercussions; it\u0027s part of why I wanted to establish a precedent of including comments per-check explaining why we feel like it\u0027s a thing worth preventing the proxy from coming up.\n\n*Maybe* I shouldn\u0027t have included the slo -\u003e symlink, dlo -\u003e symlink, and staticweb -\u003e symlink checks? All the others, Swift is likely to 500 a very broad class of requests, with the sorts of tracebacks that just lead us down rabbit holes.","commit_id":"cb1ef3d551673cba1a72300f6e72731fc86e9ad5"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"02d9287f5c3c4e8d942259687ae9d22a306978c0","unresolved":true,"context_lines":[{"line_number":768,"context_line":"                if pipe.index(mware) \u003e pipe.index(dep):"},{"line_number":769,"context_line":"                    problems.append(\u0027%s must be right of %s\u0027 % (dep, mware))"},{"line_number":770,"context_line":"        if problems:"},{"line_number":771,"context_line":"            raise InvalidPipelineError(problems)"},{"line_number":772,"context_line":""},{"line_number":773,"context_line":""},{"line_number":774,"context_line":"def parse_per_policy_config(conf):"}],"source_content_type":"text/x-python","patch_set":4,"id":"17f33692_47595550","line":771,"in_reply_to":"7faddb67_e02782c6","updated":"2025-06-10 15:54:24.000000000","message":"\u003e This is starting to sound an awful lot like swift-proxy-server but one step short of actually listening on a socket.\n\nOh, hey: https://github.com/openstack/swift/commit/0a5f0253\n\nI suppose a lot of the configured credentials and externally-stored secrets are probably lazily loaded/used, though -- might be nice to add a `--validate-credentials` option or something that sets some flag in the global conf to tell middlewares to do a more-thorough check...","commit_id":"cb1ef3d551673cba1a72300f6e72731fc86e9ad5"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"38627b0a5e25b36338ed686413100b933853016f","unresolved":true,"context_lines":[{"line_number":768,"context_line":"                if pipe.index(mware) \u003e pipe.index(dep):"},{"line_number":769,"context_line":"                    problems.append(\u0027%s must be right of %s\u0027 % (dep, mware))"},{"line_number":770,"context_line":"        if problems:"},{"line_number":771,"context_line":"            raise InvalidPipelineError(problems)"},{"line_number":772,"context_line":""},{"line_number":773,"context_line":""},{"line_number":774,"context_line":"def parse_per_policy_config(conf):"}],"source_content_type":"text/x-python","patch_set":4,"id":"7faddb67_e02782c6","line":771,"in_reply_to":"7faddb67_e471dd77","updated":"2021-07-15 20:21:47.000000000","message":"All right, a *tool* that takes a config file, loads it up, finds pipeline:main, gets a PipelineWrapper with all the goodness needed to actually run this -- I suppose I could see that.\n\nNext logical step would be to also validate that the config values are sane and won\u0027t make the proxy barf. Maybe even make sure that any keystone/pykmip creds are valid... This is starting to sound an awful lot like swift-proxy-server but one step short of actually listening on a socket.\n\n\u003e keep doing what they were doing knowing they won\u0027t make it *worse*\n\nIDK -- I think it\u0027s pretty clear that things can and will get worse the longer you run with a pipeline that trips https://bugs.launchpad.net/swift/+bug/1813725\n\nI realize that I\u0027ve taken a bit of a hostile stance -- but it\u0027s also not something that I\u0027ve done lightly. There are two main themes to the deps I specified:\n\n- listing_formats needs to be far left. The alternative is *you cannot do container listings*.\n- encryption needs to be far right. The alternative is *your data is not actually encrypted* (for formpost or copy), *your data is corrupted* (for dlo), or *large features of swift do not work* (for slo, s3api, bulk, maybe others).\n\n\u003e if they need a lengthy migration before they can switch over\n\nThat\u0027s the bit that made me think of this in the context of hardlinks -- without *some* way to ensure that symlink really is right of slo and dlo, are we setting users up to need to do some sort of stop-the-world data migration to get back on the happy path? Or are we going to get lucky and hardlink-to-large-object just *won\u0027t work*?","commit_id":"cb1ef3d551673cba1a72300f6e72731fc86e9ad5"},{"author":{"_account_id":9625,"name":"Thiago da Silva","email":"thiagodasilva@gmail.com","username":"thiago"},"change_message_id":"e880379688e192dbf178e5f55a0881bf5c6b1ff1","unresolved":false,"context_lines":[{"line_number":97,"context_line":"    \u0027dlo\u0027: ["},{"line_number":98,"context_line":"        \u0027symlink\u0027,  # so we can follow symlinks to get to segments"},{"line_number":99,"context_line":"    ],"},{"line_number":100,"context_line":"    \u0027versioned_writes\u0027: ["},{"line_number":101,"context_line":"        \u0027encryption\u0027,  # so new versions are encrypted with the right path"},{"line_number":102,"context_line":"    ],"},{"line_number":103,"context_line":"    \u0027copy\u0027: ["}],"source_content_type":"text/x-python","patch_set":5,"id":"7faddb67_60ee64b5","line":100,"range":{"start_line":100,"start_character":5,"end_line":100,"end_character":21},"updated":"2019-07-29 12:45:09.000000000","message":"should probably add symlink here, right?","commit_id":"65e846c9a7ea104a834879cf9eb669050f22c6ee"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"018fea6383aac8d59beb16173a3930459e13fda6","unresolved":false,"context_lines":[{"line_number":97,"context_line":"    \u0027dlo\u0027: ["},{"line_number":98,"context_line":"        \u0027symlink\u0027,  # so we can follow symlinks to get to segments"},{"line_number":99,"context_line":"    ],"},{"line_number":100,"context_line":"    \u0027versioned_writes\u0027: ["},{"line_number":101,"context_line":"        \u0027encryption\u0027,  # so new versions are encrypted with the right path"},{"line_number":102,"context_line":"    ],"},{"line_number":103,"context_line":"    \u0027copy\u0027: ["}],"source_content_type":"text/x-python","patch_set":5,"id":"7faddb67_cfce63b3","line":100,"range":{"start_line":100,"start_character":5,"end_line":100,"end_character":21},"in_reply_to":"7faddb67_60ee64b5","updated":"2019-07-29 16:20:04.000000000","message":"I\u0027m not actually sure that it would caues a problem currently. Once we have https://review.opendev.org/#/c/633857/ absolutely this should be updated. But until then, func tests pass (at least on my machine!) with *either* versioned_writes-then-symlink OR symlink-then-versioned_writes.","commit_id":"65e846c9a7ea104a834879cf9eb669050f22c6ee"}],"test/unit/common/test_wsgi.py":[{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"09df4f31ea476a5324a2d0e96c8853eccafa5eab","unresolved":false,"context_lines":[{"line_number":2331,"context_line":"            self.assertEqual(expected_path, object_ring.serialized_path)"},{"line_number":2332,"context_line":""},{"line_number":2333,"context_line":"    @with_tempdir"},{"line_number":2334,"context_line":"    def test_pipeline_validation(self, tempdir):"},{"line_number":2335,"context_line":"        conf_path \u003d os.path.join(tempdir, \u0027proxy-server.conf\u0027)"},{"line_number":2336,"context_line":"        conf_body \u003d \"\"\""},{"line_number":2337,"context_line":"        [DEFAULT]"}],"source_content_type":"text/x-python","patch_set":1,"id":"9fdfeff1_e4da4976","line":2334,"updated":"2019-02-13 02:29:05.000000000","message":"Seeing as we have defined a bunch of these \"soft\" (sorry Pete :P) filter deps, should we have a test for these too. Though I guess below is enough to test the validaton code itself. \n\nMaybe a middle ground, throw a test for the bugs we know we have.. I.e one that covers bug 1813725 more explicitly and similar for 1758710 and 1806468.","commit_id":"d22208e4cd345dec480a788c244277b84de0164e"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"3a98c9782d74658fb9d65c78e415190efbc2a9f6","unresolved":false,"context_lines":[{"line_number":2331,"context_line":"            self.assertEqual(expected_path, object_ring.serialized_path)"},{"line_number":2332,"context_line":""},{"line_number":2333,"context_line":"    @with_tempdir"},{"line_number":2334,"context_line":"    def test_pipeline_validation(self, tempdir):"},{"line_number":2335,"context_line":"        conf_path \u003d os.path.join(tempdir, \u0027proxy-server.conf\u0027)"},{"line_number":2336,"context_line":"        conf_body \u003d \"\"\""},{"line_number":2337,"context_line":"        [DEFAULT]"}],"source_content_type":"text/x-python","patch_set":1,"id":"5fc1f717_fc10ccaa","line":2334,"in_reply_to":"9fdfeff1_e4da4976","updated":"2019-03-23 06:25:35.000000000","message":"This one was already kind of demonstrating 1758710, but I added a couple more. This also made me realize that I should collect all the problematic placements before raising; turns out, a single pipeline can have a *bunch* of problems!","commit_id":"d22208e4cd345dec480a788c244277b84de0164e"}]}