)]}'
{"id":"openstack%2Fswift~899397","triplet_id":"openstack%2Fswift~master~I6c3765d43e0521dc0648ccd0956b93306d31e062","project":"openstack/swift","branch":"master","topic":"otel_tracing_decorator","attention_set":{"7233":{"account":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"last_update":"2026-05-14 00:07:50.000000000","reason":"A robot voted negatively on a label"}},"removed_from_attention_set":{},"hashtags":[],"change_id":"I6c3765d43e0521dc0648ccd0956b93306d31e062","subject":"request tracing: Add Otel request tracing support (decorator)","status":"NEW","created":"2023-10-26 13:06:32.000000000","updated":"2026-06-03 11:50:36.000000000","submit_type":"MERGE_IF_NECESSARY","mergeable":false,"submittable":false,"total_comment_count":1,"unresolved_comment_count":0,"has_review_started":true,"meta_rev_id":"c2a280d71d7a80cf00b31a8d1f5f03ae8bc0f7d2","_number":899397,"virtual_id_number":899397,"owner":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"actions":{},"labels":{"Verified":{"disliked":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"all":[{"tag":"autogenerated:zuul:check","value":-1,"date":"2026-06-03 07:35:41.000000000","permitted_voting_range":{"min":-2,"max":2},"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]}],"values":{"-2":"Fails","-1":"Doesn\u0027t seem to work"," 0":"No score","+1":"Works for me","+2":"Verified"},"description":"","value":-1,"default_value":0,"optional":true},"Code-Review":{"all":[{"value":0,"permitted_voting_range":{"min":-1,"max":1},"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]}],"values":{"-2":"Do not merge","-1":"This patch needs further work before it can be merged"," 0":"No score","+1":"Looks good to me, but someone else must approve","+2":"Looks good to me (core reviewer)"},"description":"","default_value":0,"optional":true},"Workflow":{"all":[{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]}],"values":{"-1":"Work in progress"," 0":"Ready for reviews","+1":"Approved"},"description":"","default_value":0,"optional":true}},"removable_reviewers":[],"reviewers":{"REVIEWER":[{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]}]},"pending_reviewers":{},"reviewer_updates":[{"updated":"2023-10-26 13:33:07.000000000","updated_by":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"reviewer":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"state":"CC"},{"updated":"2023-10-26 14:33:41.000000000","updated_by":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"reviewer":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"state":"REVIEWER"}],"messages":[{"id":"52ac1ed0437025d81b0e49d0db312c3c9ec338b6","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"date":"2023-10-26 13:06:32.000000000","message":"Uploaded patch set 1.","accounts_in_message":[],"_revision_number":1},{"id":"dbc895d8be13797215eb5ec1f0b725ed6315cf07","tag":"autogenerated:zuul:check-arm64","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2023-10-26 13:33:07.000000000","message":"Patch Set 1:\n\nBuild failed (ARM64 pipeline).\nhttps://zuul.opendev.org/t/openstack/buildset/ef42322150b84c02a56db9a0ccf2d024\n\n- swift-tox-py38-arm64 https://zuul.opendev.org/t/openstack/build/71aa33254ba64612afd90b74bb741c15 : FAILURE in 22m 18s\n- swift-tox-py39-arm64 https://zuul.opendev.org/t/openstack/build/4fb8769c575248d9b8a7bfeea79e4345 : FAILURE in 19m 43s\n- swift-tox-func-encryption-py38-arm64 https://zuul.opendev.org/t/openstack/build/f0d0060bbb284143a5a2d330d9c5068f : SUCCESS in 24m 00s\n- swift-tox-func-py38-arm64 https://zuul.opendev.org/t/openstack/build/b6f5e886f112462f88773d3c3e687796 : SUCCESS in 24m 58s","accounts_in_message":[],"_revision_number":1},{"id":"54cc8ad9eb127861b918fbef41ad89d1fc888b42","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2023-10-26 14:33:41.000000000","message":"Patch Set 1: Verified-1\n\n(11 comments)\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttps://docs.opendev.org/opendev/infra-manual/latest/developers.html#automated-testing\n\nhttps://zuul.opendev.org/t/openstack/buildset/2b313b5e68fe4f42b6946109ceaf90af\n\n- test-release-openstack https://zuul.opendev.org/t/openstack/build/8781bff351244a23af18770d869f14c7 : SUCCESS in 3m 41s\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/5eb9a7b7b9a94d8fae6344b35716b030 : SUCCESS in 9m 40s\n- requirements-check https://zuul.opendev.org/t/openstack/build/1f30c90df3d4475480b27e3b5bcc641c : FAILURE in 2m 40s\n- grenade https://zuul.opendev.org/t/openstack/build/8f6fb16772b7490eb2665e96e0cba8c6 : SUCCESS in 1h 13m 04s\n- grenade-skip-level https://zuul.opendev.org/t/openstack/build/5be014eb95b04a1da661684bea381f78 : SUCCESS in 51m 38s\n- tempest-integrated-object-storage https://zuul.opendev.org/t/openstack/build/46b035475bbd4f2d8cb08282b3c7950a : SUCCESS in 42m 48s\n- openstacksdk-functional-devstack https://zuul.opendev.org/t/openstack/build/c132d2c2f5824e54b7365f9eb265947e : SUCCESS in 1h 07m 21s\n- swift-tox-func-py39-centos-9-stream-fips https://zuul.opendev.org/t/openstack/build/886cc6025d10413383341b812f680f0b : SUCCESS in 20m 26s (non-voting)\n- swift-tox-func-encryption-py39-centos-9-stream-fips https://zuul.opendev.org/t/openstack/build/bd993c24c42645f7a40a8934e91c81fd : SUCCESS in 22m 45s (non-voting)\n- swift-tox-func-ec-py39-centos-9-stream-fips https://zuul.opendev.org/t/openstack/build/3f9ac2f381924ae5bd76aad1623513e6 : SUCCESS in 21m 16s (non-voting)\n- swift-build-image https://zuul.opendev.org/t/openstack/build/18b3a35bf3fa4e13b85a9c31f0cab3c8 : SUCCESS in 6m 21s (non-voting)\n- swift-build-image-py3 https://zuul.opendev.org/t/openstack/build/689fb6cc60c348f2a602aa34b4287f8c : SUCCESS in 6m 36s (non-voting)\n- swift-tox-py27 https://zuul.opendev.org/t/openstack/build/e800467207d04bdb85d8b2c1790bcc41 : FAILURE in 4m 14s\n- swift-tox-py36 https://zuul.opendev.org/t/openstack/build/c29de4c929da4d8488c2b8dcd8c56713 : FAILURE in 24m 57s\n- swift-tox-py39 https://zuul.opendev.org/t/openstack/build/5364b34fd1944492a094c166f4b61cbf : FAILURE in 15m 25s\n- swift-tox-py310 https://zuul.opendev.org/t/openstack/build/61569547a0874a56b049a1b8decd56ab : FAILURE in 16m 57s\n- swift-tox-py311 https://zuul.opendev.org/t/openstack/build/71a6e722063e450694ccb8e1b00e883d : FAILURE in 26m 27s\n- swift-tox-func-py27 https://zuul.opendev.org/t/openstack/build/c253df54539f4863a8e55efd81e23f8e : FAILURE in 3m 13s\n- swift-tox-func-encryption-py27 https://zuul.opendev.org/t/openstack/build/cc7af82a2c354518a5cad7ba0aec2ef3 : FAILURE in 3m 08s\n- swift-tox-func-ec-py27 https://zuul.opendev.org/t/openstack/build/b9d638333e47430a9b2565840562ba73 : FAILURE in 4m 08s\n- swift-tox-func-py38 https://zuul.opendev.org/t/openstack/build/4423b48ab99144edb584edffd4796738 : SUCCESS in 25m 06s\n- swift-tox-func-encryption-py38 https://zuul.opendev.org/t/openstack/build/05ccb4952e0b4737a17a6783bd8e80fa : SUCCESS in 17m 20s\n- swift-tox-func-ec-py38 https://zuul.opendev.org/t/openstack/build/50912a6142844c3790b5a9f8b7033334 : SUCCESS in 17m 00s\n- swift-func-cors https://zuul.opendev.org/t/openstack/build/ec66fd7221b74d32a39ae326d9682b5c : SUCCESS in 9m 26s\n- swift-tox-func-s3api-ceph-s3tests-tempauth https://zuul.opendev.org/t/openstack/build/04a8fdc85af04c3a8809220182095a4c : SUCCESS in 18m 46s (non-voting)\n- swift-tox-func-s3api-tests-tempauth https://zuul.opendev.org/t/openstack/build/af2bac1cebea4cbbb4b55f782d1d80e0 : SUCCESS in 8m 57s\n- swift-probetests-centos-7 https://zuul.opendev.org/t/openstack/build/7e552aee211c4faf9d2f8749d4bfefba : RETRY_LIMIT in 6m 01s\n- swift-probetests-centos-8-stream https://zuul.opendev.org/t/openstack/build/71600c195b804b9492192bbfd0535e54 : SUCCESS in 1h 15m 32s\n- swift-dsvm-functional https://zuul.opendev.org/t/openstack/build/d78bad4d90504756b9b7215b82b55d1f : SUCCESS in 55m 35s\n- swift-dsvm-functional-ipv6 https://zuul.opendev.org/t/openstack/build/26beebf80b954914857973941ad9582e : SUCCESS in 54m 34s\n- swift-tox-lower-constraints https://zuul.opendev.org/t/openstack/build/747a68b73d6e4f8d973f25b15d32def6 : FAILURE in 3m 25s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/3f2722e2ea3a4fe8afc0ada2ff0988c7 : FAILURE in 5m 49s\n- swift-multinode-rolling-upgrade https://zuul.opendev.org/t/openstack/build/b95412a38f4443b58afa32e42b92c29c : FAILURE in 10m 48s (non-voting)\n- tempest-ipv6-only https://zuul.opendev.org/t/openstack/build/371f07488b0a4e5ca9d7f85533a784a7 : SUCCESS in 43m 53s","accounts_in_message":[],"_revision_number":1},{"id":"8c1f5161324967fb710f88be1698aa7b9817b824","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"date":"2023-10-26 22:48:23.000000000","message":"Uploaded patch set 2.\n\nOutdated Votes:\n* Verified-1\n","accounts_in_message":[],"_revision_number":2},{"id":"616335e675e735547f680d4cddaf1757f3cc696e","tag":"autogenerated:zuul:check-arm64","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2023-10-26 23:44:32.000000000","message":"Patch Set 2:\n\nBuild failed (ARM64 pipeline).\nhttps://zuul.opendev.org/t/openstack/buildset/07e6b3c9b376436abaee89330d5ff73f\n\n- swift-tox-py38-arm64 https://zuul.opendev.org/t/openstack/build/5602087be62e4bec94510944e959570a : FAILURE in 16m 17s\n- swift-tox-py39-arm64 https://zuul.opendev.org/t/openstack/build/4b82938239694bc4a3a611804876fb04 : FAILURE in 18m 24s\n- swift-tox-func-encryption-py38-arm64 https://zuul.opendev.org/t/openstack/build/5e2caf85017847e8a8d4edd047f8429b : SUCCESS in 25m 21s\n- swift-tox-func-py38-arm64 https://zuul.opendev.org/t/openstack/build/9de7dff110864327b07524fb73c95206 : SUCCESS in 53m 37s","accounts_in_message":[],"_revision_number":2},{"id":"92e9cb9e02304f28f9fe18189c960157fe4b1038","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2023-10-26 23:57:28.000000000","message":"Patch Set 2: Verified-1\n\n(5 comments)\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttps://docs.opendev.org/opendev/infra-manual/latest/developers.html#automated-testing\n\nhttps://zuul.opendev.org/t/openstack/buildset/6a0096f176ed4de7bdd2fcdc903f27ab\n\n- test-release-openstack https://zuul.opendev.org/t/openstack/build/8e9fbb9e93564470bab4cc9947356ca7 : SUCCESS in 2m 03s\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/0e812e6e05b141d7b7ae74b24abec8c1 : SUCCESS in 8m 55s\n- requirements-check https://zuul.opendev.org/t/openstack/build/365551fbf9ad415a84aa6b1927ebbbe9 : FAILURE in 2m 57s\n- grenade https://zuul.opendev.org/t/openstack/build/9b4a64d25dc247e492e8690cb57f427e : SUCCESS in 42m 08s\n- grenade-skip-level https://zuul.opendev.org/t/openstack/build/405362be373e4c378c951d5535828a5e : SUCCESS in 38m 58s\n- tempest-integrated-object-storage https://zuul.opendev.org/t/openstack/build/42a8d2f6ca284b5caf2e0b4ab6076b00 : SUCCESS in 58m 30s\n- openstacksdk-functional-devstack https://zuul.opendev.org/t/openstack/build/7ceacb72a55f412eb2d98ba4526b5a52 : SUCCESS in 1h 08m 13s\n- swift-tox-func-py39-centos-9-stream-fips https://zuul.opendev.org/t/openstack/build/590397a2fa5f4e028439458faa9879bd : SUCCESS in 22m 56s (non-voting)\n- swift-tox-func-encryption-py39-centos-9-stream-fips https://zuul.opendev.org/t/openstack/build/1a1727d7c45a4525b4bee11c59f3618c : SUCCESS in 21m 26s (non-voting)\n- swift-tox-func-ec-py39-centos-9-stream-fips https://zuul.opendev.org/t/openstack/build/af6021f762564ba0b2dc21e864821810 : SUCCESS in 21m 34s (non-voting)\n- swift-build-image https://zuul.opendev.org/t/openstack/build/ada5edc11e9e4100a8454024143b0309 : SUCCESS in 5m 53s (non-voting)\n- swift-build-image-py3 https://zuul.opendev.org/t/openstack/build/774e81cd66b54e18bd971cee026ae020 : SUCCESS in 6m 58s (non-voting)\n- swift-tox-py27 https://zuul.opendev.org/t/openstack/build/bad8c706eabb47008d46315fd54495bf : FAILURE in 3m 02s\n- swift-tox-py36 https://zuul.opendev.org/t/openstack/build/194c5427178d47a89fb70ed4efe66c36 : FAILURE in 13m 59s\n- swift-tox-py39 https://zuul.opendev.org/t/openstack/build/19c1adf911b048b2a54332039e86a958 : FAILURE in 16m 46s\n- swift-tox-py310 https://zuul.opendev.org/t/openstack/build/63bdb7f33be84fbd9bb217eebebb85a2 : FAILURE in 18m 33s\n- swift-tox-py311 https://zuul.opendev.org/t/openstack/build/8cb2c0f3a9124e94b16bf2be2a93ed6a : FAILURE in 15m 03s\n- swift-tox-func-py27 https://zuul.opendev.org/t/openstack/build/06e3057fb9764b29bed68ae3f335f72f : FAILURE in 2m 59s\n- swift-tox-func-encryption-py27 https://zuul.opendev.org/t/openstack/build/c0e4919f49f64a948077328f3fa07c79 : FAILURE in 3m 40s\n- swift-tox-func-ec-py27 https://zuul.opendev.org/t/openstack/build/68f5842db76044f6befd3f8291da1fff : FAILURE in 3m 11s\n- swift-tox-func-py38 https://zuul.opendev.org/t/openstack/build/c5b2b6c1529b444ea3d9d32130d4da47 : SUCCESS in 19m 19s\n- swift-tox-func-encryption-py38 https://zuul.opendev.org/t/openstack/build/daae2acb31ca4b999cb07e06645b78e4 : SUCCESS in 18m 40s\n- swift-tox-func-ec-py38 https://zuul.opendev.org/t/openstack/build/e25da9f0a1f44e438e489cd5425f8030 : SUCCESS in 21m 37s\n- swift-func-cors https://zuul.opendev.org/t/openstack/build/e4fdc4134ad44e85a01ae4b9d06398f7 : SUCCESS in 6m 40s\n- swift-tox-func-s3api-ceph-s3tests-tempauth https://zuul.opendev.org/t/openstack/build/0ce2e712aa1f4b3c90d617d5957fe8f4 : SUCCESS in 17m 35s (non-voting)\n- swift-tox-func-s3api-tests-tempauth https://zuul.opendev.org/t/openstack/build/0ceb8f78329849e3b14cda73644fd9f4 : SUCCESS in 9m 02s\n- swift-probetests-centos-7 https://zuul.opendev.org/t/openstack/build/3fb44c5ade3e4933ba447a3d764d10dc : RETRY_LIMIT in 3m 42s\n- swift-probetests-centos-8-stream https://zuul.opendev.org/t/openstack/build/02ffdc735ef64855ba6f8412b5fa5119 : SUCCESS in 58m 54s\n- swift-dsvm-functional https://zuul.opendev.org/t/openstack/build/107ebf8ac1e24fcb830fc85c29c8c7fc : SUCCESS in 52m 01s\n- swift-dsvm-functional-ipv6 https://zuul.opendev.org/t/openstack/build/8be0ceafb18d4e4baafe0958c25ae274 : SUCCESS in 39m 26s\n- swift-tox-lower-constraints https://zuul.opendev.org/t/openstack/build/664df9bb59e34c2bb2f3d0960ed71baf : FAILURE in 3m 42s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/cec6d101dc034dee86d6560ef0aafad3 : FAILURE in 5m 19s\n- swift-multinode-rolling-upgrade https://zuul.opendev.org/t/openstack/build/4cbf464c3fec4f1e90ae9854dda7d906 : FAILURE in 8m 04s (non-voting)\n- tempest-ipv6-only https://zuul.opendev.org/t/openstack/build/47ddf4d15be14b0789a70af5b7264105 : SUCCESS in 44m 11s","accounts_in_message":[],"_revision_number":2},{"id":"6fcfc8ed8ee15764d5119fc82fc4708c231712cf","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"date":"2023-10-27 01:09:33.000000000","message":"Uploaded patch set 3.\n\nOutdated Votes:\n* Verified-1\n","accounts_in_message":[],"_revision_number":3},{"id":"c69b68e7c9ab2c11d684c45c9d8641fc00f91b56","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2023-10-27 02:36:55.000000000","message":"Patch Set 3: Verified-1\n\n(5 comments)\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttps://docs.opendev.org/opendev/infra-manual/latest/developers.html#automated-testing\n\nhttps://zuul.opendev.org/t/openstack/buildset/52e512489f044e33add3d8172769fca1\n\n- test-release-openstack https://zuul.opendev.org/t/openstack/build/2e41d10e0beb4726807662ad69b165d3 : SUCCESS in 2m 03s\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/568a860313e84268bea678eef93f7f7c : SUCCESS in 9m 05s\n- requirements-check https://zuul.opendev.org/t/openstack/build/89582e672c574780afde7bc2614bb1e0 : FAILURE in 3m 19s\n- grenade https://zuul.opendev.org/t/openstack/build/90fafe55ea2e4f438a8f4bc5dd662312 : SUCCESS in 1h 00m 07s\n- grenade-skip-level https://zuul.opendev.org/t/openstack/build/988f470d487248c1baf184b9c5ea14fa : SUCCESS in 37m 14s\n- tempest-integrated-object-storage https://zuul.opendev.org/t/openstack/build/e7f25caa98344573bed742b1d0fbff70 : SUCCESS in 49m 17s\n- openstacksdk-functional-devstack https://zuul.opendev.org/t/openstack/build/d664edf530d8411b836ee2d0563f48a0 : SUCCESS in 45m 27s\n- swift-tox-func-py39-centos-9-stream-fips https://zuul.opendev.org/t/openstack/build/3a952bda6d9e4f95b5da8c493cf77a7d : SUCCESS in 15m 21s (non-voting)\n- swift-tox-func-encryption-py39-centos-9-stream-fips https://zuul.opendev.org/t/openstack/build/0fbe49b18bd64d2ab6450915feff4554 : SUCCESS in 17m 41s (non-voting)\n- swift-tox-func-ec-py39-centos-9-stream-fips https://zuul.opendev.org/t/openstack/build/6a91ab9e93604c119ed9ab3424459d56 : SUCCESS in 19m 45s (non-voting)\n- swift-build-image https://zuul.opendev.org/t/openstack/build/91b1ada54ce54d3bae119bb0f4651cf5 : SUCCESS in 5m 57s (non-voting)\n- swift-build-image-py3 https://zuul.opendev.org/t/openstack/build/179e6159e89e4357b259312264a6aa31 : SUCCESS in 6m 47s (non-voting)\n- swift-tox-py27 https://zuul.opendev.org/t/openstack/build/b778870184a447c391c1dfe16702c38b : FAILURE in 5m 40s\n- swift-tox-py36 https://zuul.opendev.org/t/openstack/build/33763d55742a4e3286c896af1dcd4804 : SUCCESS in 23m 44s\n- swift-tox-py39 https://zuul.opendev.org/t/openstack/build/d92da97829924d9894922172e3a23dcf : SUCCESS in 15m 36s\n- swift-tox-py310 https://zuul.opendev.org/t/openstack/build/4ead83aee70e4d2ca861bb71a2b41c04 : SUCCESS in 20m 50s\n- swift-tox-py311 https://zuul.opendev.org/t/openstack/build/80c8cf2f4ba34edc9f2ca0d80f10411a : SUCCESS in 17m 41s\n- swift-tox-func-py27 https://zuul.opendev.org/t/openstack/build/aeb35f621e09437c8ad14bccb92cf418 : FAILURE in 5m 39s\n- swift-tox-func-encryption-py27 https://zuul.opendev.org/t/openstack/build/eaaa1bfc93334f5caa26c0ffafbf5d6b : FAILURE in 5m 43s\n- swift-tox-func-ec-py27 https://zuul.opendev.org/t/openstack/build/54b13f95e68341f0bb8498de3481dacf : FAILURE in 3m 03s\n- swift-tox-func-py38 https://zuul.opendev.org/t/openstack/build/8a65b7ecd53a4fc5b743bd32a75fd92a : SUCCESS in 20m 25s\n- swift-tox-func-encryption-py38 https://zuul.opendev.org/t/openstack/build/ccb1cf9fd43447a88c02b829196585a2 : SUCCESS in 20m 57s\n- swift-tox-func-ec-py38 https://zuul.opendev.org/t/openstack/build/f8c1d1cc53af4298a0c6f34801ebfbe0 : SUCCESS in 18m 00s\n- swift-func-cors https://zuul.opendev.org/t/openstack/build/c7108e0b857f453384c34570b59eebc5 : SUCCESS in 7m 16s\n- swift-tox-func-s3api-ceph-s3tests-tempauth https://zuul.opendev.org/t/openstack/build/160f50757b534c1798a3b8564d543f58 : SUCCESS in 17m 34s (non-voting)\n- swift-tox-func-s3api-tests-tempauth https://zuul.opendev.org/t/openstack/build/8f00b90fc7b54d5b8407e5a57d2c8985 : SUCCESS in 10m 02s\n- swift-probetests-centos-7 https://zuul.opendev.org/t/openstack/build/a25dd6b83c014fcd999f9e9129c87e1d : RETRY_LIMIT in 3m 43s\n- swift-probetests-centos-8-stream https://zuul.opendev.org/t/openstack/build/48306d60434540d380fd3798af5dd305 : SUCCESS in 1h 21m 44s\n- swift-dsvm-functional https://zuul.opendev.org/t/openstack/build/e06dec5efe3b4aecbb6ee74fd5d5dfcc : SUCCESS in 1h 07m 44s\n- swift-dsvm-functional-ipv6 https://zuul.opendev.org/t/openstack/build/c59c47ec186c4ed98b9751bbce80a0ae : SUCCESS in 1h 02m 45s\n- swift-tox-lower-constraints https://zuul.opendev.org/t/openstack/build/c8422becc1fe405cbc126dbb9624d6bb : FAILURE in 3m 23s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/dcbd9d83d8a748c3a8bd2d13611b7492 : FAILURE in 4m 58s\n- swift-multinode-rolling-upgrade https://zuul.opendev.org/t/openstack/build/5bcda00a4fa24922ab2dfbe98fd3dc03 : FAILURE in 8m 36s (non-voting)\n- tempest-ipv6-only https://zuul.opendev.org/t/openstack/build/26859ea6e2124e45b5466da7442f3a1d : SUCCESS in 1h 04m 09s","accounts_in_message":[],"_revision_number":3},{"id":"219224258d88fa86792e9b916eee1179140ccc75","tag":"autogenerated:zuul:check-arm64","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2023-10-27 02:43:24.000000000","message":"Patch Set 3:\n\nBuild failed (ARM64 pipeline).\nhttps://zuul.opendev.org/t/openstack/buildset/6b68dde17b1b42fbb91196c142ab67eb\n\n- swift-tox-py38-arm64 https://zuul.opendev.org/t/openstack/build/01115a44ad4e49e99c970e5c8234af26 : SUCCESS in 15m 39s\n- swift-tox-py39-arm64 https://zuul.opendev.org/t/openstack/build/0abe3df3016c4554900396fbcd0d05f3 : SUCCESS in 16m 30s\n- swift-tox-func-encryption-py38-arm64 https://zuul.opendev.org/t/openstack/build/37670aa745274b3fa2e8e3b640de942d : TIMED_OUT in 1h 31m 13s\n- swift-tox-func-py38-arm64 https://zuul.opendev.org/t/openstack/build/60b41f786a454882931db988082542f5 : SUCCESS in 19m 32s","accounts_in_message":[],"_revision_number":3},{"id":"2b9319ed678325e7c204c504b467fb08b1a05e7e","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"date":"2023-10-27 11:15:58.000000000","message":"Uploaded patch set 4.\n\nOutdated Votes:\n* Verified-1\n","accounts_in_message":[],"_revision_number":4},{"id":"92f2fea4e7d983d7e9d623a05b54b595c8457f6f","tag":"autogenerated:zuul:check-arm64","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2023-10-27 11:57:54.000000000","message":"Patch Set 4:\n\nBuild succeeded (ARM64 pipeline).\nhttps://zuul.opendev.org/t/openstack/buildset/82e0f4a64c654e2a9df4ee38dbb7b4f0\n\n- swift-tox-py38-arm64 https://zuul.opendev.org/t/openstack/build/f089a51b6c984036a15d6effa0639210 : SUCCESS in 24m 10s\n- swift-tox-py39-arm64 https://zuul.opendev.org/t/openstack/build/939a746d827e48278766395eac3af58b : SUCCESS in 24m 51s\n- swift-tox-func-encryption-py38-arm64 https://zuul.opendev.org/t/openstack/build/75a52a8e6730452aac5514792d6cc91b : SUCCESS in 40m 18s\n- swift-tox-func-py38-arm64 https://zuul.opendev.org/t/openstack/build/7e2015d64d85496db592534674d581c6 : SUCCESS in 32m 54s","accounts_in_message":[],"_revision_number":4},{"id":"ca9cc78057541d6265e304b7db3608551e0f388a","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2023-10-27 13:59:08.000000000","message":"Patch Set 4: Verified-1\n\n(8 comments)\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttps://docs.opendev.org/opendev/infra-manual/latest/developers.html#automated-testing\n\nhttps://zuul.opendev.org/t/openstack/buildset/61aed87ec16a45879ce9844c0fe724be\n\n- test-release-openstack https://zuul.opendev.org/t/openstack/build/c46837f4c7054861a35f7b1b8d2d1f0e : SUCCESS in 2m 02s\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/f1b14e86676d45e1897cb80eebd55f40 : SUCCESS in 8m 26s\n- requirements-check https://zuul.opendev.org/t/openstack/build/174a71f2cdb844d6a2c1bb9d10e374e2 : FAILURE in 2m 46s\n- grenade https://zuul.opendev.org/t/openstack/build/59cd7660561941fa8b46d056c5eb6ea8 : SUCCESS in 50m 38s\n- grenade-skip-level https://zuul.opendev.org/t/openstack/build/79759194e2c04da7b582df4d45a089e7 : SUCCESS in 48m 00s\n- tempest-integrated-object-storage https://zuul.opendev.org/t/openstack/build/fad3f55560534dfb9eb85859b6281b37 : SUCCESS in 41m 14s\n- openstacksdk-functional-devstack https://zuul.opendev.org/t/openstack/build/a47848e6bb5f484d9e4bdecb727e5905 : SUCCESS in 54m 41s\n- swift-tox-func-py39-centos-9-stream-fips https://zuul.opendev.org/t/openstack/build/5f11fd9bb62042cab0b2ae448a8060c1 : SUCCESS in 27m 22s (non-voting)\n- swift-tox-func-encryption-py39-centos-9-stream-fips https://zuul.opendev.org/t/openstack/build/085ea291bacf4df9b1ed36f8d2a386eb : SUCCESS in 19m 34s (non-voting)\n- swift-tox-func-ec-py39-centos-9-stream-fips https://zuul.opendev.org/t/openstack/build/8d6c839f20ec4fc78bb678d0541e7ad9 : SUCCESS in 29m 56s (non-voting)\n- swift-build-image https://zuul.opendev.org/t/openstack/build/fdd7932a779847d58d12bd342d8212f5 : SUCCESS in 8m 27s (non-voting)\n- swift-build-image-py3 https://zuul.opendev.org/t/openstack/build/4c0c3b651d8a4ede8e20c7be9ad5211c : SUCCESS in 6m 16s (non-voting)\n- swift-tox-py27 https://zuul.opendev.org/t/openstack/build/6f3ab96efa8e45519810f7d8d55aefd0 : FAILURE in 4m 42s\n- swift-tox-py36 https://zuul.opendev.org/t/openstack/build/f848cfa18e3841239a3da5a33211b40a : SUCCESS in 15m 55s\n- swift-tox-py39 https://zuul.opendev.org/t/openstack/build/b94977523f4345298cfc48d574d5806a : SUCCESS in 14m 32s\n- swift-tox-py310 https://zuul.opendev.org/t/openstack/build/d8428861b96c43d7a3070fcb5a05b368 : SUCCESS in 22m 11s\n- swift-tox-py311 https://zuul.opendev.org/t/openstack/build/1ec37441abaf4d04a4a8b6ba7110802f : SUCCESS in 16m 01s\n- swift-tox-func-py27 https://zuul.opendev.org/t/openstack/build/f925f69c31f74315be81ab73af2f14c6 : FAILURE in 3m 49s\n- swift-tox-func-encryption-py27 https://zuul.opendev.org/t/openstack/build/7663b3c71cce44229f05ddca606d72fc : FAILURE in 3m 31s\n- swift-tox-func-ec-py27 https://zuul.opendev.org/t/openstack/build/7f0d8ccdbf0f4195ac862b8307eba678 : FAILURE in 3m 04s\n- swift-tox-func-py38 https://zuul.opendev.org/t/openstack/build/0c5b39d52d7048d9987e7d4b14c891b2 : SUCCESS in 22m 32s\n- swift-tox-func-encryption-py38 https://zuul.opendev.org/t/openstack/build/f5df6d4c28f64d8e8d4495d3c6a5ba97 : SUCCESS in 15m 45s\n- swift-tox-func-ec-py38 https://zuul.opendev.org/t/openstack/build/74cf0818402b4f718ba0223c20e609c7 : SUCCESS in 26m 23s\n- swift-func-cors https://zuul.opendev.org/t/openstack/build/7ac24bf72cb84399b1f44c57211baf6b : SUCCESS in 7m 14s\n- swift-tox-func-s3api-ceph-s3tests-tempauth https://zuul.opendev.org/t/openstack/build/bbb82b0c178b42f388b1ae2d4330b89c : SUCCESS in 22m 32s (non-voting)\n- swift-tox-func-s3api-tests-tempauth https://zuul.opendev.org/t/openstack/build/65c25a3841f64474a249faf848a4e5bf : SUCCESS in 8m 14s\n- swift-probetests-centos-7 https://zuul.opendev.org/t/openstack/build/3e934c1374124f1c9183039530c58bb1 : RETRY_LIMIT in 4m 04s\n- swift-probetests-centos-8-stream https://zuul.opendev.org/t/openstack/build/db8cd603b86d482caeebd84f3288a35c : SUCCESS in 1h 16m 27s\n- swift-dsvm-functional https://zuul.opendev.org/t/openstack/build/3a8adcee02cd4ea4bf809c35a04f0856 : SUCCESS in 44m 10s\n- swift-dsvm-functional-ipv6 https://zuul.opendev.org/t/openstack/build/e7ce92dc639f4f7f9f0ac9c13b23638e : SUCCESS in 43m 32s\n- swift-tox-lower-constraints https://zuul.opendev.org/t/openstack/build/7d46e403ca654807a6195069eac4e3b0 : FAILURE in 3m 24s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/7e7f190c374a4505a8fd58a09a67ddff : FAILURE in 4m 44s\n- swift-multinode-rolling-upgrade https://zuul.opendev.org/t/openstack/build/32033bdf673d438484f6205f99186155 : FAILURE in 7m 55s (non-voting)\n- tempest-ipv6-only https://zuul.opendev.org/t/openstack/build/f2f23932f7a6477d90cccf15400b7568 : SUCCESS in 50m 32s","accounts_in_message":[],"_revision_number":4},{"id":"030501eb1bcba64cb5622351e734c995ee373b37","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"date":"2024-03-14 06:00:28.000000000","message":"Uploaded patch set 5.\n\nOutdated Votes:\n* Verified-1\n","accounts_in_message":[],"_revision_number":5},{"id":"663bc0f1c7228696e72fa71e908a680c1e9386f7","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2024-03-14 07:25:40.000000000","message":"Patch Set 5: Verified-1\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttps://docs.opendev.org/opendev/infra-manual/latest/developers.html#automated-testing\n\nhttps://zuul.opendev.org/t/openstack/buildset/5f13467774c34a779dea2f37935e852e\n\n- test-release-openstack https://zuul.opendev.org/t/openstack/build/d8d7af11c58d42c1880ba45c5500e3a3 : SUCCESS in 2m 35s\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/79a986f30cab4914a2301e9b5e0a3f64 : SUCCESS in 8m 50s\n- requirements-check https://zuul.opendev.org/t/openstack/build/57a35996da5e4fadb16bf6b972f5969c : FAILURE in 3m 04s\n- grenade https://zuul.opendev.org/t/openstack/build/d5e596e41b8f40ad97c51af64ed17193 : FAILURE in 35m 01s\n- grenade-skip-level https://zuul.opendev.org/t/openstack/build/5f2bd5df7678492391c79c9b7c038a3a : SUCCESS in 53m 27s\n- tempest-integrated-object-storage https://zuul.opendev.org/t/openstack/build/661e0034cfa64960a5f4baed977bd2dd : SUCCESS in 53m 59s\n- openstacksdk-functional-devstack https://zuul.opendev.org/t/openstack/build/0216f8a7ca0a458d81ee8f8579ee1619 : SUCCESS in 55m 27s\n- swift-tox-func-py39-centos-9-stream-fips https://zuul.opendev.org/t/openstack/build/9fcc8b1c76454069b1c7ec0dc2286971 : SUCCESS in 18m 32s (non-voting)\n- swift-tox-func-encryption-py39-centos-9-stream-fips https://zuul.opendev.org/t/openstack/build/d704afc555074f83b73c1b666ca7c0ca : SUCCESS in 24m 59s (non-voting)\n- swift-tox-func-ec-py39-centos-9-stream-fips https://zuul.opendev.org/t/openstack/build/81bbbe63b90541b4af6a37e204a2c48b : SUCCESS in 20m 39s (non-voting)\n- swift-build-image https://zuul.opendev.org/t/openstack/build/65035710bd5245eba42bc1399e88c690 : SUCCESS in 6m 09s (non-voting)\n- swift-build-image-py3 https://zuul.opendev.org/t/openstack/build/1cc46c1121594bc886ea14aba38c36e2 : SUCCESS in 6m 08s (non-voting)\n- swift-tox-py27 https://zuul.opendev.org/t/openstack/build/f6d023e49b5a470185fd60303b403d6d : FAILURE in 3m 07s\n- swift-tox-py36 https://zuul.opendev.org/t/openstack/build/27a30166419d44a99b2953c938687fa6 : FAILURE in 19m 44s\n- swift-tox-py38 https://zuul.opendev.org/t/openstack/build/16686c9f249f4e21a655180c1069cfb7 : FAILURE in 18m 28s\n- swift-tox-py311 https://zuul.opendev.org/t/openstack/build/1008a98fe0bb4b71b6d6e317e9327595 : FAILURE in 23m 20s\n- swift-tox-func-py27 https://zuul.opendev.org/t/openstack/build/873aaa99fcb1450ab990ccbf29700253 : FAILURE in 4m 38s\n- swift-tox-func-encryption-py27 https://zuul.opendev.org/t/openstack/build/c34cc37563fe42dabf5fd9fda78706ff : FAILURE in 2m 51s\n- swift-tox-func-ec-py27 https://zuul.opendev.org/t/openstack/build/59e34d26aae74d84b23a33a9c2dca60a : FAILURE in 4m 37s\n- swift-tox-func-py38 https://zuul.opendev.org/t/openstack/build/2770c2419b3f4b339910b9a19a01fcef : SUCCESS in 13m 54s\n- swift-tox-func-encryption-py38 https://zuul.opendev.org/t/openstack/build/9527bd0179c34ab28c1575084f1b1235 : SUCCESS in 18m 16s\n- swift-tox-func-ec-py38 https://zuul.opendev.org/t/openstack/build/5d57f9673eb1484994227ef22011d532 : SUCCESS in 16m 49s\n- swift-func-cors https://zuul.opendev.org/t/openstack/build/a704c7fa4d2941e4b6ba2f2fdd1ae821 : SUCCESS in 9m 09s\n- swift-tox-func-s3api-ceph-s3tests-tempauth https://zuul.opendev.org/t/openstack/build/a46e0bb642604061a075a27cdf933bf7 : SUCCESS in 20m 07s (non-voting)\n- swift-tox-func-s3api-tests-tempauth https://zuul.opendev.org/t/openstack/build/2b1246853c1e41cb8a8e33eb66ef8f91 : SUCCESS in 8m 03s\n- swift-probetests-centos-8-stream https://zuul.opendev.org/t/openstack/build/7adb57e0ebf047368368daf824284766 : SUCCESS in 1h 17m 24s\n- swift-dsvm-functional https://zuul.opendev.org/t/openstack/build/b93b128185cb43d990dcd013ee023221 : SUCCESS in 41m 37s\n- swift-dsvm-functional-ipv6 https://zuul.opendev.org/t/openstack/build/7e57437404714af7badbf6b97f4395c4 : SUCCESS in 48m 23s\n- swift-tox-lower-constraints https://zuul.opendev.org/t/openstack/build/d003950f0fc24ac1b29c853c8eaf355b : FAILURE in 3m 18s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/5f3c890920334ec59c98045416692994 : SUCCESS in 4m 24s\n- swift-multinode-rolling-upgrade https://zuul.opendev.org/t/openstack/build/97dc7f1146a940b58197db8f2d91174c : FAILURE in 7m 28s (non-voting)\n- tempest-ipv6-only https://zuul.opendev.org/t/openstack/build/92a94e1813594492bb9d6d3a955f41ac : SUCCESS in 43m 23s","accounts_in_message":[],"_revision_number":5},{"id":"04f8c3fa7018c8df6d39e39d9c1b583681e5533a","tag":"autogenerated:zuul:check-arm64","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2024-03-14 07:33:55.000000000","message":"Patch Set 5:\n\nBuild failed (ARM64 pipeline).\nhttps://zuul.opendev.org/t/openstack/buildset/edc850ebfc4e41078a9441fa9790a283\n\n- swift-tox-py38-arm64 https://zuul.opendev.org/t/openstack/build/1c8b059f6ea84ad08dd626d22e0ae84a : FAILURE in 1h 06m 35s\n- swift-tox-py39-arm64 https://zuul.opendev.org/t/openstack/build/2c412e55ca164544ab27440475c974a7 : FAILURE in 16m 18s\n- swift-tox-func-encryption-py38-arm64 https://zuul.opendev.org/t/openstack/build/be775567c96140659ce3a631a30bbf89 : TIMED_OUT in 1h 31m 11s\n- swift-tox-func-py38-arm64 https://zuul.opendev.org/t/openstack/build/88df5ffa36c84f0985855cbaffc9d387 : SUCCESS in 17m 25s","accounts_in_message":[],"_revision_number":5},{"id":"80e482f2c96a9370f5ff84fdf6ec2ebd26ccdd3d","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"date":"2024-10-18 03:10:24.000000000","message":"Uploaded patch set 6.\n\nOutdated Votes:\n* Verified-1\n","accounts_in_message":[],"_revision_number":6},{"id":"1190d9b1e9765eb190eff30250c67c4597137960","tag":"autogenerated:zuul:check-arm64","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2024-10-18 04:50:30.000000000","message":"Patch Set 6:\n\nBuild failed (ARM64 pipeline).\nhttps://zuul.opendev.org/t/openstack/buildset/0d4c55dc10c84eca9cf8581d6399030f\n\n- swift-tox-py312-arm64 https://zuul.opendev.org/t/openstack/build/127b0397c3fa4286a8c3d22f84d48539 : FAILURE in 1h 37m 58s","accounts_in_message":[],"_revision_number":6},{"id":"00f4abac7e205a66680ce1f462899c89990416c7","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2024-10-18 05:20:36.000000000","message":"Patch Set 6: Verified-1\n\n(1 comment)\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttps://docs.opendev.org/opendev/infra-manual/latest/developers.html#automated-testing\nand https://docs.openstack.org/project-team-guide/testing.html#how-to-handle-test-failures\n\nhttps://zuul.opendev.org/t/openstack/buildset/c6af1f673be24474b146eaaa776b647e\n\n- test-release-openstack https://zuul.opendev.org/t/openstack/build/f42d126c7e32461ab9de3d886fa7b5be : SUCCESS in 1m 56s\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/8df0f42f3f38474388599bbc0295cd77 : SUCCESS in 11m 09s\n- requirements-check https://zuul.opendev.org/t/openstack/build/9597cb2f8c6a4fc8a63383e02a8fddd2 : FAILURE in 3m 14s\n- grenade https://zuul.opendev.org/t/openstack/build/7ceb1a7b861b4471864d1bcf499d382d : SUCCESS in 55m 56s\n- grenade-skip-level-always https://zuul.opendev.org/t/openstack/build/9f063cf2e80b441eb9014e98223e1e07 : SUCCESS in 51m 45s\n- tempest-integrated-object-storage https://zuul.opendev.org/t/openstack/build/f49a02c61f9f4eecb6b0eacebd2f3249 : SUCCESS in 54m 57s\n- openstacksdk-functional-devstack https://zuul.opendev.org/t/openstack/build/4147129cb74849f6b36798149e7e3110 : SUCCESS in 47m 42s\n- swift-tox-func-py39-centos-9-stream-fips https://zuul.opendev.org/t/openstack/build/33849cd929404fae9050093a1f34e1a3 : SUCCESS in 18m 55s (non-voting)\n- swift-tox-func-encryption-py39-centos-9-stream-fips https://zuul.opendev.org/t/openstack/build/9aeaf1cfc9c84bd5a6907a66f11ff26e : SUCCESS in 17m 04s (non-voting)\n- swift-tox-func-ec-py39-centos-9-stream-fips https://zuul.opendev.org/t/openstack/build/a604da8b1e3043c69133089d46d0f103 : SUCCESS in 18m 05s (non-voting)\n- swift-build-image https://zuul.opendev.org/t/openstack/build/1ac2d8889d4642feb3ba237d5d6fe000 : SUCCESS in 5m 58s (non-voting)\n- swift-tox-py27 https://zuul.opendev.org/t/openstack/build/bffd80c28ee64a3784a4d4203d7ccb50 : FAILURE in 3m 09s\n- swift-tox-py36 https://zuul.opendev.org/t/openstack/build/db7a1ec4102941dfbb398a04940c5359 : FAILURE in 22m 10s\n- swift-tox-py38 https://zuul.opendev.org/t/openstack/build/4200488c11c94db4a0d9a805e181ccda : FAILURE in 21m 34s\n- swift-tox-py311 https://zuul.opendev.org/t/openstack/build/a535d36883d7486caf19a8ba9ff0f6bd : FAILURE in 28m 28s\n- swift-tox-py312 https://zuul.opendev.org/t/openstack/build/ecd4084a79ae4d89b9e890692d255d25 : FAILURE in 35m 43s\n- swift-tox-func-py27 https://zuul.opendev.org/t/openstack/build/0dfed8b9036444589306c49ca1ad4bf7 : FAILURE in 2m 59s\n- swift-tox-func-encryption-py27 https://zuul.opendev.org/t/openstack/build/212a5d1e1e3f400faa9b28c869816784 : FAILURE in 2m 48s\n- swift-tox-func-ec-py27 https://zuul.opendev.org/t/openstack/build/ba377b79481c4ae49adf0c4db5809d7d : FAILURE in 3m 16s\n- swift-tox-func-py38 https://zuul.opendev.org/t/openstack/build/42575856a48c45d6b8a0263afb158b90 : SUCCESS in 14m 13s\n- swift-tox-func-encryption-py38 https://zuul.opendev.org/t/openstack/build/ae25b57c826848599e1167a3d0c0d31b : SUCCESS in 14m 17s\n- swift-tox-func-ec-py38 https://zuul.opendev.org/t/openstack/build/49c6b23253a54dc189d61567dd3d4ab3 : SUCCESS in 14m 11s\n- swift-func-cors https://zuul.opendev.org/t/openstack/build/139c93a1eba845f98040bc8388e20ee2 : SUCCESS in 5m 49s\n- swift-tox-func-s3api-ceph-s3tests-tempauth https://zuul.opendev.org/t/openstack/build/974024c37ace4a238deaa2a9af3c4f29 : SUCCESS in 16m 40s (non-voting)\n- swift-tox-func-s3api-tests-tempauth https://zuul.opendev.org/t/openstack/build/ed71140b35554821b7c25aeb05b75e42 : SUCCESS in 9m 28s\n- swift-probetests-centos-9-stream https://zuul.opendev.org/t/openstack/build/b8d176200866417ea662e102657ff459 : TIMED_OUT in 2h 00m 59s\n- swift-dsvm-functional https://zuul.opendev.org/t/openstack/build/eaf9baf882444c47a55bbca6bc6a9189 : SUCCESS in 37m 12s\n- swift-dsvm-functional-ipv6 https://zuul.opendev.org/t/openstack/build/95027cbd184f4d458afcab18a0403613 : SUCCESS in 51m 01s\n- swift-tox-lower-constraints https://zuul.opendev.org/t/openstack/build/93ef6d80d54b49a586bdcaf9171b707b : FAILURE in 4m 15s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/dff13da80ae24fff89b16ce26a217906 : FAILURE in 4m 19s\n- swift-multinode-rolling-upgrade https://zuul.opendev.org/t/openstack/build/b017c9717f62473b87ff1aa488b039ab : FAILURE in 6m 51s\n- tempest-ipv6-only https://zuul.opendev.org/t/openstack/build/fdb853d1aa644629842b286b7815b429 : SUCCESS in 40m 26s","accounts_in_message":[],"_revision_number":6},{"id":"2c9ebc06f0a9808c16498df2717783e300427cdd","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"date":"2025-01-30 05:19:23.000000000","message":"Uploaded patch set 7.\n\nOutdated Votes:\n* Verified-1\n","accounts_in_message":[],"_revision_number":7},{"id":"4ded2f11bb944d2c0c989096a0cdcf4d5e2dbb49","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2025-01-30 06:35:38.000000000","message":"Patch Set 7: Verified-1\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttps://docs.opendev.org/opendev/infra-manual/latest/developers.html#automated-testing\nand https://docs.openstack.org/project-team-guide/testing.html#how-to-handle-test-failures\n\nhttps://zuul.opendev.org/t/openstack/buildset/809b70cc40a5439eb11bb11694e2d540\n\n- test-release-openstack https://zuul.opendev.org/t/openstack/build/2e9f6d7403f7491f840ea89d24f1d338 : SUCCESS in 1m 48s\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/7d3995ad32b14accbfcb67e04f891c92 : SUCCESS in 8m 40s\n- requirements-check https://zuul.opendev.org/t/openstack/build/3a6318ee8c814bb6a0852ff5546c5626 : FAILURE in 2m 55s\n- grenade https://zuul.opendev.org/t/openstack/build/6ba1c0cd8c2b45a297f1272bbdde64d4 : SUCCESS in 1h 09m 27s\n- grenade-skip-level-always https://zuul.opendev.org/t/openstack/build/ad294784f6d4443f8af8f5e7e0b4cca9 : SUCCESS in 46m 56s\n- tempest-integrated-object-storage https://zuul.opendev.org/t/openstack/build/18ed7daebfb44acd94e01d6529f08fb5 : SUCCESS in 40m 41s\n- openstacksdk-functional-devstack https://zuul.opendev.org/t/openstack/build/454cdf8eb85446d6873fe1f5bf91d5bf : SUCCESS in 1h 11m 33s\n- swift-tox-func-py39-centos-9-stream-fips https://zuul.opendev.org/t/openstack/build/f5e97eeace394b629c3b16d6c5667b68 : SUCCESS in 26m 11s (non-voting)\n- swift-tox-func-encryption-py39-centos-9-stream-fips https://zuul.opendev.org/t/openstack/build/ec07328752f749fda397f5d0b0c7ddb8 : SUCCESS in 26m 15s (non-voting)\n- swift-tox-func-ec-py39-centos-9-stream-fips https://zuul.opendev.org/t/openstack/build/a109c7981caa41c9b31a1acdef25bd56 : SUCCESS in 26m 11s (non-voting)\n- swift-build-image https://zuul.opendev.org/t/openstack/build/6a064ceb6c97441ba535c7026ad3703e : SUCCESS in 6m 15s (non-voting)\n- swift-tox-py36 https://zuul.opendev.org/t/openstack/build/6d8acc93938f48f6911db70530d3a139 : FAILURE in 19m 23s\n- swift-tox-py39 https://zuul.opendev.org/t/openstack/build/835cb3752b9349ae94659fc57cd7fa1e : FAILURE in 10m 57s\n- swift-tox-py312 https://zuul.opendev.org/t/openstack/build/6665f5eecd924a28baa7d3417b36085e : FAILURE in 19m 54s\n- swift-tox-func-py312 https://zuul.opendev.org/t/openstack/build/64f0d2e569d44ee3adab3b5dd9da4f1c : SUCCESS in 10m 11s\n- swift-tox-func-encryption-py312 https://zuul.opendev.org/t/openstack/build/beb624d6830347b6b3c22cfc6c3d352d : SUCCESS in 15m 29s\n- swift-tox-func-ec-py312 https://zuul.opendev.org/t/openstack/build/786b8a9a8833458899a232f1f0c48d74 : SUCCESS in 15m 16s\n- swift-func-cors https://zuul.opendev.org/t/openstack/build/ad86acf2f2db4b5f95c3720f9545dd3d : SUCCESS in 6m 17s\n- swift-tox-func-s3api-ceph-s3tests-tempauth https://zuul.opendev.org/t/openstack/build/21dc0fd24e9c44ad87af2cc6ac6da8db : FAILURE in 12m 18s (non-voting)\n- swift-tox-func-s3api-tests-tempauth https://zuul.opendev.org/t/openstack/build/3f796989f7594ba7ba7a669b74519402 : SUCCESS in 10m 45s\n- swift-probetests-centos-9-stream https://zuul.opendev.org/t/openstack/build/f9ec986cf96c4ddcbc5109d2c6eac7ae : SUCCESS in 53m 53s\n- swift-dsvm-functional https://zuul.opendev.org/t/openstack/build/c8fcaf73f0974b98ba2eadf493243899 : SUCCESS in 54m 49s\n- swift-dsvm-functional-ipv6 https://zuul.opendev.org/t/openstack/build/53e7bead725040ad856a1869fee5b612 : SUCCESS in 50m 56s\n- swift-tox-lower-constraints https://zuul.opendev.org/t/openstack/build/4c29ab34bcf74874a5288fcfb2548e32 : FAILURE in 22m 24s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/6515081bbed944dcb9fdc8fef836734c : SUCCESS in 5m 07s\n- swift-multinode-rolling-upgrade https://zuul.opendev.org/t/openstack/build/9434269ec1e24209957c10078e86b0ef : FAILURE in 6m 59s\n- tempest-integrated-object-storage-ubuntu-jammy https://zuul.opendev.org/t/openstack/build/a6456acb69fb44f89602e36b54d86e09 : SUCCESS in 41m 40s\n- tempest-ipv6-only https://zuul.opendev.org/t/openstack/build/4e3febe2ad4d482a92b619ec87784ab8 : SUCCESS in 36m 01s","accounts_in_message":[],"_revision_number":7},{"id":"0d7a25e7f0926584e42b1573907e5a1307a8a0be","tag":"autogenerated:zuul:check-arm64","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2025-01-30 07:19:04.000000000","message":"Patch Set 7:\n\nBuild failed (ARM64 pipeline).\nhttps://zuul.opendev.org/t/openstack/buildset/a92b5dbd0d134c35ae92f7924f2dad87\n\n- swift-tox-py312-arm64 https://zuul.opendev.org/t/openstack/build/9aab67da3f8e4574b27c8f4de0142119 : FAILURE in 1h 57m 30s","accounts_in_message":[],"_revision_number":7},{"id":"d0ed7392d01612330eb5a16ceea2afb6f570b3a3","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"date":"2025-05-30 06:40:05.000000000","message":"Uploaded patch set 8.\n\nOutdated Votes:\n* Verified-1\n","accounts_in_message":[],"_revision_number":8},{"id":"2fe5b224d0e3659d1a8e90a569eac54574aa8e8a","tag":"autogenerated:zuul:check-arm64","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2025-05-30 06:49:25.000000000","message":"Patch Set 8:\n\nBuild failed (ARM64 pipeline).\nhttps://zuul.opendev.org/t/openstack/buildset/017a0408f100413997419f992759fc97\n\n- swift-tox-py312-arm64 https://zuul.opendev.org/t/openstack/build/8264e034d17348fb9576ef6fc608022f : FAILURE in 7m 37s","accounts_in_message":[],"_revision_number":8},{"id":"1c0507c4da1138b9b957dace90182d238c553d93","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2025-05-30 08:33:26.000000000","message":"Patch Set 8: Verified-1\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttps://docs.opendev.org/opendev/infra-manual/latest/developers.html#automated-testing\nand https://docs.openstack.org/project-team-guide/testing.html#how-to-handle-test-failures\n\nhttps://zuul.opendev.org/t/openstack/buildset/9a942e16dab1488f93280952992b8a62\n\n- test-release-openstack https://zuul.opendev.org/t/openstack/build/b3873dd845864b318ae67787d13777ed : SUCCESS in 3m 19s\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/f4d71b32d29d4d089e281ed75d7e7219 : SUCCESS in 10m 27s\n- requirements-check https://zuul.opendev.org/t/openstack/build/85475a4da0f64b3d88e657c03bc77363 : FAILURE in 3m 19s\n- grenade https://zuul.opendev.org/t/openstack/build/487c2254d565451cbb6d791b09454f43 : SUCCESS in 57m 03s\n- grenade-skip-level-always https://zuul.opendev.org/t/openstack/build/b96efe39a059492ca257b8d144bff6ce : SUCCESS in 59m 16s\n- tempest-integrated-object-storage https://zuul.opendev.org/t/openstack/build/6e02f5509fd045359b2cfd8f1670df13 : SUCCESS in 39m 02s\n- openstacksdk-functional-devstack https://zuul.opendev.org/t/openstack/build/0d084aeae8e645b98328fe01116b570b : SUCCESS in 46m 42s\n- swift-tox-func-py39-centos-9-stream-fips https://zuul.opendev.org/t/openstack/build/f37f4d5e5caa494b96a8307a54a15cbc : SUCCESS in 15m 44s (non-voting)\n- swift-tox-func-encryption-py39-centos-9-stream-fips https://zuul.opendev.org/t/openstack/build/2448241dd4b34863ba6d505d18fa9ab1 : SUCCESS in 21m 25s (non-voting)\n- swift-tox-func-ec-py39-centos-9-stream-fips https://zuul.opendev.org/t/openstack/build/ad7b8faa65334801a83408de45dbacf0 : SUCCESS in 22m 27s (non-voting)\n- swift-build-image https://zuul.opendev.org/t/openstack/build/4698dc9313004b7b810587a0d3b89eae : SUCCESS in 4m 59s (non-voting)\n- swift-tox-py37 https://zuul.opendev.org/t/openstack/build/c7b8866e688940c095c9213a814044d0 : FAILURE in 4m 05s\n- swift-tox-py39 https://zuul.opendev.org/t/openstack/build/cb4910066ca14067a70656fa56c2fe56 : FAILURE in 4m 25s\n- swift-tox-py312 https://zuul.opendev.org/t/openstack/build/cd30997f49ca4d71a56327fa2d03a625 : FAILURE in 10m 11s\n- swift-tox-py313 https://zuul.opendev.org/t/openstack/build/f6bbbd64526d42bcbc5a7df5ecd30c12 : FAILURE in 5m 57s\n- swift-tox-func-py312 https://zuul.opendev.org/t/openstack/build/d135182d64594ba0a7240f28a5eb9bbd : SUCCESS in 13m 53s\n- swift-tox-func-encryption-py312 https://zuul.opendev.org/t/openstack/build/615b6175e62c486081c3ad8f8dcaf021 : SUCCESS in 12m 35s\n- swift-tox-func-ec-py312 https://zuul.opendev.org/t/openstack/build/ea6a05216dd442a4b936492acd7bfdb1 : SUCCESS in 16m 07s\n- swift-func-cors https://zuul.opendev.org/t/openstack/build/e263375237d94260a46ec6d50cea468a : SUCCESS in 6m 14s\n- swift-tox-func-ceph-s3tests-tempauth https://zuul.opendev.org/t/openstack/build/7a94fd69bad34b2f8bf298390e363a5a : SUCCESS in 13m 10s (non-voting)\n- swift-tox-func-s3api-compat-tests-tempauth https://zuul.opendev.org/t/openstack/build/c5ecd7cb560947feaabf7c4e44661af1 : SUCCESS in 6m 21s\n- swift-probetests-centos-9-stream https://zuul.opendev.org/t/openstack/build/d1587f7c670c461ca2961fde223c1766 : SUCCESS in 52m 34s\n- swift-dsvm-functional https://zuul.opendev.org/t/openstack/build/02236815ec4a4ab188af6cbe581acb8a : SUCCESS in 35m 03s\n- swift-dsvm-functional-ipv6 https://zuul.opendev.org/t/openstack/build/cdbf87290222456280956a2f9459d496 : SUCCESS in 35m 19s\n- swift-tox-lower-constraints https://zuul.opendev.org/t/openstack/build/d40a0fe4c5044c609eb457b271da57d1 : FAILURE in 4m 33s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/a214812e5ef34252b4d5c64107658d9d : SUCCESS in 3m 36s\n- swift-multinode-rolling-upgrade https://zuul.opendev.org/t/openstack/build/f4b77e5a432c47099f9ae9b21ef3cde1 : FAILURE in 6m 12s\n- tempest-integrated-object-storage-ubuntu-jammy https://zuul.opendev.org/t/openstack/build/ccdb15e32840473d8e8ca3c1c8c0423f : SUCCESS in 47m 18s\n- tempest-ipv6-only https://zuul.opendev.org/t/openstack/build/d08595c2e45845aca3d971a0e697c4b9 : SUCCESS in 1h 02m 56s","accounts_in_message":[],"_revision_number":8},{"id":"4e44e8b5c663aede17d740333ef80c65ff6db86e","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"date":"2025-07-14 07:25:46.000000000","message":"Uploaded patch set 9.\n\nOutdated Votes:\n* Verified-1\n","accounts_in_message":[],"_revision_number":9},{"id":"63007d578662d96dd83605c7b1d506b61b62be55","tag":"autogenerated:zuul:check-arm64","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2025-07-14 07:41:41.000000000","message":"Patch Set 9:\n\nBuild failed (ARM64 pipeline).\nhttps://zuul.opendev.org/t/openstack/buildset/2c71763fad4c4d9ea96e190aee69aada\n\n- swift-tox-py312-arm64 https://zuul.opendev.org/t/openstack/build/7a7861ecfc544826ab905d219ddf1090 : FAILURE in 8m 47s","accounts_in_message":[],"_revision_number":9},{"id":"279b855a76584b6eb36bf6ed096877bb9fbf3e74","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2025-07-14 10:29:26.000000000","message":"Patch Set 9: Verified-1\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttps://docs.opendev.org/opendev/infra-manual/latest/developers.html#automated-testing\nand https://docs.openstack.org/project-team-guide/testing.html#how-to-handle-test-failures\n\nhttps://zuul.opendev.org/t/openstack/buildset/aca5990c64cc46d985750ca51e14070b\n\n- test-release-openstack https://zuul.opendev.org/t/openstack/build/3dbac2db25ed449788ab701d501d900b : SUCCESS in 3m 06s\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/39303350e88a4ce1af3fc4cae35a5487 : FAILURE in 5m 44s\n- requirements-check https://zuul.opendev.org/t/openstack/build/f396e3a8a1014a9ba897d624a78f9c72 : FAILURE in 2m 44s\n- grenade https://zuul.opendev.org/t/openstack/build/d9a26c97c2af4f239ec780f270446ca8 : SUCCESS in 28m 01s\n- grenade-skip-level-always https://zuul.opendev.org/t/openstack/build/91abbd32a96545f58998c01e73f69d65 : TIMED_OUT in 3h 02m 05s\n- tempest-integrated-object-storage https://zuul.opendev.org/t/openstack/build/6c0cc10e790d418aa29348120f48189d : SUCCESS in 39m 41s\n- openstacksdk-functional-devstack https://zuul.opendev.org/t/openstack/build/6a6e93dfd4c04aa2aca37a9d03a4d328 : SUCCESS in 55m 07s\n- swift-tox-func-py39-centos-9-stream-fips https://zuul.opendev.org/t/openstack/build/9c88b725a1954aee9c9ca544603a4958 : SUCCESS in 18m 46s (non-voting)\n- swift-tox-func-encryption-py39-centos-9-stream-fips https://zuul.opendev.org/t/openstack/build/4a135db520384c6bb4bee6eeec665894 : FAILURE in 7m 31s (non-voting)\n- swift-tox-func-ec-py39-centos-9-stream-fips https://zuul.opendev.org/t/openstack/build/5652a1ba7b284e9b922b8fef079e18c6 : SUCCESS in 19m 46s (non-voting)\n- swift-build-image https://zuul.opendev.org/t/openstack/build/6252c03247d4495fa7c546efae4d155a : SUCCESS in 5m 47s (non-voting)\n- swift-tox-py37 https://zuul.opendev.org/t/openstack/build/85fd6df0f99746528b3a53d97d077294 : FAILURE in 4m 50s\n- swift-tox-py39 https://zuul.opendev.org/t/openstack/build/c958a3276fbe4ad5a6e0bf731a995337 : FAILURE in 5m 17s\n- swift-tox-py312 https://zuul.opendev.org/t/openstack/build/8edcc8ebd32541dea1e1772b2f31da12 : FAILURE in 6m 31s\n- swift-tox-py313 https://zuul.opendev.org/t/openstack/build/c42e204e590c4fe19a20e5eecdf551b3 : FAILURE in 6m 54s\n- swift-tox-func-py312 https://zuul.opendev.org/t/openstack/build/9ac7f98ce54d46eeacc812feca7a6781 : SUCCESS in 8m 03s\n- swift-tox-func-encryption-py312 https://zuul.opendev.org/t/openstack/build/a692a008aacc4619bf303c33d9340356 : FAILURE in 2m 44s\n- swift-tox-func-ec-py312 https://zuul.opendev.org/t/openstack/build/3d23a97ad5fe4eafa939980ceaedb597 : SUCCESS in 14m 29s\n- swift-func-cors https://zuul.opendev.org/t/openstack/build/20ef8c619c8f432bb849875be5f7a472 : SUCCESS in 5m 54s\n- swift-tox-func-ceph-s3tests-tempauth https://zuul.opendev.org/t/openstack/build/2f4bcf4a268a44718eadaef399235b54 : SUCCESS in 14m 04s (non-voting)\n- swift-tox-func-s3api-compat-tests-tempauth https://zuul.opendev.org/t/openstack/build/6e4843188d05407bb248b71d32ec1728 : FAILURE in 4m 56s\n- swift-probetests-centos-9-stream https://zuul.opendev.org/t/openstack/build/c048afbc76224e03b60e008e73de021c : SUCCESS in 1h 02m 01s\n- swift-dsvm-functional https://zuul.opendev.org/t/openstack/build/6c9141728613449da2b1008dedb4bc78 : FAILURE in 10m 32s\n- swift-dsvm-functional-ipv6 https://zuul.opendev.org/t/openstack/build/04f700dcdcfa439d90171e60887e919c : FAILURE in 12m 20s\n- swift-tox-lower-constraints https://zuul.opendev.org/t/openstack/build/c404f4f9a9324c4b980bf375a3fe59db : FAILURE in 5m 06s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/648d6e2101814c24a983f1992dff5d3a : SUCCESS in 4m 30s\n- swift-multinode-rolling-upgrade https://zuul.opendev.org/t/openstack/build/59b102050d314ae7ae4811d1dfb2c246 : FAILURE in 7m 19s\n- tempest-integrated-object-storage-ubuntu-jammy https://zuul.opendev.org/t/openstack/build/e2f77d1e5ec44f9bac9360b9c6eb9b80 : SUCCESS in 57m 56s\n- tempest-ipv6-only https://zuul.opendev.org/t/openstack/build/0b1d2a2688674d6d8d2199f227dd02c0 : SUCCESS in 39m 06s","accounts_in_message":[],"_revision_number":9},{"id":"d6340a4c12e13150f15ef6edc404483f06ad4cee","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"date":"2025-08-20 02:50:39.000000000","message":"Uploaded patch set 10.\n\nOutdated Votes:\n* Verified-1\n","accounts_in_message":[],"_revision_number":10},{"id":"9bb52ff2b97d9434bd8020f1038fe3a19ef6a4bb","tag":"autogenerated:zuul:check-arm64","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2025-08-20 03:36:32.000000000","message":"Patch Set 10:\n\nBuild failed (ARM64 pipeline).\nhttps://zuul.opendev.org/t/openstack/buildset/37bb208912844516b2e26f3bf79d58ed\n\n- swift-tox-py312-arm64 https://zuul.opendev.org/t/openstack/build/0c32c0f4d240441c99d12bb54fc255a8 : FAILURE in 14m 13s","accounts_in_message":[],"_revision_number":10},{"id":"114b4793a1cfbf5ce0cc388ecf3506438b426d74","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2025-08-20 04:01:22.000000000","message":"Patch Set 10: Verified-1\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttps://docs.opendev.org/opendev/infra-manual/latest/developers.html#automated-testing\nand https://docs.openstack.org/project-team-guide/testing.html#how-to-handle-test-failures\n\nhttps://zuul.opendev.org/t/openstack/buildset/57e7586048be476fae8fcedc2aca2e4e\n\n- test-release-openstack https://zuul.opendev.org/t/openstack/build/58414c63575148209ee27c577c5582c6 : SUCCESS in 3m 41s\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/62d933c7777c4ca09b74eb02f61dd860 : FAILURE in 5m 53s\n- requirements-check https://zuul.opendev.org/t/openstack/build/90b379f3a0cd417a82bcea0cc3e302b0 : FAILURE in 2m 45s\n- grenade https://zuul.opendev.org/t/openstack/build/8da1c7ea87c74eb6a72e73dbfd23e8f5 : SUCCESS in 1h 02m 32s\n- grenade-skip-level-always https://zuul.opendev.org/t/openstack/build/ddc0b04048424c899628b7024e50f816 : SUCCESS in 1h 04m 32s\n- tempest-integrated-object-storage https://zuul.opendev.org/t/openstack/build/7d1b41b261fd460fbfc131945081bf7a : SUCCESS in 53m 17s\n- openstacksdk-functional-devstack https://zuul.opendev.org/t/openstack/build/cf95c6923c5a47fd981d41b301c920bf : SUCCESS in 31m 32s\n- swift-tox-func-py39-centos-9-stream-fips https://zuul.opendev.org/t/openstack/build/e8fa6f1379b74620a0a340bed3fbfa8c : SUCCESS in 19m 53s (non-voting)\n- swift-tox-func-encryption-py39-centos-9-stream-fips https://zuul.opendev.org/t/openstack/build/5a92ebcff88d4030a6de356eabc4b9fc : FAILURE in 6m 48s (non-voting)\n- swift-tox-func-ec-py39-centos-9-stream-fips https://zuul.opendev.org/t/openstack/build/4a19ac2002d74203b2f58a83b650ac2f : SUCCESS in 21m 24s (non-voting)\n- swift-build-image https://zuul.opendev.org/t/openstack/build/d7a262daa305405daebb5e24ae8f58b3 : SUCCESS in 6m 50s (non-voting)\n- swift-tox-py37 https://zuul.opendev.org/t/openstack/build/2b344e9dcbd4448da3bf0c7a412b191c : FAILURE in 10m 23s\n- swift-tox-py39 https://zuul.opendev.org/t/openstack/build/e2cc7587c4df410da9749eb701dd993d : FAILURE in 6m 16s\n- swift-tox-py312 https://zuul.opendev.org/t/openstack/build/ac078118c2204cf2bbd5c2e8082b4ea1 : FAILURE in 9m 29s\n- swift-tox-py313 https://zuul.opendev.org/t/openstack/build/925c660eec4b4f788ccfb27ba903af92 : FAILURE in 4m 16s\n- swift-tox-func-py312 https://zuul.opendev.org/t/openstack/build/60e22ebfa52748419ab6e424e1ee04ac : SUCCESS in 13m 33s\n- swift-tox-func-encryption-py312 https://zuul.opendev.org/t/openstack/build/1e1a8f709c714da6a63ca28d91724c3a : FAILURE in 3m 16s\n- swift-tox-func-ec-py312 https://zuul.opendev.org/t/openstack/build/5593fefc50974807a49fb29f0891373d : SUCCESS in 9m 54s\n- swift-func-cors https://zuul.opendev.org/t/openstack/build/0ae2034f6dca4e58ab78622aa972270e : SUCCESS in 7m 11s\n- swift-tox-func-ceph-s3tests-tempauth https://zuul.opendev.org/t/openstack/build/4e57da249dde4151977e23a3c4eb5471 : SUCCESS in 17m 46s (non-voting)\n- swift-tox-func-s3api-compat-tests-tempauth https://zuul.opendev.org/t/openstack/build/aff3643d2ecd47cf8d156fc5918e7364 : FAILURE in 9m 37s\n- swift-probetests-centos-9-stream https://zuul.opendev.org/t/openstack/build/27f6e358e80949b8b9f418578dcc2a71 : SUCCESS in 1h 02m 59s\n- swift-dsvm-functional https://zuul.opendev.org/t/openstack/build/b38e7504ee3640f6b8c1e9e3f17e6a3d : FAILURE in 16m 52s\n- swift-dsvm-functional-ipv6 https://zuul.opendev.org/t/openstack/build/c3db95dd09cf46f9bc780f79fcc81404 : FAILURE in 16m 49s\n- swift-tox-lower-constraints https://zuul.opendev.org/t/openstack/build/c8fb0313d448490bb87cd28534554d72 : FAILURE in 12m 44s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/c67edcc57f1343e18538b02fe3545826 : SUCCESS in 4m 18s\n- swift-multinode-rolling-upgrade https://zuul.opendev.org/t/openstack/build/13122d736c704230a693c5e86eeb12d2 : FAILURE in 7m 13s\n- tempest-integrated-object-storage-ubuntu-jammy https://zuul.opendev.org/t/openstack/build/5b3847317f164e6ea96e8278ec59cacd : SUCCESS in 27m 48s\n- tempest-ipv6-only https://zuul.opendev.org/t/openstack/build/e45b277b9f364ed68fe3857d15f8b035 : SUCCESS in 26m 56s","accounts_in_message":[],"_revision_number":10},{"id":"f6069593a7c15c0814a59a5b82738b5edfce2a4a","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"date":"2025-09-04 07:25:27.000000000","message":"Uploaded patch set 11.\n\nOutdated Votes:\n* Verified-1\n","accounts_in_message":[],"_revision_number":11},{"id":"d5a6b11c92981136fb16e1290c4f31aea7c78058","tag":"autogenerated:zuul:check-arm64","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2025-09-04 07:39:31.000000000","message":"Patch Set 11:\n\nBuild failed (ARM64 pipeline).\nhttps://zuul.opendev.org/t/openstack/buildset/ea57843a00bd49fbadcd6d2fe3566951\n\n- swift-tox-py312-arm64 https://zuul.opendev.org/t/openstack/build/1a8b55936ca446e683965f1e97834fe7 : FAILURE in 11m 16s","accounts_in_message":[],"_revision_number":11},{"id":"c5a58410492bea6115a21ec8f811be58dd588924","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2025-09-04 08:38:13.000000000","message":"Patch Set 11: Verified-1\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttps://docs.opendev.org/opendev/infra-manual/latest/developers.html#automated-testing\nand https://docs.openstack.org/project-team-guide/testing.html#how-to-handle-test-failures\n\nhttps://zuul.opendev.org/t/openstack/buildset/1f415f5be5d34264927130939ca40ea8\n\n- test-release-openstack https://zuul.opendev.org/t/openstack/build/ad722207cf494e458757b627f4b322d5 : SUCCESS in 2m 45s\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/f57732f1cbc94dadad57c4a438678d9a : SUCCESS in 11m 14s\n- requirements-check https://zuul.opendev.org/t/openstack/build/81967941f24941faaba8edeb2ca62e10 : FAILURE in 2m 42s\n- grenade https://zuul.opendev.org/t/openstack/build/d2dc2adeb2114e31afce35f199e3e012 : SUCCESS in 1h 05m 39s\n- grenade-skip-level-always https://zuul.opendev.org/t/openstack/build/33a0400513d947f09c199de912083b7c : SUCCESS in 40m 24s\n- tempest-integrated-object-storage https://zuul.opendev.org/t/openstack/build/4166a3fe295548718d20a33eb87de022 : SUCCESS in 44m 08s\n- openstacksdk-functional-devstack https://zuul.opendev.org/t/openstack/build/0a498026eb674f48b93331a1fdb4c1a7 : SUCCESS in 1h 04m 46s\n- swift-tox-func-py39-centos-9-stream-fips https://zuul.opendev.org/t/openstack/build/47d0b15fe50148189297160b4896446c : SUCCESS in 19m 28s (non-voting)\n- swift-tox-func-encryption-py39-centos-9-stream-fips https://zuul.opendev.org/t/openstack/build/5f6b331999d94e8bb58ebce95c1d3dca : SUCCESS in 20m 10s (non-voting)\n- swift-tox-func-ec-py39-centos-9-stream-fips https://zuul.opendev.org/t/openstack/build/05cf19468bee4b7d8b69be4c53595644 : SUCCESS in 19m 47s (non-voting)\n- swift-build-image https://zuul.opendev.org/t/openstack/build/80519a546c1f43d8b0b2059194f657b9 : SUCCESS in 4m 06s (non-voting)\n- swift-tox-py37 https://zuul.opendev.org/t/openstack/build/a3f31a27e20745ccb85796ec70e74697 : FAILURE in 8m 32s\n- swift-tox-py39 https://zuul.opendev.org/t/openstack/build/2d9708e0b9604f7aae4040f14b8bac07 : FAILURE in 4m 32s\n- swift-tox-py312 https://zuul.opendev.org/t/openstack/build/6c3f88acae7c4513b6d1353e3e3b4c4b : FAILURE in 4m 23s\n- swift-tox-py313 https://zuul.opendev.org/t/openstack/build/380427744691481cabc855cadb23b1c1 : FAILURE in 7m 29s\n- swift-tox-func-py312 https://zuul.opendev.org/t/openstack/build/81c3773a44cc481bb2052fab4e497b37 : SUCCESS in 10m 50s\n- swift-tox-func-encryption-py312 https://zuul.opendev.org/t/openstack/build/1241f1c0d8094db3a5debf71a1a99b2d : SUCCESS in 13m 27s\n- swift-tox-func-ec-py312 https://zuul.opendev.org/t/openstack/build/72fb2f9dba924e258e243252b3e84652 : SUCCESS in 15m 11s\n- swift-func-cors https://zuul.opendev.org/t/openstack/build/d6c714b690674d1fb26a242d5a750eb3 : SUCCESS in 7m 57s\n- swift-tox-func-ceph-s3tests-tempauth https://zuul.opendev.org/t/openstack/build/a7746fdb8ca24eb9898aa1b88bcc9b8b : SUCCESS in 19m 02s (non-voting)\n- swift-tox-func-s3api-compat-tests-tempauth https://zuul.opendev.org/t/openstack/build/82dc05a758cd444c8af84af41a7754c7 : SUCCESS in 5m 47s\n- swift-probetests-centos-9-stream https://zuul.opendev.org/t/openstack/build/18b20c197e5c4011a95f0fe7bed45929 : SUCCESS in 1h 00m 50s\n- swift-dsvm-functional https://zuul.opendev.org/t/openstack/build/9468f940f7dc4a9a9a414c9f8d3fae94 : SUCCESS in 44m 54s\n- swift-dsvm-functional-ipv6 https://zuul.opendev.org/t/openstack/build/19da17f24de040e99be5786682193dca : SUCCESS in 22m 01s\n- swift-tox-lower-constraints https://zuul.opendev.org/t/openstack/build/7d65451e722d467bb4f8dffcb36d8486 : FAILURE in 9m 19s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/837e41c80c404c6b9ebc42f32def3a5e : SUCCESS in 4m 24s\n- swift-multinode-rolling-upgrade https://zuul.opendev.org/t/openstack/build/6a929f201df040a998c7ceab0afe284a : FAILURE in 7m 15s\n- tempest-integrated-object-storage-ubuntu-jammy https://zuul.opendev.org/t/openstack/build/bd7f2570bcc94209bac783d1e2deb1d9 : SUCCESS in 50m 35s\n- tempest-ipv6-only https://zuul.opendev.org/t/openstack/build/af18036bf65e40c390714d8cbbb93218 : SUCCESS in 29m 01s","accounts_in_message":[],"_revision_number":11},{"id":"0b4643735ae4fc801542d2eb48d4ebb1e8c70a6c","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"date":"2025-09-15 07:22:22.000000000","message":"Uploaded patch set 12.\n\nOutdated Votes:\n* Verified-1\n","accounts_in_message":[],"_revision_number":12},{"id":"717b506bd64e03d6c672e934888be4edcd6b1850","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2025-09-15 08:48:04.000000000","message":"Patch Set 12: Verified-1\n\n(2 comments)\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttps://docs.opendev.org/opendev/infra-manual/latest/developers.html#automated-testing\nand https://docs.openstack.org/project-team-guide/testing.html#how-to-handle-test-failures\n\nhttps://zuul.opendev.org/t/openstack/buildset/103e2fe5ded24ad98293e7a4fda9764b\n\n- test-release-openstack https://zuul.opendev.org/t/openstack/build/74748ccaaba441fdb1f33b5c2ffbc981 : SUCCESS in 3m 02s\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/7ad77e19e12b4f6b90da82d84e851091 : SUCCESS in 11m 29s\n- requirements-check https://zuul.opendev.org/t/openstack/build/82b9bfdc41de44f1b8916c7b28e0a482 : FAILURE in 2m 33s\n- grenade https://zuul.opendev.org/t/openstack/build/e33634ef955440cd96622287dbf92a5a : SUCCESS in 46m 02s\n- grenade-skip-level-always https://zuul.opendev.org/t/openstack/build/de56d0f7325349908ab38dcc73ea606d : SUCCESS in 1h 04m 36s\n- tempest-integrated-object-storage https://zuul.opendev.org/t/openstack/build/43ba59a07e654b3ca84febb71a4516b8 : SUCCESS in 51m 11s\n- openstacksdk-functional-devstack https://zuul.opendev.org/t/openstack/build/66a0de97a92240319e0847316e7ad094 : SUCCESS in 1h 04m 28s\n- swift-tox-func-py39-centos-9-stream-fips https://zuul.opendev.org/t/openstack/build/9aea42f32b0049339d7b35fc1f62b8a7 : SUCCESS in 20m 27s (non-voting)\n- swift-tox-func-encryption-py39-centos-9-stream-fips https://zuul.opendev.org/t/openstack/build/147e7391152b4a33bcdfbf5dc492b892 : SUCCESS in 15m 21s (non-voting)\n- swift-tox-func-ec-py39-centos-9-stream-fips https://zuul.opendev.org/t/openstack/build/8ed8cd8c5c9641829070d6185e521088 : SUCCESS in 22m 40s (non-voting)\n- swift-build-image https://zuul.opendev.org/t/openstack/build/d2d4618ccec743f1943c7fd9b5a41060 : SUCCESS in 7m 16s (non-voting)\n- swift-tox-py37 https://zuul.opendev.org/t/openstack/build/c02eada431254b75aae9b40036d2daf4 : FAILURE in 26m 38s\n- swift-tox-py39 https://zuul.opendev.org/t/openstack/build/0dc991dfa65f48ba990102d2af7ba947 : SUCCESS in 22m 42s\n- swift-tox-py312 https://zuul.opendev.org/t/openstack/build/8a8a537e92094459848c540f3f8d4bfa : SUCCESS in 29m 15s\n- swift-tox-py313 https://zuul.opendev.org/t/openstack/build/714d0f3aada44e3bb34f4a562f08813c : SUCCESS in 32m 23s\n- swift-tox-func-py312 https://zuul.opendev.org/t/openstack/build/4b44c6d013a44079a98b644ce946aa9a : SUCCESS in 12m 57s\n- swift-tox-func-encryption-py312 https://zuul.opendev.org/t/openstack/build/77db3a5c371d463a9cfeced81aec9624 : SUCCESS in 13m 36s\n- swift-tox-func-ec-py312 https://zuul.opendev.org/t/openstack/build/2fc3457825f645d791e77912b0c2ac16 : SUCCESS in 17m 17s\n- swift-func-cors https://zuul.opendev.org/t/openstack/build/f859a472f9274ba8918f667c6bec4463 : SUCCESS in 8m 47s\n- swift-tox-func-ceph-s3tests-tempauth https://zuul.opendev.org/t/openstack/build/a4a84b60e70748ca81c5df56301b99e7 : SUCCESS in 20m 25s (non-voting)\n- swift-tox-func-s3api-compat-tests-tempauth https://zuul.opendev.org/t/openstack/build/03ef98459004472495623ed65cbb9d20 : SUCCESS in 7m 49s\n- swift-probetests-centos-9-stream https://zuul.opendev.org/t/openstack/build/46324d9b048b49c4992460fe5540856f : SUCCESS in 59m 15s\n- swift-dsvm-functional https://zuul.opendev.org/t/openstack/build/4a8d4aa4672b41a78b8e5189f40b7eb0 : SUCCESS in 49m 52s\n- swift-dsvm-functional-ipv6 https://zuul.opendev.org/t/openstack/build/a95a8bbf962f4ab09fc2c7d07cb5ae54 : SUCCESS in 53m 25s\n- swift-tox-lower-constraints https://zuul.opendev.org/t/openstack/build/c988bab9cbb2412ebc529196e3bcfd01 : TIMED_OUT in 40m 44s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/85d43a0ffd804f83ae8d4a949d49565f : FAILURE in 3m 54s\n- swift-multinode-rolling-upgrade https://zuul.opendev.org/t/openstack/build/367c3ce0260347c3994ac0a27aa21949 : FAILURE in 7m 05s\n- tempest-integrated-object-storage-ubuntu-jammy https://zuul.opendev.org/t/openstack/build/885e9cf431474cb7a93b34c77edd396f : SUCCESS in 50m 57s\n- tempest-ipv6-only https://zuul.opendev.org/t/openstack/build/416793777e4b4d7dbbcb12475c914e68 : SUCCESS in 41m 50s","accounts_in_message":[],"_revision_number":12},{"id":"c36825ffd38a7ab017dc1d17b021c23fab817a6a","tag":"autogenerated:zuul:check-arm64","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2025-09-15 08:57:23.000000000","message":"Patch Set 12:\n\nBuild succeeded (ARM64 pipeline).\nhttps://zuul.opendev.org/t/openstack/buildset/a795cfde7bfd4b92a6f29c8b30c7b725\n\n- swift-tox-py312-arm64 https://zuul.opendev.org/t/openstack/build/9201cb9c9c104052bad12ea929d52787 : SUCCESS in 1h 19m 00s","accounts_in_message":[],"_revision_number":12},{"id":"c283e8bb9bb26d805ea2a5bc76a2603b243c9a5c","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"date":"2025-09-19 03:58:47.000000000","message":"Uploaded patch set 13.\n\nOutdated Votes:\n* Verified-1\n","accounts_in_message":[],"_revision_number":13},{"id":"148b78b5229c9d61520035095175951a7e800357","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2025-09-19 05:00:13.000000000","message":"Patch Set 13: Verified-1\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttps://docs.opendev.org/opendev/infra-manual/latest/developers.html#automated-testing\nand https://docs.openstack.org/project-team-guide/testing.html#how-to-handle-test-failures\n\nhttps://zuul.opendev.org/t/openstack/buildset/d0a3357c37d74e5b8e95c35e75ddd0e5\n\n- test-release-openstack https://zuul.opendev.org/t/openstack/build/60fd3ea380394c87b34609d5f93ee9d2 : SUCCESS in 2m 28s\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/53e83c3796314e74a68c8b82063492aa : SUCCESS in 10m 32s\n- requirements-check https://zuul.opendev.org/t/openstack/build/f037faffe41e483c8d4bcf914a878cd1 : FAILURE in 1m 53s\n- grenade https://zuul.opendev.org/t/openstack/build/a0ede39e72a54fe4a767dee2369229c7 : SUCCESS in 58m 55s\n- grenade-skip-level-always https://zuul.opendev.org/t/openstack/build/580b552e424d4a498b306c634040286c : SUCCESS in 1h 00m 31s\n- tempest-integrated-object-storage https://zuul.opendev.org/t/openstack/build/9aa63b3149e64291a9ed7cd2811639cb : SUCCESS in 58m 25s\n- openstacksdk-functional-devstack https://zuul.opendev.org/t/openstack/build/b180799320604193ae325afdc6f96824 : SUCCESS in 57m 31s\n- swift-tox-func-py39-centos-9-stream-fips https://zuul.opendev.org/t/openstack/build/9ea30a1e02484fda97ef3b78301d9bc4 : SUCCESS in 20m 35s (non-voting)\n- swift-tox-func-encryption-py39-centos-9-stream-fips https://zuul.opendev.org/t/openstack/build/c1dd482396fc4b1e914fe3ee0fdbb917 : SUCCESS in 12m 13s (non-voting)\n- swift-tox-func-ec-py39-centos-9-stream-fips https://zuul.opendev.org/t/openstack/build/fc002de3f4cd4e55a0d4b505ca075452 : SUCCESS in 20m 01s (non-voting)\n- swift-build-image https://zuul.opendev.org/t/openstack/build/f6e4994f14b94523b52fccdc88c4d3d2 : SUCCESS in 6m 51s (non-voting)\n- swift-tox-py37 https://zuul.opendev.org/t/openstack/build/b8a55670bfbb4fb1b865d312513a9f3b : SUCCESS in 12m 45s\n- swift-tox-py39 https://zuul.opendev.org/t/openstack/build/3119f6c9d9004209b50725972e900cbc : SUCCESS in 15m 58s\n- swift-tox-py312 https://zuul.opendev.org/t/openstack/build/a27c0118ca974be7b3583bd34c7e339a : SUCCESS in 11m 13s\n- swift-tox-py313 https://zuul.opendev.org/t/openstack/build/7a72014622af45989ccf545878895d61 : SUCCESS in 29m 12s\n- swift-tox-func-py312 https://zuul.opendev.org/t/openstack/build/43ec5520ec7942f682a83493e916f4a1 : SUCCESS in 8m 26s\n- swift-tox-func-encryption-py312 https://zuul.opendev.org/t/openstack/build/1b5063bbdc054154b57fc9fd95c2c2e1 : SUCCESS in 15m 46s\n- swift-tox-func-ec-py312 https://zuul.opendev.org/t/openstack/build/6728c77cc56745cb8cd3502ffa419d9d : SUCCESS in 15m 49s\n- swift-func-cors https://zuul.opendev.org/t/openstack/build/015e86b924614d2da85d39a59d991302 : SUCCESS in 6m 44s\n- swift-tox-func-ceph-s3tests-tempauth https://zuul.opendev.org/t/openstack/build/cfe9e0719db34b289d57a7522f30106f : SUCCESS in 14m 20s (non-voting)\n- swift-tox-func-s3api-compat-tests-tempauth https://zuul.opendev.org/t/openstack/build/8e143fe55b364c8b94705c9ed6e7d341 : SUCCESS in 5m 12s\n- swift-probetests-centos-9-stream https://zuul.opendev.org/t/openstack/build/ad905b09b02446ee887aa435c682a06e : SUCCESS in 55m 34s\n- swift-dsvm-functional https://zuul.opendev.org/t/openstack/build/432538f7f45949ebaaf79238259dc107 : SUCCESS in 55m 32s\n- swift-dsvm-functional-ipv6 https://zuul.opendev.org/t/openstack/build/1397ee708f634e3490b6209d23fcc653 : SUCCESS in 23m 14s\n- swift-tox-lower-constraints https://zuul.opendev.org/t/openstack/build/94e6dff379cb44199e3d6733d871a7dd : TIMED_OUT in 40m 51s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/b036bdaf44ce49eda327a649645c717a : SUCCESS in 3m 11s\n- swift-multinode-rolling-upgrade https://zuul.opendev.org/t/openstack/build/09a30e91d7964b1d820e8806d99361d8 : FAILURE in 10m 18s\n- tempest-integrated-object-storage-ubuntu-jammy https://zuul.opendev.org/t/openstack/build/313d08fe8f4744c0a4371e7405558e7e : SUCCESS in 49m 25s\n- tempest-ipv6-only https://zuul.opendev.org/t/openstack/build/2c297ae5228641f89394f27de7fca9ff : SUCCESS in 28m 58s","accounts_in_message":[],"_revision_number":13},{"id":"921c6c1f4d674a663f6ebc4259ada9ae16615281","tag":"autogenerated:zuul:check-arm64","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2025-09-19 05:07:39.000000000","message":"Patch Set 13:\n\nBuild succeeded (ARM64 pipeline).\nhttps://zuul.opendev.org/t/openstack/buildset/d17eb6ec5cd046bd9cc437b1c4304918\n\n- swift-tox-py312-arm64 https://zuul.opendev.org/t/openstack/build/7523372520754fd298f350113d9dbd99 : SUCCESS in 1h 04m 25s","accounts_in_message":[],"_revision_number":13},{"id":"363171f14cb9ab9286c43a6e3cc875744898304d","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"date":"2026-02-02 03:39:35.000000000","message":"Uploaded patch set 14.\n\nOutdated Votes:\n* Verified-1\n","accounts_in_message":[],"_revision_number":14},{"id":"491fc290af0c7dde00c71218528f9397d8893c7b","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2026-02-02 04:47:38.000000000","message":"Patch Set 14: Verified-1\n\n(7 comments)\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttps://docs.opendev.org/opendev/infra-manual/latest/developers.html#automated-testing\nand https://docs.openstack.org/project-team-guide/testing.html#how-to-handle-test-failures\n\nhttps://zuul.opendev.org/t/openstack/buildset/a289ce798f784cdc81b4cfe3186b86d2\n\n- test-release-openstack https://zuul.opendev.org/t/openstack/build/ecba72619d0444e6a5b06e21da9bd191 : SUCCESS in 2m 28s\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/dc814865669744db808853bfeaff028f : SUCCESS in 8m 02s\n- requirements-check https://zuul.opendev.org/t/openstack/build/a8e84220bd8d426ab832cbc1e457ae4e : FAILURE in 3m 36s\n- grenade https://zuul.opendev.org/t/openstack/build/7da95df9f7fe4fa280554bbd8da855a9 : SUCCESS in 29m 23s\n- grenade-skip-level-always https://zuul.opendev.org/t/openstack/build/8d4375082d7146609eac0f491ebd72e5 : SUCCESS in 45m 23s\n- tempest-integrated-object-storage https://zuul.opendev.org/t/openstack/build/0e81aef7ee0840af90e93c0ec3261881 : SUCCESS in 54m 43s\n- openstacksdk-functional-devstack https://zuul.opendev.org/t/openstack/build/a0ddd40b2a5b43c98368479e67368ceb : SUCCESS in 1h 02m 49s\n- swift-tox-func-py39-centos-9-stream-fips https://zuul.opendev.org/t/openstack/build/cd6d000366b847c1ac2d3feb55f9907d : SUCCESS in 10m 09s (non-voting)\n- swift-tox-func-encryption-py39-centos-9-stream-fips https://zuul.opendev.org/t/openstack/build/285c3e44f21f49efb28be53d0404f719 : SUCCESS in 19m 31s (non-voting)\n- swift-tox-func-ec-py39-centos-9-stream-fips https://zuul.opendev.org/t/openstack/build/5bb8f019a6d842a9909c76393cc2418c : SUCCESS in 10m 50s (non-voting)\n- swift-build-image https://zuul.opendev.org/t/openstack/build/933424ab0d4942b2963cc75517237c7f : SUCCESS in 7m 09s (non-voting)\n- swift-tox-py37 https://zuul.opendev.org/t/openstack/build/9578528c37a64df39893ad36b454894f : SUCCESS in 25m 59s\n- swift-tox-py39 https://zuul.opendev.org/t/openstack/build/71586b8503774a738c38009cf1cd3986 : SUCCESS in 21m 21s\n- swift-tox-py312 https://zuul.opendev.org/t/openstack/build/77780ee824954e7ab2546ed4360d207c : SUCCESS in 21m 26s\n- swift-tox-py313 https://zuul.opendev.org/t/openstack/build/99557e8e4f534bb0a95347b18411ca4b : SUCCESS in 32m 12s\n- swift-tox-func-py312 https://zuul.opendev.org/t/openstack/build/6d190de7d9cf42459713549d9b5d43cb : SUCCESS in 15m 28s\n- swift-tox-func-encryption-py312 https://zuul.opendev.org/t/openstack/build/4548d25671b64b8c89567de5ac342f63 : SUCCESS in 15m 23s\n- swift-tox-func-ec-py312 https://zuul.opendev.org/t/openstack/build/9147566a984d4843a0fbab8a4d84fae3 : SUCCESS in 16m 43s\n- swift-func-cors https://zuul.opendev.org/t/openstack/build/cb1b9f2a14044fa2bfa4b515ade84bb1 : SUCCESS in 6m 36s\n- swift-tox-func-ceph-s3tests-tempauth https://zuul.opendev.org/t/openstack/build/0a31a08548f2445b929cb8d10d5994ca : SUCCESS in 7m 58s (non-voting)\n- swift-tox-func-s3api-compat-tests-tempauth https://zuul.opendev.org/t/openstack/build/33397440001245939a7b8cdce7dcad55 : SUCCESS in 9m 31s\n- swift-probetests-centos-9-stream https://zuul.opendev.org/t/openstack/build/f5e4b20cdb31427daadd61a1bff662f6 : SUCCESS in 37m 57s\n- swift-dsvm-functional https://zuul.opendev.org/t/openstack/build/b385437bfb6449c6bd6c50c3c89cae2e : SUCCESS in 21m 57s\n- swift-dsvm-functional-ipv6 https://zuul.opendev.org/t/openstack/build/394d7311597b417399651dea1ecf539a : SUCCESS in 56m 47s\n- swift-tox-lower-constraints https://zuul.opendev.org/t/openstack/build/59eebc7294bf4443b0722f461156e2c0 : TIMED_OUT in 40m 55s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/e139da521c8841a89cc8719ddfe6f4e1 : FAILURE in 4m 21s\n- swift-multinode-rolling-upgrade https://zuul.opendev.org/t/openstack/build/ad58b59644074022af166ba65c2305e7 : FAILURE in 7m 30s\n- tempest-integrated-object-storage-ubuntu-jammy https://zuul.opendev.org/t/openstack/build/ea7138007094409c91e41068356e4094 : SUCCESS in 52m 33s\n- tempest-ipv6-only https://zuul.opendev.org/t/openstack/build/5fa82d3bed754524bfc1794322a51c67 : SUCCESS in 47m 44s","accounts_in_message":[],"_revision_number":14},{"id":"99188d4d54884a409dd057bcd8de9a29a7e29d35","tag":"autogenerated:zuul:check-arm64","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2026-02-02 05:43:13.000000000","message":"Patch Set 14:\n\nBuild succeeded (ARM64 pipeline).\nhttps://zuul.opendev.org/t/openstack/buildset/232d6273477a4da082c622d49cf8207a\n\n- swift-tox-py312-arm64 https://zuul.opendev.org/t/openstack/build/1d26d7da36cf4c0e9a4bcece4750961e : SUCCESS in 58m 33s","accounts_in_message":[],"_revision_number":14},{"id":"a5ced192aba94fd1e89a99e8ec32ea8bf3f75036","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"date":"2026-05-05 07:49:45.000000000","message":"Uploaded patch set 15.\n\nOutdated Votes:\n* Verified-1 (copy condition: \"NEVER\")\n","accounts_in_message":[],"_revision_number":15},{"id":"a514b3604e798fe4b86c35adc38af463adadf62d","tag":"autogenerated:zuul:check-arm64","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2026-05-05 08:23:00.000000000","message":"Patch Set 15:\n\nBuild succeeded (ARM64 pipeline).\nhttps://zuul.opendev.org/t/openstack/buildset/8b546906447045fd8aca9b487b5d7a04\n\n- swift-tox-py312-arm64 https://zuul.opendev.org/t/openstack/build/528b1514648f46d2bf5e19110708af88 : SUCCESS in 31m 24s","accounts_in_message":[],"_revision_number":15},{"id":"e408881dc7c41a21ce9c6cb83d68ae274f4a0e09","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2026-05-05 09:05:50.000000000","message":"Patch Set 15: Verified-1\n\n(8 comments)\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttps://docs.opendev.org/opendev/infra-manual/latest/developers.html#automated-testing\nand https://docs.openstack.org/project-team-guide/testing.html#how-to-handle-test-failures\n\nhttps://zuul.opendev.org/t/openstack/buildset/59d61ae7c9c9496db53ea5e7b2af5af6\n\n- test-release-openstack https://zuul.opendev.org/t/openstack/build/0c3a6d0084da4dc68cfe987d74068955 : SUCCESS in 3m 01s\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/a8e87e17e0aa474e9905e3954e0e3c30 : SUCCESS in 11m 58s\n- requirements-check https://zuul.opendev.org/t/openstack/build/7f3f20222a234c4caf635431c26c15d0 : FAILURE in 2m 55s\n- grenade https://zuul.opendev.org/t/openstack/build/09e356d855944ae3b43891599898f41e : SUCCESS in 1h 08m 36s\n- grenade-skip-level-always https://zuul.opendev.org/t/openstack/build/1ddeb230e9f3469283f5b34f0f8869e2 : SUCCESS in 31m 42s\n- tempest-integrated-object-storage https://zuul.opendev.org/t/openstack/build/187a5a9d974c453a8ca428772a5f54e4 : SUCCESS in 31m 43s\n- openstacksdk-functional-devstack https://zuul.opendev.org/t/openstack/build/f83ae935fa2f43f4a2c181724a057c1f : SUCCESS in 1h 09m 00s\n- swift-tox-func-py39-centos-9-stream-fips https://zuul.opendev.org/t/openstack/build/b1c1b791a151440fadc440c5e59f0991 : SUCCESS in 19m 54s (non-voting)\n- swift-tox-func-encryption-py39-centos-9-stream-fips https://zuul.opendev.org/t/openstack/build/b34f935585ba41e18725783637af7348 : SUCCESS in 20m 49s (non-voting)\n- swift-tox-func-ec-py39-centos-9-stream-fips https://zuul.opendev.org/t/openstack/build/162d474b395e4592921e3f4dfa97c20e : SUCCESS in 18m 02s (non-voting)\n- swift-build-image https://zuul.opendev.org/t/openstack/build/f5d20a5149834cd8ae182d324eef8704 : SUCCESS in 8m 59s (non-voting)\n- swift-tox-py37 https://zuul.opendev.org/t/openstack/build/fa2d78eb62464577b1481d38e8e97e52 : SUCCESS in 18m 20s\n- swift-tox-py310 https://zuul.opendev.org/t/openstack/build/464ba30a3d8c47368c1e28d68e5a3ffa : SUCCESS in 22m 32s\n- swift-tox-py313 https://zuul.opendev.org/t/openstack/build/7fefbd1093a844a286d90d8cde14e784 : SUCCESS in 32m 50s\n- swift-tox-py314 https://zuul.opendev.org/t/openstack/build/868c743ed242414aa45e5d4c503862da : SUCCESS in 20m 32s\n- swift-tox-func-py312 https://zuul.opendev.org/t/openstack/build/f9c8f817098b4218b42457be121daae1 : SUCCESS in 15m 28s\n- swift-tox-func-encryption-py312 https://zuul.opendev.org/t/openstack/build/6135f262cb484e4ea032da959f24d8fe : SUCCESS in 14m 58s\n- swift-tox-func-ec-py312 https://zuul.opendev.org/t/openstack/build/8923c5a645eb497b93910f744e2a7ec3 : SUCCESS in 16m 57s\n- swift-func-cors https://zuul.opendev.org/t/openstack/build/67f5efbaf3f3423ab3e073349e325584 : SUCCESS in 8m 27s\n- swift-tox-func-ceph-s3tests-tempauth https://zuul.opendev.org/t/openstack/build/6daa85759a904bfea797fb9e2396c904 : SUCCESS in 18m 43s (non-voting)\n- swift-tox-func-s3api-compat-tests-tempauth https://zuul.opendev.org/t/openstack/build/164e35783deb41069862813ad2325b2a : SUCCESS in 9m 17s\n- swift-probetests-centos-9-stream https://zuul.opendev.org/t/openstack/build/7a4e8a4ca4b2425baf3730a08161f172 : SUCCESS in 1h 09m 42s\n- swift-dsvm-functional https://zuul.opendev.org/t/openstack/build/ddd885043dfc47838dbb40fcd4358073 : SUCCESS in 57m 45s\n- swift-dsvm-functional-ipv6 https://zuul.opendev.org/t/openstack/build/1746df4d82ef46858442319367e14a74 : SUCCESS in 55m 33s\n- swift-tox-lower-constraints https://zuul.opendev.org/t/openstack/build/106ff0fe741b45ddb005e8074387cf61 : TIMED_OUT in 40m 46s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/ea2e735c94c74a4398f36f1837e48abc : FAILURE in 4m 46s\n- swift-multinode-rolling-upgrade https://zuul.opendev.org/t/openstack/build/a83729a153224bd6bc04d525c66427c6 : FAILURE in 8m 27s\n- tempest-ipv6-only https://zuul.opendev.org/t/openstack/build/c50e93613be14a52bcdfb0b44f583b3c : SUCCESS in 1h 05m 30s","accounts_in_message":[],"_revision_number":15},{"id":"617bc05b2e8063243983a5d1508ee1124f303796","author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"date":"2026-05-14 00:07:43.000000000","message":"Patch Set 15:\n\n(1 comment)","accounts_in_message":[],"_revision_number":15},{"id":"31847ef5161cc85e20eeacf75587e03e08f91cd7","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2026-05-14 00:07:50.000000000","message":"Patch Set 15:\n\nMerge Failed.\n\nThis change or one of its cross-repo dependencies was unable to be automatically merged with the current state of its repository. Please rebase the change and upload a new patchset.\nWarning:\n  Error merging gerrit/openstack/swift for 899397,15","accounts_in_message":[],"_revision_number":15},{"id":"667b282050422d4202ef63fd7112a4cbfca53def","tag":"autogenerated:zuul:check-arm64","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2026-05-14 00:07:50.000000000","message":"Patch Set 15:\n\nMerge Failed.\n\nThis change or one of its cross-repo dependencies was unable to be automatically merged with the current state of its repository. Please rebase the change and upload a new patchset.\nWarning:\n  Error merging gerrit/openstack/swift for 899397,15","accounts_in_message":[],"_revision_number":15},{"id":"a60f805f8afa5c13f526dd943c2933c00ed7c562","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"date":"2026-06-03 06:07:00.000000000","message":"Uploaded patch set 16.\n\nOutdated Votes:\n* Verified-1 (copy condition: \"NEVER\")\n","accounts_in_message":[],"_revision_number":16},{"id":"2b2f29c374a81e09da7fcb73ca628da2223b817b","tag":"autogenerated:zuul:check","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2026-06-03 07:35:41.000000000","message":"Patch Set 16: Verified-1\n\n(8 comments)\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttps://docs.opendev.org/opendev/infra-manual/latest/developers.html#automated-testing\nand https://docs.openstack.org/project-team-guide/testing.html#how-to-handle-test-failures\n\nhttps://zuul.opendev.org/t/openstack/buildset/9636f7b1eb664c32bd69d6f3a166284e\n\n- test-release-openstack https://zuul.opendev.org/t/openstack/build/02407b0557de4fb2be0f458c78b2fe03 : SUCCESS in 4m 08s\n- openstack-tox-docs https://zuul.opendev.org/t/openstack/build/23df8a2d371e484382e50329b172f1f4 : SUCCESS in 10m 13s\n- requirements-check https://zuul.opendev.org/t/openstack/build/bba2ce9e84ad4045a784ae8a1a091148 : FAILURE in 2m 41s\n- build-openstack-releasenotes https://zuul.opendev.org/t/openstack/build/82180dc031104840a7c6ea8c80dc8a8b : SUCCESS in 9m 49s\n- grenade https://zuul.opendev.org/t/openstack/build/b8d7b9f19aa04328bd634d128cd8005b : SUCCESS in 1h 12m 43s\n- grenade-skip-level-always https://zuul.opendev.org/t/openstack/build/b1fddc9479d04b5da216e4496155c091 : SUCCESS in 1h 22m 29s\n- tempest-integrated-object-storage https://zuul.opendev.org/t/openstack/build/3f6d0867a83b41089d56987eddbb8169 : SUCCESS in 59m 47s\n- openstacksdk-functional-devstack https://zuul.opendev.org/t/openstack/build/567507ebaacc4e50bf6e36b9ca6acaa4 : SUCCESS in 39m 35s\n- swift-tox-func-py39-centos-9-stream-fips https://zuul.opendev.org/t/openstack/build/357a981b75e04248b22015375f9c6c3f : SUCCESS in 20m 49s (non-voting)\n- swift-tox-func-encryption-py39-centos-9-stream-fips https://zuul.opendev.org/t/openstack/build/c6ddf818cd82498c9c8f37d92246a9e9 : SUCCESS in 20m 07s (non-voting)\n- swift-tox-func-ec-py39-centos-9-stream-fips https://zuul.opendev.org/t/openstack/build/d9d0f335c7db42b79fc3796e11b4eec4 : SUCCESS in 23m 51s (non-voting)\n- swift-build-image https://zuul.opendev.org/t/openstack/build/983aa0dfa378497b9e8b2feec7ac1d90 : SUCCESS in 9m 56s (non-voting)\n- swift-tox-py37 https://zuul.opendev.org/t/openstack/build/cec68e9ba740474199d4385736552751 : SUCCESS in 30m 02s\n- swift-tox-py310 https://zuul.opendev.org/t/openstack/build/6e7d0c3fbbdb4ed48abb15dd8febd0b2 : SUCCESS in 23m 18s\n- swift-tox-py313 https://zuul.opendev.org/t/openstack/build/6d15a8b404654f45ba61ab65213b86e1 : SUCCESS in 31m 56s\n- swift-tox-py314 https://zuul.opendev.org/t/openstack/build/f06171ef6ee947f4ba2609589bfcff70 : SUCCESS in 19m 29s\n- swift-tox-func-py312 https://zuul.opendev.org/t/openstack/build/51936359545d49c9a1531558818c38e6 : SUCCESS in 15m 53s\n- swift-tox-func-encryption-py312 https://zuul.opendev.org/t/openstack/build/fffa8ce968e84ece847564895da1da07 : SUCCESS in 9m 44s\n- swift-tox-func-ec-py312 https://zuul.opendev.org/t/openstack/build/2b5765b74cd14698a5d095d42b3ca4c3 : SUCCESS in 17m 12s\n- swift-func-cors https://zuul.opendev.org/t/openstack/build/efe5e214a8d04b4195c6512a6c3fdcde : SUCCESS in 6m 42s\n- swift-tox-func-ceph-s3tests-tempauth https://zuul.opendev.org/t/openstack/build/131a60eb8f5d422a9cb1b398ea3ed92d : SUCCESS in 19m 02s (non-voting)\n- swift-tox-func-s3api-compat-tests-tempauth https://zuul.opendev.org/t/openstack/build/493403a831764692a9e73bf5d03d92ac : SUCCESS in 8m 40s\n- swift-probetests-centos-9-stream https://zuul.opendev.org/t/openstack/build/34450bc478be4fbfa9e02ad9718a2adb : SUCCESS in 38m 23s\n- swift-dsvm-functional https://zuul.opendev.org/t/openstack/build/58a34c757e52412dae7217bd2ff925e8 : SUCCESS in 53m 40s\n- swift-dsvm-functional-ipv6 https://zuul.opendev.org/t/openstack/build/9a7664c90a774209b6f8f586e5c1c7fd : SUCCESS in 54m 55s\n- swift-tox-lower-constraints https://zuul.opendev.org/t/openstack/build/6df2bbb97a014fc9bea25e584c43cf0d : FAILURE in 6m 17s\n- openstack-tox-pep8 https://zuul.opendev.org/t/openstack/build/8baf26bbc8454baca2943b247d6ac2dd : FAILURE in 4m 26s\n- swift-multinode-rolling-upgrade https://zuul.opendev.org/t/openstack/build/7e736996f5194e189f213265d474494d : FAILURE in 8m 05s\n- tempest-ipv6-only https://zuul.opendev.org/t/openstack/build/58cd1d41e43c4f3eaa33abe4c83a1daa : SUCCESS in 1h 11m 01s","accounts_in_message":[],"_revision_number":16},{"id":"c2a280d71d7a80cf00b31a8d1f5f03ae8bc0f7d2","tag":"autogenerated:zuul:check-arm64","author":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]},"date":"2026-06-03 11:50:36.000000000","message":"Patch Set 16:\n\nBuild succeeded (ARM64 pipeline).\nhttps://zuul.opendev.org/t/openstack/buildset/abafbbf0868b46148f73b6a59de1275f\n\n- swift-tox-py312-arm64 https://zuul.opendev.org/t/openstack/build/6428047e6e294fc2b21fb1a0b5104d67 : SUCCESS in 1h 15m 02s","accounts_in_message":[],"_revision_number":16}],"current_revision_number":16,"current_revision":"ac655f34c65608b6a4c9371533133a31f2b6f52c","revisions":{"e3045d1f17b2894c9a7cc6be3f6e40c7c944aa43":{"kind":"REWORK","_number":1,"created":"2023-10-26 13:06:32.000000000","uploader":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"ref":"refs/changes/97/899397/1","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/swift","ref":"refs/changes/97/899397/1","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/swift refs/changes/97/899397/1 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/swift refs/changes/97/899397/1 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/swift refs/changes/97/899397/1 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/swift refs/changes/97/899397/1"}}},"commit":{"parents":[{"commit":"195f54ecb5c7520b7ae19157be77047a0f4e68ea","subject":"Merge \"Add a swift-reload command\"","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/195f54ecb5c7520b7ae19157be77047a0f4e68ea"}]}],"author":{"name":"Matthew Oliver","email":"matt@oliver.net.au","date":"2021-06-24 07:34:28.000000000","tz":600},"committer":{"name":"Matthew Oliver","email":"matt@oliver.net.au","date":"2023-10-26 13:05:04.000000000","tz":660},"subject":"request tracing: Add Otel request tracing support (decorator)","message":"request tracing: Add Otel request tracing support (decorator)\n\nWe\u0027ve been wanting some micro benchmarks inside things like the proxy\nserver in order to test and find bottle necks etc. Also if would be\nawesome if we could support handing out a trac_sig/key to a user and if\nthey add this to their requiest extra tracing data could be dumped.\n\nBasic idea is we will be able to provide trace_sig which will be a HMAC from\nsome trace key and expiry time. If it passes it\u0027ll set TRACE_ACTIVATED_KEY\ninto environ, triggering the activation of tracing that request through the swift cluster.\n\nTracing is handled by a RequestTraceMiddleware wsgi mddleware.\nThis middleware is used to enable request tracing.\nIt supports tracing by giving a valid hmac signature provided in the\nquery string and/or it introduces another option which will trace every x\nrequest. This can be handy if you want to insert the middleware into\nsomething like the internal client config.\n\nThe request will start and stop tracing as the request hits this\nmiddleware. As such, to get as much info as possible it\u0027s best to have it\nas close to the front of the pipeline as possible.\n\nThe proxy has been instrumented so when making calls to storage nodes,\nso long as the middleware is also inserted into the storage node\npipelines, the tracing context is injected into the sub requests allowing\nus to trace a single request throughout the cluster.\n\nRequest tracing utilises the OpenTelemetry framework, which has now\nbecome a swift dependency. We could pull this out and warn.\n\nBecause a trace is scoped by the request. There is a trace_data object that\nis used to store some tracing metadata. Namely:\n\n - The call stack (so we can name middleware uniquely)\n - The current tracer object. As it\u0027s this object that creates spans.\n\nTracing and eventlet has issues. So we also make the GreenAsyncPool\ntrace aware, and also created sub-classed GreenPile to\nTraceAwareGreenPile, so it can pass a span (if it exists) into the\neventlet environment, allowing them to be traced correctly.\n\nTrace: be able to force the current span and SLO\n\nWhen we\u0027ve instrumented a response interator, like the SegmentIterater\nused by SLO, by the time the iter is being called by the client the\nrest of the request spans have returned.\nAlthough we force some of the spans to not be ended, the OpTel tracer object\nby the time it gets to a new iteration, it can\u0027t find the current\nrunning span to tie newly created spans off. This generates new traces\nrather the joining the existing trace.\n\nBecause eventlet uses the same process for its pool of threads, we\nstopped using otels global get_current_trace helper, and instead track them in\nthe request env.\nPutting the 2 together, we can optionally force the current span to come\nback into focus before the tracer object creates a new span, thereby\nkeeping the trace connected even when we\u0027re out in a response iter.\n\nTracing is handled by a new class decorator, wsgi_trace, that can be added to a\nmiddleware class. It\u0027ll use it\u0027s own `__call__` method that\nautomatically sets up a span and the tracing env for the middleware then\ncalls the wrapped class `__call__` function.\n\nAll the tracing code is in the swift.common.tracing namespace. And there\nyou\u0027ll find all the helper methods to further instrument code. To just\nname a few:\n\n  - trace_add: add a key value pair to the span.\n  - new_trace_span: a context manager to help create new spans\n  - trace_exception: attach a thrown exception to the span.\n\nAll the helper methods require passing in a wsgi env, this is where the\ntracing object will live if it exists. If tracing ins\u0027t enabled for the\nrequest, it\u0027s just a noop.\n\nChange-Id: I6c3765d43e0521dc0648ccd0956b93306d31e062\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/e3045d1f17b2894c9a7cc6be3f6e40c7c944aa43"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/e3045d1f17b2894c9a7cc6be3f6e40c7c944aa43"}]},"branch":"refs/heads/master"},"0d7441e28721d626dca2415b631ddef99abf5dbd":{"kind":"REWORK","_number":2,"created":"2023-10-26 22:48:23.000000000","uploader":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"ref":"refs/changes/97/899397/2","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/swift","ref":"refs/changes/97/899397/2","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/swift refs/changes/97/899397/2 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/swift refs/changes/97/899397/2 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/swift refs/changes/97/899397/2 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/swift refs/changes/97/899397/2"}}},"commit":{"parents":[{"commit":"195f54ecb5c7520b7ae19157be77047a0f4e68ea","subject":"Merge \"Add a swift-reload command\"","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/195f54ecb5c7520b7ae19157be77047a0f4e68ea"}]}],"author":{"name":"Matthew Oliver","email":"matt@oliver.net.au","date":"2021-06-24 07:34:28.000000000","tz":600},"committer":{"name":"Matthew Oliver","email":"matt@oliver.net.au","date":"2023-10-26 22:47:52.000000000","tz":660},"subject":"request tracing: Add Otel request tracing support (decorator)","message":"request tracing: Add Otel request tracing support (decorator)\n\nWe\u0027ve been wanting some micro benchmarks inside things like the proxy\nserver in order to test and find bottle necks etc. Also if would be\nawesome if we could support handing out a trac_sig/key to a user and if\nthey add this to their requiest extra tracing data could be dumped.\n\nBasic idea is we will be able to provide trace_sig which will be a HMAC from\nsome trace key and expiry time. If it passes it\u0027ll set TRACE_ACTIVATED_KEY\ninto environ, triggering the activation of tracing that request through the swift cluster.\n\nTracing is handled by a RequestTraceMiddleware wsgi mddleware.\nThis middleware is used to enable request tracing.\nIt supports tracing by giving a valid hmac signature provided in the\nquery string and/or it introduces another option which will trace every x\nrequest. This can be handy if you want to insert the middleware into\nsomething like the internal client config.\n\nThe request will start and stop tracing as the request hits this\nmiddleware. As such, to get as much info as possible it\u0027s best to have it\nas close to the front of the pipeline as possible.\n\nThe proxy has been instrumented so when making calls to storage nodes,\nso long as the middleware is also inserted into the storage node\npipelines, the tracing context is injected into the sub requests allowing\nus to trace a single request throughout the cluster.\n\nRequest tracing utilises the OpenTelemetry framework, which has now\nbecome a swift dependency. We could pull this out and warn.\n\nBecause a trace is scoped by the request. There is a trace_data object that\nis used to store some tracing metadata. Namely:\n\n - The call stack (so we can name middleware uniquely)\n - The current tracer object. As it\u0027s this object that creates spans.\n\nTracing and eventlet has issues. So we also make the GreenAsyncPool\ntrace aware, and also created sub-classed GreenPile to\nTraceAwareGreenPile, so it can pass a span (if it exists) into the\neventlet environment, allowing them to be traced correctly.\n\nTrace: be able to force the current span and SLO\n\nWhen we\u0027ve instrumented a response interator, like the SegmentIterater\nused by SLO, by the time the iter is being called by the client the\nrest of the request spans have returned.\nAlthough we force some of the spans to not be ended, the OpTel tracer object\nby the time it gets to a new iteration, it can\u0027t find the current\nrunning span to tie newly created spans off. This generates new traces\nrather the joining the existing trace.\n\nBecause eventlet uses the same process for its pool of threads, we\nstopped using otels global get_current_trace helper, and instead track them in\nthe request env.\nPutting the 2 together, we can optionally force the current span to come\nback into focus before the tracer object creates a new span, thereby\nkeeping the trace connected even when we\u0027re out in a response iter.\n\nTracing is handled by a new class decorator, wsgi_trace, that can be added to a\nmiddleware class. It\u0027ll use it\u0027s own `__call__` method that\nautomatically sets up a span and the tracing env for the middleware then\ncalls the wrapped class `__call__` function.\n\nAll the tracing code is in the swift.common.tracing namespace. And there\nyou\u0027ll find all the helper methods to further instrument code. To just\nname a few:\n\n  - trace_add: add a key value pair to the span.\n  - new_trace_span: a context manager to help create new spans\n  - trace_exception: attach a thrown exception to the span.\n\nAll the helper methods require passing in a wsgi env, this is where the\ntracing object will live if it exists. If tracing ins\u0027t enabled for the\nrequest, it\u0027s just a noop.\n\nChange-Id: I6c3765d43e0521dc0648ccd0956b93306d31e062\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/0d7441e28721d626dca2415b631ddef99abf5dbd"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/0d7441e28721d626dca2415b631ddef99abf5dbd"}]},"branch":"refs/heads/master"},"430ee0a98e5ceb099134786b93b9a496d09a6657":{"kind":"REWORK","_number":3,"created":"2023-10-27 01:09:33.000000000","uploader":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"ref":"refs/changes/97/899397/3","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/swift","ref":"refs/changes/97/899397/3","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/swift refs/changes/97/899397/3 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/swift refs/changes/97/899397/3 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/swift refs/changes/97/899397/3 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/swift refs/changes/97/899397/3"}}},"commit":{"parents":[{"commit":"195f54ecb5c7520b7ae19157be77047a0f4e68ea","subject":"Merge \"Add a swift-reload command\"","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/195f54ecb5c7520b7ae19157be77047a0f4e68ea"}]}],"author":{"name":"Matthew Oliver","email":"matt@oliver.net.au","date":"2021-06-24 07:34:28.000000000","tz":600},"committer":{"name":"Matthew Oliver","email":"matt@oliver.net.au","date":"2023-10-27 01:09:04.000000000","tz":660},"subject":"request tracing: Add Otel request tracing support (decorator)","message":"request tracing: Add Otel request tracing support (decorator)\n\nWe\u0027ve been wanting some micro benchmarks inside things like the proxy\nserver in order to test and find bottle necks etc. Also if would be\nawesome if we could support handing out a trac_sig/key to a user and if\nthey add this to their requiest extra tracing data could be dumped.\n\nBasic idea is we will be able to provide trace_sig which will be a HMAC from\nsome trace key and expiry time. If it passes it\u0027ll set TRACE_ACTIVATED_KEY\ninto environ, triggering the activation of tracing that request through the swift cluster.\n\nTracing is handled by a RequestTraceMiddleware wsgi mddleware.\nThis middleware is used to enable request tracing.\nIt supports tracing by giving a valid hmac signature provided in the\nquery string and/or it introduces another option which will trace every x\nrequest. This can be handy if you want to insert the middleware into\nsomething like the internal client config.\n\nThe request will start and stop tracing as the request hits this\nmiddleware. As such, to get as much info as possible it\u0027s best to have it\nas close to the front of the pipeline as possible.\n\nThe proxy has been instrumented so when making calls to storage nodes,\nso long as the middleware is also inserted into the storage node\npipelines, the tracing context is injected into the sub requests allowing\nus to trace a single request throughout the cluster.\n\nRequest tracing utilises the OpenTelemetry framework, which has now\nbecome a swift dependency. We could pull this out and warn.\n\nBecause a trace is scoped by the request. There is a trace_data object that\nis used to store some tracing metadata. Namely:\n\n - The call stack (so we can name middleware uniquely)\n - The current tracer object. As it\u0027s this object that creates spans.\n\nTracing and eventlet has issues. So we also make the GreenAsyncPool\ntrace aware, and also created sub-classed GreenPile to\nTraceAwareGreenPile, so it can pass a span (if it exists) into the\neventlet environment, allowing them to be traced correctly.\n\nTrace: be able to force the current span and SLO\n\nWhen we\u0027ve instrumented a response interator, like the SegmentIterater\nused by SLO, by the time the iter is being called by the client the\nrest of the request spans have returned.\nAlthough we force some of the spans to not be ended, the OpTel tracer object\nby the time it gets to a new iteration, it can\u0027t find the current\nrunning span to tie newly created spans off. This generates new traces\nrather the joining the existing trace.\n\nBecause eventlet uses the same process for its pool of threads, we\nstopped using otels global get_current_trace helper, and instead track them in\nthe request env.\nPutting the 2 together, we can optionally force the current span to come\nback into focus before the tracer object creates a new span, thereby\nkeeping the trace connected even when we\u0027re out in a response iter.\n\nTracing is handled by a new class decorator, wsgi_trace, that can be added to a\nmiddleware class. It\u0027ll use it\u0027s own `__call__` method that\nautomatically sets up a span and the tracing env for the middleware then\ncalls the wrapped class `__call__` function.\n\nAll the tracing code is in the swift.common.tracing namespace. And there\nyou\u0027ll find all the helper methods to further instrument code. To just\nname a few:\n\n  - trace_add: add a key value pair to the span.\n  - new_trace_span: a context manager to help create new spans\n  - trace_exception: attach a thrown exception to the span.\n\nAll the helper methods require passing in a wsgi env, this is where the\ntracing object will live if it exists. If tracing ins\u0027t enabled for the\nrequest, it\u0027s just a noop.\n\nChange-Id: I6c3765d43e0521dc0648ccd0956b93306d31e062\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/430ee0a98e5ceb099134786b93b9a496d09a6657"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/430ee0a98e5ceb099134786b93b9a496d09a6657"}]},"branch":"refs/heads/master"},"3dd541dccf541166fddb97ec67c0af017592fe81":{"kind":"REWORK","_number":4,"created":"2023-10-27 11:15:58.000000000","uploader":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"ref":"refs/changes/97/899397/4","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/swift","ref":"refs/changes/97/899397/4","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/swift refs/changes/97/899397/4 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/swift refs/changes/97/899397/4 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/swift refs/changes/97/899397/4 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/swift refs/changes/97/899397/4"}}},"commit":{"parents":[{"commit":"195f54ecb5c7520b7ae19157be77047a0f4e68ea","subject":"Merge \"Add a swift-reload command\"","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/195f54ecb5c7520b7ae19157be77047a0f4e68ea"}]}],"author":{"name":"Matthew Oliver","email":"matt@oliver.net.au","date":"2021-06-24 07:34:28.000000000","tz":600},"committer":{"name":"Matthew Oliver","email":"matt@oliver.net.au","date":"2023-10-27 11:14:49.000000000","tz":660},"subject":"request tracing: Add Otel request tracing support (decorator)","message":"request tracing: Add Otel request tracing support (decorator)\n\nWe\u0027ve been wanting some micro benchmarks inside things like the proxy\nserver in order to test and find bottle necks etc. Also if would be\nawesome if we could support handing out a trac_sig/key to a user and if\nthey add this to their requiest extra tracing data could be dumped.\n\nBasic idea is we will be able to provide trace_sig which will be a HMAC from\nsome trace key and expiry time. If it passes it\u0027ll set TRACE_ACTIVATED_KEY\ninto environ, triggering the activation of tracing that request through the swift cluster.\n\nTracing is handled by a RequestTraceMiddleware wsgi mddleware.\nThis middleware is used to enable request tracing.\nIt supports tracing by giving a valid hmac signature provided in the\nquery string and/or it introduces another option which will trace every x\nrequest. This can be handy if you want to insert the middleware into\nsomething like the internal client config.\n\nThe request will start and stop tracing as the request hits this\nmiddleware. As such, to get as much info as possible it\u0027s best to have it\nas close to the front of the pipeline as possible.\n\nThe proxy has been instrumented so when making calls to storage nodes,\nso long as the middleware is also inserted into the storage node\npipelines, the tracing context is injected into the sub requests allowing\nus to trace a single request throughout the cluster.\n\nRequest tracing utilises the OpenTelemetry framework, which has now\nbecome a swift dependency. We could pull this out and warn.\n\nBecause a trace is scoped by the request. There is a trace_data object that\nis used to store some tracing metadata. Namely:\n\n - The call stack (so we can name middleware uniquely)\n - The current tracer object. As it\u0027s this object that creates spans.\n\nTracing and eventlet has issues. So we also make the GreenAsyncPool\ntrace aware, and also created sub-classed GreenPile to\nTraceAwareGreenPile, so it can pass a span (if it exists) into the\neventlet environment, allowing them to be traced correctly.\n\nTrace: be able to force the current span and SLO\n\nWhen we\u0027ve instrumented a response interator, like the SegmentIterater\nused by SLO, by the time the iter is being called by the client the\nrest of the request spans have returned.\nAlthough we force some of the spans to not be ended, the OpTel tracer object\nby the time it gets to a new iteration, it can\u0027t find the current\nrunning span to tie newly created spans off. This generates new traces\nrather the joining the existing trace.\n\nBecause eventlet uses the same process for its pool of threads, we\nstopped using otels global get_current_trace helper, and instead track them in\nthe request env.\nPutting the 2 together, we can optionally force the current span to come\nback into focus before the tracer object creates a new span, thereby\nkeeping the trace connected even when we\u0027re out in a response iter.\n\nTracing is handled by a new class decorator, wsgi_trace, that can be added to a\nmiddleware class. It\u0027ll use it\u0027s own `__call__` method that\nautomatically sets up a span and the tracing env for the middleware then\ncalls the wrapped class `__call__` function.\n\nAll the tracing code is in the swift.common.tracing namespace. And there\nyou\u0027ll find all the helper methods to further instrument code. To just\nname a few:\n\n  - trace_add: add a key value pair to the span.\n  - new_trace_span: a context manager to help create new spans\n  - trace_exception: attach a thrown exception to the span.\n\nAll the helper methods require passing in a wsgi env, this is where the\ntracing object will live if it exists. If tracing ins\u0027t enabled for the\nrequest, it\u0027s just a noop.\n\nChange-Id: I6c3765d43e0521dc0648ccd0956b93306d31e062\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/3dd541dccf541166fddb97ec67c0af017592fe81"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/3dd541dccf541166fddb97ec67c0af017592fe81"}]},"branch":"refs/heads/master"},"dfc5427383195ee05de96bb204db25f85bd537c5":{"kind":"REWORK","_number":5,"created":"2024-03-14 06:00:28.000000000","uploader":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"ref":"refs/changes/97/899397/5","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/swift","ref":"refs/changes/97/899397/5","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/swift refs/changes/97/899397/5 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/swift refs/changes/97/899397/5 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/swift refs/changes/97/899397/5 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/swift refs/changes/97/899397/5"}}},"commit":{"parents":[{"commit":"b6dc24dbc0eca46141544dcdaabe7cdc281c9ce7","subject":"Merge \"s3api test for zero byte mpu\"","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/b6dc24dbc0eca46141544dcdaabe7cdc281c9ce7"}]}],"author":{"name":"Matthew Oliver","email":"matt@oliver.net.au","date":"2021-06-24 07:34:28.000000000","tz":600},"committer":{"name":"Matthew Oliver","email":"matt@oliver.net.au","date":"2024-03-14 05:59:52.000000000","tz":660},"subject":"request tracing: Add Otel request tracing support (decorator)","message":"request tracing: Add Otel request tracing support (decorator)\n\nWe\u0027ve been wanting some micro benchmarks inside things like the proxy\nserver in order to test and find bottle necks etc. Also if would be\nawesome if we could support handing out a trac_sig/key to a user and if\nthey add this to their requiest extra tracing data could be dumped.\n\nBasic idea is we will be able to provide trace_sig which will be a HMAC from\nsome trace key and expiry time. If it passes it\u0027ll set TRACE_ACTIVATED_KEY\ninto environ, triggering the activation of tracing that request through the swift cluster.\n\nTracing is handled by a RequestTraceMiddleware wsgi mddleware.\nThis middleware is used to enable request tracing.\nIt supports tracing by giving a valid hmac signature provided in the\nquery string and/or it introduces another option which will trace every x\nrequest. This can be handy if you want to insert the middleware into\nsomething like the internal client config.\n\nThe request will start and stop tracing as the request hits this\nmiddleware. As such, to get as much info as possible it\u0027s best to have it\nas close to the front of the pipeline as possible.\n\nThe proxy has been instrumented so when making calls to storage nodes,\nso long as the middleware is also inserted into the storage node\npipelines, the tracing context is injected into the sub requests allowing\nus to trace a single request throughout the cluster.\n\nRequest tracing utilises the OpenTelemetry framework, which has now\nbecome a swift dependency. We could pull this out and warn.\n\nBecause a trace is scoped by the request. There is a trace_data object that\nis used to store some tracing metadata. Namely:\n\n - The call stack (so we can name middleware uniquely)\n - The current tracer object. As it\u0027s this object that creates spans.\n\nTracing and eventlet has issues. So we also make the GreenAsyncPool\ntrace aware, and also created sub-classed GreenPile to\nTraceAwareGreenPile, so it can pass a span (if it exists) into the\neventlet environment, allowing them to be traced correctly.\n\nTrace: be able to force the current span and SLO\n\nWhen we\u0027ve instrumented a response interator, like the SegmentIterater\nused by SLO, by the time the iter is being called by the client the\nrest of the request spans have returned.\nAlthough we force some of the spans to not be ended, the OpTel tracer object\nby the time it gets to a new iteration, it can\u0027t find the current\nrunning span to tie newly created spans off. This generates new traces\nrather the joining the existing trace.\n\nBecause eventlet uses the same process for its pool of threads, we\nstopped using otels global get_current_trace helper, and instead track them in\nthe request env.\nPutting the 2 together, we can optionally force the current span to come\nback into focus before the tracer object creates a new span, thereby\nkeeping the trace connected even when we\u0027re out in a response iter.\n\nTracing is handled by a new class decorator, wsgi_trace, that can be added to a\nmiddleware class. It\u0027ll use it\u0027s own `__call__` method that\nautomatically sets up a span and the tracing env for the middleware then\ncalls the wrapped class `__call__` function.\n\nAll the tracing code is in the swift.common.tracing namespace. And there\nyou\u0027ll find all the helper methods to further instrument code. To just\nname a few:\n\n  - trace_add: add a key value pair to the span.\n  - new_trace_span: a context manager to help create new spans\n  - trace_exception: attach a thrown exception to the span.\n\nAll the helper methods require passing in a wsgi env, this is where the\ntracing object will live if it exists. If tracing ins\u0027t enabled for the\nrequest, it\u0027s just a noop.\n\nChange-Id: I6c3765d43e0521dc0648ccd0956b93306d31e062\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/dfc5427383195ee05de96bb204db25f85bd537c5"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/dfc5427383195ee05de96bb204db25f85bd537c5"}]},"branch":"refs/heads/master"},"5f1be04c526d86f2b608f71b7a6f707351e1bb60":{"kind":"REWORK","_number":6,"created":"2024-10-18 03:10:24.000000000","uploader":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"ref":"refs/changes/97/899397/6","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/swift","ref":"refs/changes/97/899397/6","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/swift refs/changes/97/899397/6 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/swift refs/changes/97/899397/6 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/swift refs/changes/97/899397/6 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/swift refs/changes/97/899397/6"}}},"commit":{"parents":[{"commit":"8ded39bccd9192b570e24cf039a3a4c8e3ccc216","subject":"Merge \"logging: Add UPDATE to valid http methods\"","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/8ded39bccd9192b570e24cf039a3a4c8e3ccc216"}]}],"author":{"name":"Matthew Oliver","email":"matt@oliver.net.au","date":"2021-06-24 07:34:28.000000000","tz":600},"committer":{"name":"Matthew Oliver","email":"matt@oliver.net.au","date":"2024-10-18 03:09:49.000000000","tz":660},"subject":"request tracing: Add Otel request tracing support (decorator)","message":"request tracing: Add Otel request tracing support (decorator)\n\nWe\u0027ve been wanting some micro benchmarks inside things like the proxy\nserver in order to test and find bottle necks etc. Also if would be\nawesome if we could support handing out a trac_sig/key to a user and if\nthey add this to their requiest extra tracing data could be dumped.\n\nBasic idea is we will be able to provide trace_sig which will be a HMAC from\nsome trace key and expiry time. If it passes it\u0027ll set TRACE_ACTIVATED_KEY\ninto environ, triggering the activation of tracing that request through the swift cluster.\n\nTracing is handled by a RequestTraceMiddleware wsgi mddleware.\nThis middleware is used to enable request tracing.\nIt supports tracing by giving a valid hmac signature provided in the\nquery string and/or it introduces another option which will trace every x\nrequest. This can be handy if you want to insert the middleware into\nsomething like the internal client config.\n\nThe request will start and stop tracing as the request hits this\nmiddleware. As such, to get as much info as possible it\u0027s best to have it\nas close to the front of the pipeline as possible.\n\nThe proxy has been instrumented so when making calls to storage nodes,\nso long as the middleware is also inserted into the storage node\npipelines, the tracing context is injected into the sub requests allowing\nus to trace a single request throughout the cluster.\n\nRequest tracing utilises the OpenTelemetry framework, which has now\nbecome a swift dependency. We could pull this out and warn.\n\nBecause a trace is scoped by the request. There is a trace_data object that\nis used to store some tracing metadata. Namely:\n\n - The call stack (so we can name middleware uniquely)\n - The current tracer object. As it\u0027s this object that creates spans.\n\nTracing and eventlet has issues. So we also make the GreenAsyncPool\ntrace aware, and also created sub-classed GreenPile to\nTraceAwareGreenPile, so it can pass a span (if it exists) into the\neventlet environment, allowing them to be traced correctly.\n\nTrace: be able to force the current span and SLO\n\nWhen we\u0027ve instrumented a response interator, like the SegmentIterater\nused by SLO, by the time the iter is being called by the client the\nrest of the request spans have returned.\nAlthough we force some of the spans to not be ended, the OpTel tracer object\nby the time it gets to a new iteration, it can\u0027t find the current\nrunning span to tie newly created spans off. This generates new traces\nrather the joining the existing trace.\n\nBecause eventlet uses the same process for its pool of threads, we\nstopped using otels global get_current_trace helper, and instead track them in\nthe request env.\nPutting the 2 together, we can optionally force the current span to come\nback into focus before the tracer object creates a new span, thereby\nkeeping the trace connected even when we\u0027re out in a response iter.\n\nTracing is handled by a new class decorator, wsgi_trace, that can be added to a\nmiddleware class. It\u0027ll use it\u0027s own `__call__` method that\nautomatically sets up a span and the tracing env for the middleware then\ncalls the wrapped class `__call__` function.\n\nAll the tracing code is in the swift.common.tracing namespace. And there\nyou\u0027ll find all the helper methods to further instrument code. To just\nname a few:\n\n  - trace_add: add a key value pair to the span.\n  - new_trace_span: a context manager to help create new spans\n  - trace_exception: attach a thrown exception to the span.\n\nAll the helper methods require passing in a wsgi env, this is where the\ntracing object will live if it exists. If tracing ins\u0027t enabled for the\nrequest, it\u0027s just a noop.\n\nChange-Id: I6c3765d43e0521dc0648ccd0956b93306d31e062\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/5f1be04c526d86f2b608f71b7a6f707351e1bb60"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/5f1be04c526d86f2b608f71b7a6f707351e1bb60"}]},"branch":"refs/heads/master"},"f3d3d1d31500b994972a427337817059208c4d37":{"kind":"REWORK","_number":7,"created":"2025-01-30 05:19:23.000000000","uploader":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"ref":"refs/changes/97/899397/7","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/swift","ref":"refs/changes/97/899397/7","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/swift refs/changes/97/899397/7 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/swift refs/changes/97/899397/7 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/swift refs/changes/97/899397/7 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/swift refs/changes/97/899397/7"}}},"commit":{"parents":[{"commit":"0dfa38d025bcb254849e8061af05e6214b5fdacf","subject":"docs: Fix version call-out for stale_worker_timeout","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/0dfa38d025bcb254849e8061af05e6214b5fdacf"}]}],"author":{"name":"Matthew Oliver","email":"matt@oliver.net.au","date":"2021-06-24 07:34:28.000000000","tz":600},"committer":{"name":"Matthew Oliver","email":"matt@oliver.net.au","date":"2025-01-30 05:18:47.000000000","tz":660},"subject":"request tracing: Add Otel request tracing support (decorator)","message":"request tracing: Add Otel request tracing support (decorator)\n\nWe\u0027ve been wanting some micro benchmarks inside things like the proxy\nserver in order to test and find bottle necks etc. Also if would be\nawesome if we could support handing out a trac_sig/key to a user and if\nthey add this to their requiest extra tracing data could be dumped.\n\nBasic idea is we will be able to provide trace_sig which will be a HMAC from\nsome trace key and expiry time. If it passes it\u0027ll set TRACE_ACTIVATED_KEY\ninto environ, triggering the activation of tracing that request through the swift cluster.\n\nTracing is handled by a RequestTraceMiddleware wsgi mddleware.\nThis middleware is used to enable request tracing.\nIt supports tracing by giving a valid hmac signature provided in the\nquery string and/or it introduces another option which will trace every x\nrequest. This can be handy if you want to insert the middleware into\nsomething like the internal client config.\n\nThe request will start and stop tracing as the request hits this\nmiddleware. As such, to get as much info as possible it\u0027s best to have it\nas close to the front of the pipeline as possible.\n\nThe proxy has been instrumented so when making calls to storage nodes,\nso long as the middleware is also inserted into the storage node\npipelines, the tracing context is injected into the sub requests allowing\nus to trace a single request throughout the cluster.\n\nRequest tracing utilises the OpenTelemetry framework, which has now\nbecome a swift dependency. We could pull this out and warn.\n\nBecause a trace is scoped by the request. There is a trace_data object that\nis used to store some tracing metadata. Namely:\n\n - The call stack (so we can name middleware uniquely)\n - The current tracer object. As it\u0027s this object that creates spans.\n\nTracing and eventlet has issues. So we also make the GreenAsyncPool\ntrace aware, and also created sub-classed GreenPile to\nTraceAwareGreenPile, so it can pass a span (if it exists) into the\neventlet environment, allowing them to be traced correctly.\n\nTrace: be able to force the current span and SLO\n\nWhen we\u0027ve instrumented a response interator, like the SegmentIterater\nused by SLO, by the time the iter is being called by the client the\nrest of the request spans have returned.\nAlthough we force some of the spans to not be ended, the OpTel tracer object\nby the time it gets to a new iteration, it can\u0027t find the current\nrunning span to tie newly created spans off. This generates new traces\nrather the joining the existing trace.\n\nBecause eventlet uses the same process for its pool of threads, we\nstopped using otels global get_current_trace helper, and instead track them in\nthe request env.\nPutting the 2 together, we can optionally force the current span to come\nback into focus before the tracer object creates a new span, thereby\nkeeping the trace connected even when we\u0027re out in a response iter.\n\nTracing is handled by a new class decorator, wsgi_trace, that can be added to a\nmiddleware class. It\u0027ll use it\u0027s own `__call__` method that\nautomatically sets up a span and the tracing env for the middleware then\ncalls the wrapped class `__call__` function.\n\nAll the tracing code is in the swift.common.tracing namespace. And there\nyou\u0027ll find all the helper methods to further instrument code. To just\nname a few:\n\n  - trace_add: add a key value pair to the span.\n  - new_trace_span: a context manager to help create new spans\n  - trace_exception: attach a thrown exception to the span.\n\nAll the helper methods require passing in a wsgi env, this is where the\ntracing object will live if it exists. If tracing ins\u0027t enabled for the\nrequest, it\u0027s just a noop.\n\nChange-Id: I6c3765d43e0521dc0648ccd0956b93306d31e062\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/f3d3d1d31500b994972a427337817059208c4d37"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/f3d3d1d31500b994972a427337817059208c4d37"}]},"branch":"refs/heads/master"},"67c1474d332cfc1f76015af2af561ecd4df1b176":{"kind":"REWORK","_number":8,"created":"2025-05-30 06:40:05.000000000","uploader":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"ref":"refs/changes/97/899397/8","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/swift","ref":"refs/changes/97/899397/8","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/swift refs/changes/97/899397/8 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/swift refs/changes/97/899397/8 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/swift refs/changes/97/899397/8 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/swift refs/changes/97/899397/8"}}},"commit":{"parents":[{"commit":"d0e97650f3d1e832135c3b220d0928f5cd986501","subject":"Merge \"CI: run S3 cross-compat tests as part of DSVM jobs\"","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/d0e97650f3d1e832135c3b220d0928f5cd986501"}]}],"author":{"name":"Matthew Oliver","email":"matt@oliver.net.au","date":"2021-06-24 07:34:28.000000000","tz":600},"committer":{"name":"Matthew Oliver","email":"matt@oliver.net.au","date":"2025-05-29 07:13:58.000000000","tz":600},"subject":"request tracing: Add Otel request tracing support (decorator)","message":"request tracing: Add Otel request tracing support (decorator)\n\nWe\u0027ve been wanting some micro benchmarks inside things like the proxy\nserver in order to test and find bottle necks etc. Also if would be\nawesome if we could support handing out a trac_sig/key to a user and if\nthey add this to their requiest extra tracing data could be dumped.\n\nBasic idea is we will be able to provide trace_sig which will be a HMAC from\nsome trace key and expiry time. If it passes it\u0027ll set TRACE_ACTIVATED_KEY\ninto environ, triggering the activation of tracing that request through the swift cluster.\n\nTracing is handled by a RequestTraceMiddleware wsgi mddleware.\nThis middleware is used to enable request tracing.\nIt supports tracing by giving a valid hmac signature provided in the\nquery string and/or it introduces another option which will trace every x\nrequest. This can be handy if you want to insert the middleware into\nsomething like the internal client config.\n\nThe request will start and stop tracing as the request hits this\nmiddleware. As such, to get as much info as possible it\u0027s best to have it\nas close to the front of the pipeline as possible.\n\nThe proxy has been instrumented so when making calls to storage nodes,\nso long as the middleware is also inserted into the storage node\npipelines, the tracing context is injected into the sub requests allowing\nus to trace a single request throughout the cluster.\n\nRequest tracing utilises the OpenTelemetry framework, which has now\nbecome a swift dependency. We could pull this out and warn.\n\nBecause a trace is scoped by the request. There is a trace_data object that\nis used to store some tracing metadata. Namely:\n\n - The call stack (so we can name middleware uniquely)\n - The current tracer object. As it\u0027s this object that creates spans.\n\nTracing and eventlet has issues. So we also make the GreenAsyncPool\ntrace aware, and also created sub-classed GreenPile to\nTraceAwareGreenPile, so it can pass a span (if it exists) into the\neventlet environment, allowing them to be traced correctly.\n\nTrace: be able to force the current span and SLO\n\nWhen we\u0027ve instrumented a response interator, like the SegmentIterater\nused by SLO, by the time the iter is being called by the client the\nrest of the request spans have returned.\nAlthough we force some of the spans to not be ended, the OpTel tracer object\nby the time it gets to a new iteration, it can\u0027t find the current\nrunning span to tie newly created spans off. This generates new traces\nrather the joining the existing trace.\n\nBecause eventlet uses the same process for its pool of threads, we\nstopped using otels global get_current_trace helper, and instead track them in\nthe request env.\nPutting the 2 together, we can optionally force the current span to come\nback into focus before the tracer object creates a new span, thereby\nkeeping the trace connected even when we\u0027re out in a response iter.\n\nTracing is handled by a new class decorator, wsgi_trace, that can be added to a\nmiddleware class. It\u0027ll use it\u0027s own `__call__` method that\nautomatically sets up a span and the tracing env for the middleware then\ncalls the wrapped class `__call__` function.\n\nAll the tracing code is in the swift.common.tracing namespace. And there\nyou\u0027ll find all the helper methods to further instrument code. To just\nname a few:\n\n  - trace_add: add a key value pair to the span.\n  - new_trace_span: a context manager to help create new spans\n  - trace_exception: attach a thrown exception to the span.\n\nAll the helper methods require passing in a wsgi env, this is where the\ntracing object will live if it exists. If tracing ins\u0027t enabled for the\nrequest, it\u0027s just a noop.\n\nChange-Id: I6c3765d43e0521dc0648ccd0956b93306d31e062\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/67c1474d332cfc1f76015af2af561ecd4df1b176"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/67c1474d332cfc1f76015af2af561ecd4df1b176"}]},"branch":"refs/heads/master"},"fd9124c660bb8614e6bea85260a844fe5f3b9917":{"kind":"TRIVIAL_REBASE_WITH_MESSAGE_UPDATE","_number":9,"created":"2025-07-14 07:25:46.000000000","uploader":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"ref":"refs/changes/97/899397/9","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/swift","ref":"refs/changes/97/899397/9","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/swift refs/changes/97/899397/9 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/swift refs/changes/97/899397/9 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/swift refs/changes/97/899397/9 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/swift refs/changes/97/899397/9"}}},"commit":{"parents":[{"commit":"de579d63d7ac51ee5f31e9f81a54960615f255d4","subject":"Imported Translations from Zanata","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/de579d63d7ac51ee5f31e9f81a54960615f255d4"}]}],"author":{"name":"Matthew Oliver","email":"matt@oliver.net.au","date":"2021-06-24 07:34:28.000000000","tz":600},"committer":{"name":"Matthew Oliver","email":"matt@oliver.net.au","date":"2025-07-14 07:24:52.000000000","tz":600},"subject":"request tracing: Add Otel request tracing support (decorator)","message":"request tracing: Add Otel request tracing support (decorator)\n\nWe\u0027ve been wanting some micro benchmarks inside things like the proxy\nserver in order to test and find bottle necks etc. Also if would be\nawesome if we could support handing out a trac_sig/key to a user and if\nthey add this to their requiest extra tracing data could be dumped.\n\nBasic idea is we will be able to provide trace_sig which will be a HMAC from\nsome trace key and expiry time. If it passes it\u0027ll set TRACE_ACTIVATED_KEY\ninto environ, triggering the activation of tracing that request through the swift cluster.\n\nTracing is handled by a RequestTraceMiddleware wsgi mddleware.\nThis middleware is used to enable request tracing.\nIt supports tracing by giving a valid hmac signature provided in the\nquery string and/or it introduces another option which will trace every x\nrequest. This can be handy if you want to insert the middleware into\nsomething like the internal client config.\n\nThe request will start and stop tracing as the request hits this\nmiddleware. As such, to get as much info as possible it\u0027s best to have it\nas close to the front of the pipeline as possible.\n\nThe proxy has been instrumented so when making calls to storage nodes,\nso long as the middleware is also inserted into the storage node\npipelines, the tracing context is injected into the sub requests allowing\nus to trace a single request throughout the cluster.\n\nRequest tracing utilises the OpenTelemetry framework, which has now\nbecome a swift dependency. We could pull this out and warn.\n\nBecause a trace is scoped by the request. There is a trace_data object that\nis used to store some tracing metadata. Namely:\n\n - The call stack (so we can name middleware uniquely)\n - The current tracer object. As it\u0027s this object that creates spans.\n\nTracing and eventlet has issues. So we also make the GreenAsyncPool\ntrace aware, and also created sub-classed GreenPile to\nTraceAwareGreenPile, so it can pass a span (if it exists) into the\neventlet environment, allowing them to be traced correctly.\n\nTrace: be able to force the current span and SLO\n\nWhen we\u0027ve instrumented a response interator, like the SegmentIterater\nused by SLO, by the time the iter is being called by the client the\nrest of the request spans have returned.\nAlthough we force some of the spans to not be ended, the OpTel tracer object\nby the time it gets to a new iteration, it can\u0027t find the current\nrunning span to tie newly created spans off. This generates new traces\nrather the joining the existing trace.\n\nBecause eventlet uses the same process for its pool of threads, we\nstopped using otels global get_current_trace helper, and instead track them in\nthe request env.\nPutting the 2 together, we can optionally force the current span to come\nback into focus before the tracer object creates a new span, thereby\nkeeping the trace connected even when we\u0027re out in a response iter.\n\nTracing is handled by a new class decorator, wsgi_trace, that can be added to a\nmiddleware class. It\u0027ll use it\u0027s own `__call__` method that\nautomatically sets up a span and the tracing env for the middleware then\ncalls the wrapped class `__call__` function.\n\nAll the tracing code is in the swift.common.tracing namespace. And there\nyou\u0027ll find all the helper methods to further instrument code. To just\nname a few:\n\n  - trace_add: add a key value pair to the span.\n  - new_trace_span: a context manager to help create new spans\n  - trace_exception: attach a thrown exception to the span.\n\nAll the helper methods require passing in a wsgi env, this is where the\ntracing object will live if it exists. If tracing ins\u0027t enabled for the\nrequest, it\u0027s just a noop.\n\nChange-Id: I6c3765d43e0521dc0648ccd0956b93306d31e062\nSigned-off-by: Matthew Oliver \u003cmatt@oliver.net.au\u003e\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/fd9124c660bb8614e6bea85260a844fe5f3b9917"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/fd9124c660bb8614e6bea85260a844fe5f3b9917"}]},"branch":"refs/heads/master"},"4ffb96c9442f5f7af53490c103fb5b4e80758bc7":{"kind":"TRIVIAL_REBASE_WITH_MESSAGE_UPDATE","_number":10,"created":"2025-08-20 02:50:39.000000000","uploader":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"ref":"refs/changes/97/899397/10","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/swift","ref":"refs/changes/97/899397/10","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/swift refs/changes/97/899397/10 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/swift refs/changes/97/899397/10 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/swift refs/changes/97/899397/10 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/swift refs/changes/97/899397/10"}}},"commit":{"parents":[{"commit":"4d2426591e8fe47f0c7cee5d73cf4bc0004e33a0","subject":"CI: Stop using bionic","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/4d2426591e8fe47f0c7cee5d73cf4bc0004e33a0"}]}],"author":{"name":"Matthew Oliver","email":"matt@oliver.net.au","date":"2021-06-24 07:34:28.000000000","tz":600},"committer":{"name":"Matthew Oliver","email":"matt@oliver.net.au","date":"2025-08-20 02:50:21.000000000","tz":600},"subject":"request tracing: Add Otel request tracing support (decorator)","message":"request tracing: Add Otel request tracing support (decorator)\n\nWe\u0027ve been wanting some micro benchmarks inside our WSGI environments,\nlike the proxy server, in order to test and find bottle necks etc.  But\non top of that, it\u0027ll also be awesome if we could support handing out a\ntrac_sig/key to a user and if they add this to their request path extra\ntracing data could be dumped.\n\nInitially we see a few ways of activating a trace:\n 1. provide trace_sig which will be a HMAC from some trace key and expiry time.\n 2. Trace every x request.\n 3. Continue a trace if it finds it previously activated and trace\n    context found inserted into a otel http header.\n\nWhen one of these are met, a tracer is created and added to the request\nenvironment which is used to trace the rest of the request.\n\nTracing is handled by a RequestTraceMiddleware wsgi mddleware.\nThis middleware is used to enable request tracing.\nThe request will start and stop tracing as the request hits this\nmiddleware. As such, to get as much info as possible it\u0027s best to have it\nas close to the front of the pipeline as possible.\n\nThe proxy has been instrumented so when making calls to storage nodes\nthe tracing context is injected into the sub requests allowing up to\ntrace a single request throughout the cluster. So long as the trace\nmiddleware has also been included inside the storage node pipelines too.\n\nRequest tracing utilises the OpenTelemetry framework, which has now\nbecome a swift dependency. We could pull this out and warn.\n\nBecause a trace is scoped by the request. There is a trace_data object that\nis used to store some tracing metadata. Namely:\n\n - The call stack (so we can name middleware uniquely)\n - The current tracer object. As it\u0027s this object that creates spans.\n\nTracing is handled by a new class decorator, wsgi_trace, that can be added to a\nmiddleware class. It\u0027ll use it\u0027s own `__call__` method that\nautomatically sets up a span and the tracing env for the middleware then\ncalls the wrapped class `__call__` function.\n\nAll the tracing code is in the swift.common.tracing namespace. And there\nyou\u0027ll find all the helper methods to further instrument code. To just\nname a few:\n\n  - trace_add: add a key value pair to the span.\n  - new_trace_span: a context manager to help create new spans\n  - trace_exception: attach a thrown exception to the span.\n\nAll the helper methods require passing in a wsgi env, this is where the\ntracing object will live if it exists. If tracing ins\u0027t enabled for the\nrequest, it\u0027s just a noop.\n\nUnfortunately, tracing and eventlet don\u0027t always play nice. As such we\ncan\u0027t just use the global nature of the python otel implementation and\nhave to do some interesting scaffolding to get them to work together.\nSuch as, make the GreenAsyncPool trace aware, and also created\nsub-classed GreenPile to TraceAwareGreenPile, so it can pass a span (if\nit exists) into the eventlet environment, allowing them to be traced\ncorrectly.\nFor example, sometimes an eventlet thread can loose the current context\nso we need to force it.\n\nBecause SLOs are a heavily used middleware, we\u0027ve heavily instrumented\nSLOs, including the SegmentIterater response iterator\nused by SLO. By the time the iter is being called by the client the\nrest of the request spans have returned, so here is another time we need\nto take our saved current span and force it back so the tracers can link\nup correctly at the collection point, rather then starting \"new\" traces.\n\nChange-Id: I6c3765d43e0521dc0648ccd0956b93306d31e062\nSigned-off-by: Matthew Oliver \u003cmatt@oliver.net.au\u003e\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/4ffb96c9442f5f7af53490c103fb5b4e80758bc7"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/4ffb96c9442f5f7af53490c103fb5b4e80758bc7"}]},"branch":"refs/heads/master"},"ef5da58702b48a1c3445915afcd98b0d18741083":{"kind":"REWORK","_number":11,"created":"2025-09-04 07:25:27.000000000","uploader":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"ref":"refs/changes/97/899397/11","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/swift","ref":"refs/changes/97/899397/11","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/swift refs/changes/97/899397/11 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/swift refs/changes/97/899397/11 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/swift refs/changes/97/899397/11 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/swift refs/changes/97/899397/11"}}},"commit":{"parents":[{"commit":"e13f4abcd7af52f635d100c2bd495cecd8a9ca78","subject":"tests: Skip some tests if crc32c is not available","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/e13f4abcd7af52f635d100c2bd495cecd8a9ca78"}]}],"author":{"name":"Matthew Oliver","email":"matt@oliver.net.au","date":"2021-06-24 07:34:28.000000000","tz":600},"committer":{"name":"Matthew Oliver","email":"matt@oliver.net.au","date":"2025-09-04 07:25:10.000000000","tz":600},"subject":"request tracing: Add Otel request tracing support (decorator)","message":"request tracing: Add Otel request tracing support (decorator)\n\nWe\u0027ve been wanting some micro benchmarks inside our WSGI environments,\nlike the proxy server, in order to test and find bottle necks etc.  But\non top of that, it\u0027ll also be awesome if we could support handing out a\ntrac_sig/key to a user and if they add this to their request path extra\ntracing data could be dumped.\n\nInitially we see a few ways of activating a trace:\n 1. provide trace_sig which will be a HMAC from some trace key and expiry time.\n 2. Trace every x request.\n 3. Continue a trace if it finds it previously activated and trace\n    context found inserted into a otel http header.\n\nWhen one of these are met, a tracer is created and added to the request\nenvironment which is used to trace the rest of the request.\n\nTracing is handled by a RequestTraceMiddleware wsgi mddleware.\nThis middleware is used to enable request tracing.\nThe request will start and stop tracing as the request hits this\nmiddleware. As such, to get as much info as possible it\u0027s best to have it\nas close to the front of the pipeline as possible.\n\nThe proxy has been instrumented so when making calls to storage nodes\nthe tracing context is injected into the sub requests allowing up to\ntrace a single request throughout the cluster. So long as the trace\nmiddleware has also been included inside the storage node pipelines too.\n\nRequest tracing utilises the OpenTelemetry framework, which has now\nbecome a swift dependency. We could pull this out and warn.\n\nBecause a trace is scoped by the request. There is a trace_data object that\nis used to store some tracing metadata. Namely:\n\n - The call stack (so we can name middleware uniquely)\n - The current tracer object. As it\u0027s this object that creates spans.\n\nTracing is handled by a new class decorator, wsgi_trace, that can be added to a\nmiddleware class. It\u0027ll use it\u0027s own `__call__` method that\nautomatically sets up a span and the tracing env for the middleware then\ncalls the wrapped class `__call__` function.\n\nAll the tracing code is in the swift.common.tracing namespace. And there\nyou\u0027ll find all the helper methods to further instrument code. To just\nname a few:\n\n  - trace_add: add a key value pair to the span.\n  - new_trace_span: a context manager to help create new spans\n  - trace_exception: attach a thrown exception to the span.\n\nAll the helper methods require passing in a wsgi env, this is where the\ntracing object will live if it exists. If tracing ins\u0027t enabled for the\nrequest, it\u0027s just a noop.\n\nUnfortunately, tracing and eventlet don\u0027t always play nice. As such we\ncan\u0027t just use the global nature of the python otel implementation and\nhave to do some interesting scaffolding to get them to work together.\nSuch as, make the GreenAsyncPool trace aware, and also created\nsub-classed GreenPile to TraceAwareGreenPile, so it can pass a span (if\nit exists) into the eventlet environment, allowing them to be traced\ncorrectly.\nFor example, sometimes an eventlet thread can loose the current context\nso we need to force it.\n\nBecause SLOs are a heavily used middleware, we\u0027ve heavily instrumented\nSLOs, including the SegmentIterater response iterator\nused by SLO. By the time the iter is being called by the client the\nrest of the request spans have returned, so here is another time we need\nto take our saved current span and force it back so the tracers can link\nup correctly at the collection point, rather then starting \"new\" traces.\n\nChange-Id: I6c3765d43e0521dc0648ccd0956b93306d31e062\nSigned-off-by: Matthew Oliver \u003cmatt@oliver.net.au\u003e\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/ef5da58702b48a1c3445915afcd98b0d18741083"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/ef5da58702b48a1c3445915afcd98b0d18741083"}]},"branch":"refs/heads/master"},"a378fe8d66661f0bba9ea82c38f899c796295ef6":{"kind":"REWORK","_number":12,"created":"2025-09-15 07:22:22.000000000","uploader":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"ref":"refs/changes/97/899397/12","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/swift","ref":"refs/changes/97/899397/12","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/swift refs/changes/97/899397/12 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/swift refs/changes/97/899397/12 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/swift refs/changes/97/899397/12 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/swift refs/changes/97/899397/12"}}},"commit":{"parents":[{"commit":"e13f4abcd7af52f635d100c2bd495cecd8a9ca78","subject":"tests: Skip some tests if crc32c is not available","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/e13f4abcd7af52f635d100c2bd495cecd8a9ca78"}]}],"author":{"name":"Matthew Oliver","email":"matt@oliver.net.au","date":"2021-06-24 07:34:28.000000000","tz":600},"committer":{"name":"Matthew Oliver","email":"matt@oliver.net.au","date":"2025-09-15 07:21:05.000000000","tz":600},"subject":"request tracing: Add Otel request tracing support (decorator)","message":"request tracing: Add Otel request tracing support (decorator)\n\nWe\u0027ve been wanting some micro benchmarks inside our WSGI environments,\nlike the proxy server, in order to test and find bottle necks etc.  But\non top of that, it\u0027ll also be awesome if we could support handing out a\ntrac_sig/key to a user and if they add this to their request path extra\ntracing data could be dumped.\n\nInitially we see a few ways of activating a trace:\n 1. provide trace_sig which will be a HMAC from some trace key and expiry time.\n 2. Trace every x request.\n 3. Continue a trace if it finds it previously activated and trace\n    context found inserted into a otel http header.\n\nWhen one of these are met, a tracer is created and added to the request\nenvironment which is used to trace the rest of the request.\n\nTracing is handled by a RequestTraceMiddleware wsgi mddleware.\nThis middleware is used to enable request tracing.\nThe request will start and stop tracing as the request hits this\nmiddleware. As such, to get as much info as possible it\u0027s best to have it\nas close to the front of the pipeline as possible.\n\nThe proxy has been instrumented so when making calls to storage nodes\nthe tracing context is injected into the sub requests allowing up to\ntrace a single request throughout the cluster. So long as the trace\nmiddleware has also been included inside the storage node pipelines too.\n\nRequest tracing utilises the OpenTelemetry framework, which has now\nbecome a swift dependency. We could pull this out and warn.\n\nBecause a trace is scoped by the request. There is a trace_data object that\nis used to store some tracing metadata. Namely:\n\n - The call stack (so we can name middleware uniquely)\n - The current tracer object. As it\u0027s this object that creates spans.\n\nTracing is handled by a new class decorator, wsgi_trace, that can be added to a\nmiddleware class. It\u0027ll use it\u0027s own `__call__` method that\nautomatically sets up a span and the tracing env for the middleware then\ncalls the wrapped class `__call__` function.\n\nAll the tracing code is in the swift.common.tracing namespace. And there\nyou\u0027ll find all the helper methods to further instrument code. To just\nname a few:\n\n  - trace_add: add a key value pair to the span.\n  - new_trace_span: a context manager to help create new spans\n  - trace_exception: attach a thrown exception to the span.\n\nAll the helper methods require passing in a wsgi env, this is where the\ntracing object will live if it exists. If tracing ins\u0027t enabled for the\nrequest, it\u0027s just a noop.\n\nUnfortunately, tracing and eventlet don\u0027t always play nice. As such we\ncan\u0027t just use the global nature of the python otel implementation and\nhave to do some interesting scaffolding to get them to work together.\nSuch as, make the GreenAsyncPool trace aware, and also created\nsub-classed GreenPile to TraceAwareGreenPile, so it can pass a span (if\nit exists) into the eventlet environment, allowing them to be traced\ncorrectly.\nFor example, sometimes an eventlet thread can loose the current context\nso we need to force it.\n\nBecause SLOs are a heavily used middleware, we\u0027ve heavily instrumented\nSLOs, including the SegmentIterater response iterator\nused by SLO. By the time the iter is being called by the client the\nrest of the request spans have returned, so here is another time we need\nto take our saved current span and force it back so the tracers can link\nup correctly at the collection point, rather then starting \"new\" traces.\n\nSigned-off-by: Matthew Oliver \u003cmatt@oliver.net.au\u003e\nChange-Id: I6c3765d43e0521dc0648ccd0956b93306d31e062\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/a378fe8d66661f0bba9ea82c38f899c796295ef6"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/a378fe8d66661f0bba9ea82c38f899c796295ef6"}]},"branch":"refs/heads/master"},"ac1a11d8438e99fbf2643c354baf4292c557a972":{"kind":"REWORK","_number":13,"created":"2025-09-19 03:58:47.000000000","uploader":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"ref":"refs/changes/97/899397/13","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/swift","ref":"refs/changes/97/899397/13","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/swift refs/changes/97/899397/13 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/swift refs/changes/97/899397/13 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/swift refs/changes/97/899397/13 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/swift refs/changes/97/899397/13"}}},"commit":{"parents":[{"commit":"e13f4abcd7af52f635d100c2bd495cecd8a9ca78","subject":"tests: Skip some tests if crc32c is not available","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/e13f4abcd7af52f635d100c2bd495cecd8a9ca78"}]}],"author":{"name":"Matthew Oliver","email":"matt@oliver.net.au","date":"2021-06-24 07:34:28.000000000","tz":600},"committer":{"name":"Matthew Oliver","email":"matt@oliver.net.au","date":"2025-09-19 03:58:25.000000000","tz":600},"subject":"request tracing: Add Otel request tracing support (decorator)","message":"request tracing: Add Otel request tracing support (decorator)\n\nWe\u0027ve been wanting some micro benchmarks inside our WSGI environments,\nlike the proxy server, in order to test and find bottle necks etc.  But\non top of that, it\u0027ll also be awesome if we could support handing out a\ntrac_sig/key to a user and if they add this to their request path extra\ntracing data could be dumped.\n\nInitially we see a few ways of activating a trace:\n 1. provide trace_sig which will be a HMAC from some trace key and expiry time.\n 2. Trace every x request.\n 3. Continue a trace if it finds it previously activated and trace\n    context found inserted into a otel http header.\n\nWhen one of these are met, a tracer is created and added to the request\nenvironment which is used to trace the rest of the request.\n\nTracing is handled by a RequestTraceMiddleware wsgi mddleware.\nThis middleware is used to enable request tracing.\nThe request will start and stop tracing as the request hits this\nmiddleware. As such, to get as much info as possible it\u0027s best to have it\nas close to the front of the pipeline as possible.\n\nThe proxy has been instrumented so when making calls to storage nodes\nthe tracing context is injected into the sub requests allowing up to\ntrace a single request throughout the cluster. So long as the trace\nmiddleware has also been included inside the storage node pipelines too.\n\nRequest tracing utilises the OpenTelemetry framework, which has now\nbecome a swift dependency. We could pull this out and warn.\n\nBecause a trace is scoped by the request. There is a trace_data object that\nis used to store some tracing metadata. Namely:\n\n - The call stack (so we can name middleware uniquely)\n - The current tracer object. As it\u0027s this object that creates spans.\n\nTracing is handled by a new class decorator, wsgi_trace, that can be added to a\nmiddleware class. It\u0027ll use it\u0027s own `__call__` method that\nautomatically sets up a span and the tracing env for the middleware then\ncalls the wrapped class `__call__` function.\n\nAll the tracing code is in the swift.common.tracing namespace. And there\nyou\u0027ll find all the helper methods to further instrument code. To just\nname a few:\n\n  - trace_add: add a key value pair to the span.\n  - new_trace_span: a context manager to help create new spans\n  - trace_exception: attach a thrown exception to the span.\n\nAll the helper methods require passing in a wsgi env, this is where the\ntracing object will live if it exists. If tracing ins\u0027t enabled for the\nrequest, it\u0027s just a noop.\n\nUnfortunately, tracing and eventlet don\u0027t always play nice. As such we\ncan\u0027t just use the global nature of the python otel implementation and\nhave to do some interesting scaffolding to get them to work together.\nSuch as, make the GreenAsyncPool trace aware, and also created\nsub-classed GreenPile to TraceAwareGreenPile, so it can pass a span (if\nit exists) into the eventlet environment, allowing them to be traced\ncorrectly.\nFor example, sometimes an eventlet thread can loose the current context\nso we need to force it.\n\nBecause SLOs are a heavily used middleware, we\u0027ve heavily instrumented\nSLOs, including the SegmentIterater response iterator\nused by SLO. By the time the iter is being called by the client the\nrest of the request spans have returned, so here is another time we need\nto take our saved current span and force it back so the tracers can link\nup correctly at the collection point, rather then starting \"new\" traces.\n\nSigned-off-by: Matthew Oliver \u003cmatt@oliver.net.au\u003e\nChange-Id: I6c3765d43e0521dc0648ccd0956b93306d31e062\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/ac1a11d8438e99fbf2643c354baf4292c557a972"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/ac1a11d8438e99fbf2643c354baf4292c557a972"}]},"branch":"refs/heads/master"},"4b65d8c09f406409fba46fa651693a489a6971b2":{"kind":"REWORK","_number":14,"created":"2026-02-02 03:39:35.000000000","uploader":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"ref":"refs/changes/97/899397/14","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/swift","ref":"refs/changes/97/899397/14","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/swift refs/changes/97/899397/14 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/swift refs/changes/97/899397/14 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/swift refs/changes/97/899397/14 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/swift refs/changes/97/899397/14"}}},"commit":{"parents":[{"commit":"7bf728b6198400aa2057bf42d34f434e6dc71407","subject":"direct_client.py: proxy app: assign timestamp in a single place","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/7bf728b6198400aa2057bf42d34f434e6dc71407"}]}],"author":{"name":"Matthew Oliver","email":"matt@oliver.net.au","date":"2021-06-24 07:34:28.000000000","tz":600},"committer":{"name":"Matthew Oliver","email":"matt@oliver.net.au","date":"2026-02-02 03:39:09.000000000","tz":660},"subject":"request tracing: Add Otel request tracing support (decorator)","message":"request tracing: Add Otel request tracing support (decorator)\n\nWe\u0027ve been wanting some micro benchmarks inside our WSGI environments,\nlike the proxy server, in order to test and find bottle necks etc.  But\non top of that, it\u0027ll also be awesome if we could support handing out a\ntrac_sig/key to a user and if they add this to their request path extra\ntracing data could be dumped.\n\nInitially we see a few ways of activating a trace:\n 1. provide trace_sig which will be a HMAC from some trace key and expiry time.\n 2. Trace every x request.\n 3. Continue a trace if it finds it previously activated and trace\n    context found inserted into a otel http header.\n\nWhen one of these are met, a tracer is created and added to the request\nenvironment which is used to trace the rest of the request.\n\nTracing is handled by a RequestTraceMiddleware wsgi mddleware.\nThis middleware is used to enable request tracing.\nThe request will start and stop tracing as the request hits this\nmiddleware. As such, to get as much info as possible it\u0027s best to have it\nas close to the front of the pipeline as possible.\n\nThe proxy has been instrumented so when making calls to storage nodes\nthe tracing context is injected into the sub requests allowing up to\ntrace a single request throughout the cluster. So long as the trace\nmiddleware has also been included inside the storage node pipelines too.\n\nRequest tracing utilises the OpenTelemetry framework, which has now\nbecome a swift dependency. We could pull this out and warn.\n\nBecause a trace is scoped by the request. There is a trace_data object that\nis used to store some tracing metadata. Namely:\n\n - The call stack (so we can name middleware uniquely)\n - The current tracer object. As it\u0027s this object that creates spans.\n\nTracing is handled by a new class decorator, wsgi_trace, that can be added to a\nmiddleware class. It\u0027ll use it\u0027s own `__call__` method that\nautomatically sets up a span and the tracing env for the middleware then\ncalls the wrapped class `__call__` function.\n\nAll the tracing code is in the swift.common.tracing namespace. And there\nyou\u0027ll find all the helper methods to further instrument code. To just\nname a few:\n\n  - trace_add: add a key value pair to the span.\n  - new_trace_span: a context manager to help create new spans\n  - trace_exception: attach a thrown exception to the span.\n\nAll the helper methods require passing in a wsgi env, this is where the\ntracing object will live if it exists. If tracing ins\u0027t enabled for the\nrequest, it\u0027s just a noop.\n\nUnfortunately, tracing and eventlet don\u0027t always play nice. As such we\ncan\u0027t just use the global nature of the python otel implementation and\nhave to do some interesting scaffolding to get them to work together.\nSuch as, make the GreenAsyncPool trace aware, and also created\nsub-classed GreenPile to TraceAwareGreenPile, so it can pass a span (if\nit exists) into the eventlet environment, allowing them to be traced\ncorrectly.\nFor example, sometimes an eventlet thread can loose the current context\nso we need to force it.\n\nBecause SLOs are a heavily used middleware, we\u0027ve heavily instrumented\nSLOs, including the SegmentIterater response iterator\nused by SLO. By the time the iter is being called by the client the\nrest of the request spans have returned, so here is another time we need\nto take our saved current span and force it back so the tracers can link\nup correctly at the collection point, rather then starting \"new\" traces.\n\nSigned-off-by: Matthew Oliver \u003cmatt@oliver.net.au\u003e\nChange-Id: I6c3765d43e0521dc0648ccd0956b93306d31e062\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/4b65d8c09f406409fba46fa651693a489a6971b2"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/4b65d8c09f406409fba46fa651693a489a6971b2"}]},"branch":"refs/heads/master"},"1ce814d9d06ecaa0c752ac552f05d9c3be41aea1":{"kind":"REWORK","_number":15,"created":"2026-05-05 07:49:45.000000000","uploader":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"ref":"refs/changes/97/899397/15","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/swift","ref":"refs/changes/97/899397/15","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/swift refs/changes/97/899397/15 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/swift refs/changes/97/899397/15 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/swift refs/changes/97/899397/15 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/swift refs/changes/97/899397/15"}}},"commit":{"parents":[{"commit":"453e2356ba92fdefbe9e0886028d8414a1063684","subject":"Merge \"Consolidate swift.common.concurrency import statments\"","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/453e2356ba92fdefbe9e0886028d8414a1063684"}]}],"author":{"name":"Matthew Oliver","email":"matt@oliver.net.au","date":"2021-06-24 07:34:28.000000000","tz":600},"committer":{"name":"Matthew Oliver","email":"matt@oliver.net.au","date":"2026-05-05 07:49:24.000000000","tz":600},"subject":"request tracing: Add Otel request tracing support (decorator)","message":"request tracing: Add Otel request tracing support (decorator)\n\nWe\u0027ve been wanting some micro benchmarks inside our WSGI environments,\nlike the proxy server, in order to test and find bottle necks etc.  But\non top of that, it\u0027ll also be awesome if we could support handing out a\ntrac_sig/key to a user and if they add this to their request path extra\ntracing data could be dumped.\n\nInitially we see a few ways of activating a trace:\n 1. provide trace_sig which will be a HMAC from some trace key and expiry time.\n 2. Trace every x request.\n 3. Continue a trace if it finds it previously activated and trace\n    context found inserted into a otel http header.\n\nWhen one of these are met, a tracer is created and added to the request\nenvironment which is used to trace the rest of the request.\n\nTracing is handled by a RequestTraceMiddleware wsgi mddleware.\nThis middleware is used to enable request tracing.\nThe request will start and stop tracing as the request hits this\nmiddleware. As such, to get as much info as possible it\u0027s best to have it\nas close to the front of the pipeline as possible.\n\nThe proxy has been instrumented so when making calls to storage nodes\nthe tracing context is injected into the sub requests allowing up to\ntrace a single request throughout the cluster. So long as the trace\nmiddleware has also been included inside the storage node pipelines too.\n\nRequest tracing utilises the OpenTelemetry framework, which has now\nbecome a swift dependency. We could pull this out and warn.\n\nBecause a trace is scoped by the request. There is a trace_data object that\nis used to store some tracing metadata. Namely:\n\n - The call stack (so we can name middleware uniquely)\n - The current tracer object. As it\u0027s this object that creates spans.\n\nTracing is handled by a new class decorator, wsgi_trace, that can be added to a\nmiddleware class. It\u0027ll use it\u0027s own `__call__` method that\nautomatically sets up a span and the tracing env for the middleware then\ncalls the wrapped class `__call__` function.\n\nAll the tracing code is in the swift.common.tracing namespace. And there\nyou\u0027ll find all the helper methods to further instrument code. To just\nname a few:\n\n  - trace_add: add a key value pair to the span.\n  - new_trace_span: a context manager to help create new spans\n  - trace_exception: attach a thrown exception to the span.\n\nAll the helper methods require passing in a wsgi env, this is where the\ntracing object will live if it exists. If tracing ins\u0027t enabled for the\nrequest, it\u0027s just a noop.\n\nUnfortunately, tracing and eventlet don\u0027t always play nice. As such we\ncan\u0027t just use the global nature of the python otel implementation and\nhave to do some interesting scaffolding to get them to work together.\nSuch as, make the GreenAsyncPool trace aware, and also created\nsub-classed GreenPile to TraceAwareGreenPile, so it can pass a span (if\nit exists) into the eventlet environment, allowing them to be traced\ncorrectly.\nFor example, sometimes an eventlet thread can loose the current context\nso we need to force it.\n\nBecause SLOs are a heavily used middleware, we\u0027ve heavily instrumented\nSLOs, including the SegmentIterater response iterator\nused by SLO. By the time the iter is being called by the client the\nrest of the request spans have returned, so here is another time we need\nto take our saved current span and force it back so the tracers can link\nup correctly at the collection point, rather then starting \"new\" traces.\n\nSigned-off-by: Matthew Oliver \u003cmatt@oliver.net.au\u003e\nChange-Id: I6c3765d43e0521dc0648ccd0956b93306d31e062\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/1ce814d9d06ecaa0c752ac552f05d9c3be41aea1"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/1ce814d9d06ecaa0c752ac552f05d9c3be41aea1"}]},"branch":"refs/heads/master"},"ac655f34c65608b6a4c9371533133a31f2b6f52c":{"kind":"REWORK","_number":16,"created":"2026-06-03 06:07:00.000000000","uploader":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"ref":"refs/changes/97/899397/16","fetch":{"anonymous http":{"url":"https://review.opendev.org/openstack/swift","ref":"refs/changes/97/899397/16","commands":{"Checkout":"git fetch https://review.opendev.org/openstack/swift refs/changes/97/899397/16 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.opendev.org/openstack/swift refs/changes/97/899397/16 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.opendev.org/openstack/swift refs/changes/97/899397/16 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.opendev.org/openstack/swift refs/changes/97/899397/16"}}},"commit":{"parents":[{"commit":"6e6010253b285269daf711896781c9ad27b84800","subject":"Merge \"Add x-amz-server-side-encryption on s3api\"","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/6e6010253b285269daf711896781c9ad27b84800"}]}],"author":{"name":"Matthew Oliver","email":"matt@oliver.net.au","date":"2021-06-24 07:34:28.000000000","tz":600},"committer":{"name":"Matthew Oliver","email":"matt@oliver.net.au","date":"2026-06-03 06:06:01.000000000","tz":600},"subject":"request tracing: Add Otel request tracing support (decorator)","message":"request tracing: Add Otel request tracing support (decorator)\n\nWe\u0027ve been wanting some micro benchmarks inside our WSGI environments,\nlike the proxy server, in order to test and find bottle necks etc.  But\non top of that, it\u0027ll also be awesome if we could support handing out a\ntrac_sig/key to a user and if they add this to their request path extra\ntracing data could be dumped.\n\nInitially we see a few ways of activating a trace:\n 1. provide trace_sig which will be a HMAC from some trace key and expiry time.\n 2. Trace every x request.\n 3. Continue a trace if it finds it previously activated and trace\n    context found inserted into a otel http header.\n\nWhen one of these are met, a tracer is created and added to the request\nenvironment which is used to trace the rest of the request.\n\nTracing is handled by a RequestTraceMiddleware wsgi mddleware.\nThis middleware is used to enable request tracing.\nThe request will start and stop tracing as the request hits this\nmiddleware. As such, to get as much info as possible it\u0027s best to have it\nas close to the front of the pipeline as possible.\n\nThe proxy has been instrumented so when making calls to storage nodes\nthe tracing context is injected into the sub requests allowing up to\ntrace a single request throughout the cluster. So long as the trace\nmiddleware has also been included inside the storage node pipelines too.\n\nRequest tracing utilises the OpenTelemetry framework, which has now\nbecome a swift dependency. We could pull this out and warn.\n\nBecause a trace is scoped by the request. There is a trace_data object that\nis used to store some tracing metadata. Namely:\n\n - The call stack (so we can name middleware uniquely)\n - The current tracer object. As it\u0027s this object that creates spans.\n\nTracing is handled by a new class decorator, wsgi_trace, that can be added to a\nmiddleware class. It\u0027ll use it\u0027s own `__call__` method that\nautomatically sets up a span and the tracing env for the middleware then\ncalls the wrapped class `__call__` function.\n\nAll the tracing code is in the swift.common.tracing namespace. And there\nyou\u0027ll find all the helper methods to further instrument code. To just\nname a few:\n\n  - trace_add: add a key value pair to the span.\n  - new_trace_span: a context manager to help create new spans\n  - trace_exception: attach a thrown exception to the span.\n\nAll the helper methods require passing in a wsgi env, this is where the\ntracing object will live if it exists. If tracing ins\u0027t enabled for the\nrequest, it\u0027s just a noop.\n\nUnfortunately, tracing and eventlet don\u0027t always play nice. As such we\ncan\u0027t just use the global nature of the python otel implementation and\nhave to do some interesting scaffolding to get them to work together.\nSuch as, make the GreenAsyncPool trace aware, and also created\nsub-classed GreenPile to TraceAwareGreenPile, so it can pass a span (if\nit exists) into the eventlet environment, allowing them to be traced\ncorrectly.\nFor example, sometimes an eventlet thread can loose the current context\nso we need to force it.\n\nBecause SLOs are a heavily used middleware, we\u0027ve heavily instrumented\nSLOs, including the SegmentIterater response iterator\nused by SLO. By the time the iter is being called by the client the\nrest of the request spans have returned, so here is another time we need\nto take our saved current span and force it back so the tracers can link\nup correctly at the collection point, rather then starting \"new\" traces.\n\nSigned-off-by: Matthew Oliver \u003cmatt@oliver.net.au\u003e\nChange-Id: I6c3765d43e0521dc0648ccd0956b93306d31e062\n","web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/ac655f34c65608b6a4c9371533133a31f2b6f52c"}],"resolve_conflicts_web_links":[{"name":"gitea","tooltip":"Open in GitWeb","url":"https://opendev.org/openstack/swift/commit/ac655f34c65608b6a4c9371533133a31f2b6f52c"}]},"branch":"refs/heads/master"}},"requirements":[],"submit_records":[{"rule_name":"gerrit~DefaultSubmitRule","status":"OK","labels":[{"label":"Verified","status":"MAY","applied_by":{"_account_id":22348,"name":"Zuul","username":"zuul","tags":["SERVICE_USER"]}},{"label":"Code-Review","status":"MAY"},{"label":"Workflow","status":"MAY"}]}],"submit_requirements":[{"name":"Verified","description":"Verified in gate by CI","status":"UNSATISFIED","is_legacy":false,"submittability_expression_result":{"expression":"label:Verified\u003dMAX AND -label:Verified\u003dMIN","fulfilled":false,"status":"FAIL","passing_atoms":[],"failing_atoms":["label:Verified\u003dMAX","label:Verified\u003dMIN"],"atom_explanations":{"label:Verified\u003dMAX":"","label:Verified\u003dMIN":""}}},{"name":"Code-Review","description":"Code reviewed by core reviewer","status":"UNSATISFIED","is_legacy":false,"submittability_expression_result":{"expression":"label:Code-Review\u003dMAX AND -label:Code-Review\u003dMIN","fulfilled":false,"status":"FAIL","passing_atoms":[],"failing_atoms":["label:Code-Review\u003dMAX","label:Code-Review\u003dMIN"],"atom_explanations":{"label:Code-Review\u003dMAX":"","label:Code-Review\u003dMIN":""}}},{"name":"Workflow","description":"Approved for gate by core reviewer","status":"UNSATISFIED","is_legacy":false,"submittability_expression_result":{"expression":"label:Workflow\u003dMAX AND -label:Workflow\u003dMIN","fulfilled":false,"status":"FAIL","passing_atoms":[],"failing_atoms":["label:Workflow\u003dMAX","label:Workflow\u003dMIN"],"atom_explanations":{"label:Workflow\u003dMAX":"","label:Workflow\u003dMIN":""}}}]}