)]}'
{"doc/source/developer/specs/tracing.rst":[{"author":{"_account_id":5263,"name":"Jeremy Stanley","display_name":"fungi","email":"fungi@yuggoth.org","username":"fungi","status":"missing, presumed fed"},"change_message_id":"9b71d9b1607388d43a4efe5791090dd0d130c33b","unresolved":false,"context_lines":[{"line_number":23,"context_line":"operators (and potentially users) triage and diagnose issues more"},{"line_number":24,"context_line":"quickly and accurately.  Even if examining logs is ultimately required"},{"line_number":25,"context_line":"in order to fully diagnose an issue, being able to narrow down the"},{"line_number":26,"context_line":"scope using analsys tools can greatly simplify the process."},{"line_number":27,"context_line":""},{"line_number":28,"context_line":"Proposed Solution"},{"line_number":29,"context_line":"-----------------"}],"source_content_type":"text/x-rst","patch_set":1,"id":"a82d096f_a6f83c3f","line":26,"updated":"2022-08-02 16:02:38.000000000","message":"Nit: \"analysis\"","commit_id":"30c046842459891d03654d6ceeaa46fa3f27bd57"},{"author":{"_account_id":1,"name":"James E. Blair","email":"jim@acmegating.com","username":"corvus"},"change_message_id":"708876c59edf37eab135b175c1e5837f5e78036d","unresolved":false,"context_lines":[{"line_number":23,"context_line":"operators (and potentially users) triage and diagnose issues more"},{"line_number":24,"context_line":"quickly and accurately.  Even if examining logs is ultimately required"},{"line_number":25,"context_line":"in order to fully diagnose an issue, being able to narrow down the"},{"line_number":26,"context_line":"scope using analsys tools can greatly simplify the process."},{"line_number":27,"context_line":""},{"line_number":28,"context_line":"Proposed Solution"},{"line_number":29,"context_line":"-----------------"}],"source_content_type":"text/x-rst","patch_set":1,"id":"82681fb4_87532264","line":26,"in_reply_to":"a82d096f_a6f83c3f","updated":"2022-08-02 20:56:44.000000000","message":"I should really turn on flyspell.","commit_id":"30c046842459891d03654d6ceeaa46fa3f27bd57"},{"author":{"_account_id":9311,"name":"Tristan Cacqueray","email":"tdecacqu@redhat.com","username":"tristanC"},"change_message_id":"7ee932f57e6cc438e2fcbcc81e727767f3f84847","unresolved":true,"context_lines":[{"line_number":50,"context_line":""},{"line_number":51,"context_line":"Usage of OpenTelemetry should be entirely optional and supplementary"},{"line_number":52,"context_line":"for any Zuul deployment.  Log messages alone should continue to be"},{"line_number":53,"context_line":"sufficient to analyze any potential problem."},{"line_number":54,"context_line":""},{"line_number":55,"context_line":"Should a deployer wish to use OpenTelemetry tracing data, a very"},{"line_number":56,"context_line":"simple deployment for smaller sites may be constructed by running only"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bd4ef1c6_84224376","line":53,"updated":"2022-08-02 13:28:19.000000000","message":"Could the existing logging infrastructure be replaced by a console exporter? That is how the tracing library works in Rust, and it\u0027s quite useful because the same message are used for both console output and for opentelemetry service like jaeger.\n\nOtherwise, isn\u0027t this spec going to duplicate the existing work done on logging?","commit_id":"30c046842459891d03654d6ceeaa46fa3f27bd57"},{"author":{"_account_id":9311,"name":"Tristan Cacqueray","email":"tdecacqu@redhat.com","username":"tristanC"},"change_message_id":"34790cc644b44c6f6422e74039f4328138cc8e24","unresolved":false,"context_lines":[{"line_number":50,"context_line":""},{"line_number":51,"context_line":"Usage of OpenTelemetry should be entirely optional and supplementary"},{"line_number":52,"context_line":"for any Zuul deployment.  Log messages alone should continue to be"},{"line_number":53,"context_line":"sufficient to analyze any potential problem."},{"line_number":54,"context_line":""},{"line_number":55,"context_line":"Should a deployer wish to use OpenTelemetry tracing data, a very"},{"line_number":56,"context_line":"simple deployment for smaller sites may be constructed by running only"}],"source_content_type":"text/x-rst","patch_set":1,"id":"7e93b10e_565644c5","line":53,"in_reply_to":"0ff22820_9db506e9","updated":"2022-08-02 23:31:59.000000000","message":"What would be missing from the span that only appears in logs? I understand we don\u0027t want to trace low level debug messages, but shouldn\u0027t most of the important logs be part of the trace?","commit_id":"30c046842459891d03654d6ceeaa46fa3f27bd57"},{"author":{"_account_id":9311,"name":"Tristan Cacqueray","email":"tdecacqu@redhat.com","username":"tristanC"},"change_message_id":"ce81842b135d9f5529914a4008cb65cf4ff06a56","unresolved":false,"context_lines":[{"line_number":50,"context_line":""},{"line_number":51,"context_line":"Usage of OpenTelemetry should be entirely optional and supplementary"},{"line_number":52,"context_line":"for any Zuul deployment.  Log messages alone should continue to be"},{"line_number":53,"context_line":"sufficient to analyze any potential problem."},{"line_number":54,"context_line":""},{"line_number":55,"context_line":"Should a deployer wish to use OpenTelemetry tracing data, a very"},{"line_number":56,"context_line":"simple deployment for smaller sites may be constructed by running only"}],"source_content_type":"text/x-rst","patch_set":1,"id":"c987b5b9_83961289","line":53,"in_reply_to":"788f0bf1_bbcc07a8","updated":"2022-08-03 12:16:45.000000000","message":"Thank you for the details, that makes sense to me. I hope we\u0027ll be able to consolidate telemetry and logging in the future, and starting with span sounds great.","commit_id":"30c046842459891d03654d6ceeaa46fa3f27bd57"},{"author":{"_account_id":1,"name":"James E. Blair","email":"jim@acmegating.com","username":"corvus"},"change_message_id":"d2df83744a1046f483c43ebd4e6c92f82ce71156","unresolved":false,"context_lines":[{"line_number":50,"context_line":""},{"line_number":51,"context_line":"Usage of OpenTelemetry should be entirely optional and supplementary"},{"line_number":52,"context_line":"for any Zuul deployment.  Log messages alone should continue to be"},{"line_number":53,"context_line":"sufficient to analyze any potential problem."},{"line_number":54,"context_line":""},{"line_number":55,"context_line":"Should a deployer wish to use OpenTelemetry tracing data, a very"},{"line_number":56,"context_line":"simple deployment for smaller sites may be constructed by running only"}],"source_content_type":"text/x-rst","patch_set":1,"id":"788f0bf1_bbcc07a8","line":53,"in_reply_to":"7e93b10e_565644c5","updated":"2022-08-03 00:49:07.000000000","message":"OpenTelemetry describes a system for exporting three types of data: traces, metrics, logs.  This spec only proposes that we use the OpenTelemetry protocol to export spans, not log messages or metrics (since we already have infrastructure for the others).\n\nOpenTelemetry spans can also have event information (which can look a little bit like a log entry from a data structure point of view, but might be rendered visually as a span with no duration).  We can associate some events with spans but we shouldn\u0027t over-use that (See the \"Events and Exceptions\" section in this spec).\n\nI do think a lot of the interesting information in logs will appear in spans (request ids, etc), but it will be in a different form, so I don\u0027t think there needs to be overlap with logs.  If we say that, for example, the beginning and ending of a merge request span is redundant with the existing log entries, we\u0027re probably talking about 10-20 log invocations that would be redundant with spans out of the 1400 log invocations Zuul currently has.\n\nIt is possible that we could decide that using OpenTelemetry to export log entries instead of the python logging library, however, there are many different ways people are configuring logging right now (console, file, elasticsearch, splunk, etc) all of which we would need to continue to support, preferably without requiring any additional tooling.  I think it would be best to get some experience with OpenTelemetry for tracing before we decide to ask it to do more.","commit_id":"30c046842459891d03654d6ceeaa46fa3f27bd57"},{"author":{"_account_id":1,"name":"James E. Blair","email":"jim@acmegating.com","username":"corvus"},"change_message_id":"2ec40ee81f74cbe7cd459fa1b660034e18ea02aa","unresolved":false,"context_lines":[{"line_number":50,"context_line":""},{"line_number":51,"context_line":"Usage of OpenTelemetry should be entirely optional and supplementary"},{"line_number":52,"context_line":"for any Zuul deployment.  Log messages alone should continue to be"},{"line_number":53,"context_line":"sufficient to analyze any potential problem."},{"line_number":54,"context_line":""},{"line_number":55,"context_line":"Should a deployer wish to use OpenTelemetry tracing data, a very"},{"line_number":56,"context_line":"simple deployment for smaller sites may be constructed by running only"}],"source_content_type":"text/x-rst","patch_set":1,"id":"0ff22820_9db506e9","line":53,"in_reply_to":"bd4ef1c6_84224376","updated":"2022-08-02 14:59:58.000000000","message":"This spec says that we will only use OpenTelemetry for emitting spans, not logs; we\u0027ll continue to use the existing python logging framework for that.  I definitely don\u0027t want to duplicate log infrastructure.\n\n(It\u0027s perfectly fine to only emit spans with Open Telemetry, and ignore the logs (and metrics) it supports.)","commit_id":"30c046842459891d03654d6ceeaa46fa3f27bd57"},{"author":{"_account_id":5263,"name":"Jeremy Stanley","display_name":"fungi","email":"fungi@yuggoth.org","username":"fungi","status":"missing, presumed fed"},"change_message_id":"9b71d9b1607388d43a4efe5791090dd0d130c33b","unresolved":false,"context_lines":[{"line_number":122,"context_line":""},{"line_number":123,"context_line":"Because Zuul processes series of events which may stretch for long"},{"line_number":124,"context_line":"periods of time, we should specify what events and actions should"},{"line_number":125,"context_line":"correspond to spans and traces.  Spans can have arbitrary metadat"},{"line_number":126,"context_line":"associated with them, so we will be able to search by event or job"},{"line_number":127,"context_line":"ids."},{"line_number":128,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"a2e819a6_0b1ee47a","line":125,"updated":"2022-08-02 16:02:38.000000000","message":"Nit: \"metadata\"","commit_id":"30c046842459891d03654d6ceeaa46fa3f27bd57"},{"author":{"_account_id":5263,"name":"Jeremy Stanley","display_name":"fungi","email":"fungi@yuggoth.org","username":"fungi","status":"missing, presumed fed"},"change_message_id":"9b71d9b1607388d43a4efe5791090dd0d130c33b","unresolved":false,"context_lines":[{"line_number":124,"context_line":"periods of time, we should specify what events and actions should"},{"line_number":125,"context_line":"correspond to spans and traces.  Spans can have arbitrary metadat"},{"line_number":126,"context_line":"associated with them, so we will be able to search by event or job"},{"line_number":127,"context_line":"ids."},{"line_number":128,"context_line":""},{"line_number":129,"context_line":"The following sections describe traces and their child spans."},{"line_number":130,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"203b029e_a64bbd63","line":127,"updated":"2022-08-02 16:02:38.000000000","message":"Nit: elswhere you capitalize as \"IDs\"","commit_id":"30c046842459891d03654d6ceeaa46fa3f27bd57"},{"author":{"_account_id":5263,"name":"Jeremy Stanley","display_name":"fungi","email":"fungi@yuggoth.org","username":"fungi","status":"missing, presumed fed"},"change_message_id":"9b71d9b1607388d43a4efe5791090dd0d130c33b","unresolved":false,"context_lines":[{"line_number":128,"context_line":""},{"line_number":129,"context_line":"The following sections describe traces and their child spans."},{"line_number":130,"context_line":""},{"line_number":131,"context_line":"Event Ingestion"},{"line_number":132,"context_line":"+++++++++++++++"},{"line_number":133,"context_line":""},{"line_number":134,"context_line":"A trace will begin when Zuul receives an event and end when that event"}],"source_content_type":"text/x-rst","patch_set":1,"id":"f33e20ec_d697fb4c","line":131,"updated":"2022-08-02 16:02:38.000000000","message":"Was this meant to be underlined?","commit_id":"30c046842459891d03654d6ceeaa46fa3f27bd57"},{"author":{"_account_id":1,"name":"James E. Blair","email":"jim@acmegating.com","username":"corvus"},"change_message_id":"708876c59edf37eab135b175c1e5837f5e78036d","unresolved":false,"context_lines":[{"line_number":128,"context_line":""},{"line_number":129,"context_line":"The following sections describe traces and their child spans."},{"line_number":130,"context_line":""},{"line_number":131,"context_line":"Event Ingestion"},{"line_number":132,"context_line":"+++++++++++++++"},{"line_number":133,"context_line":""},{"line_number":134,"context_line":"A trace will begin when Zuul receives an event and end when that event"}],"source_content_type":"text/x-rst","patch_set":1,"id":"88344272_d82f1e70","line":131,"in_reply_to":"f33e20ec_d697fb4c","updated":"2022-08-02 20:56:44.000000000","message":"This is the next level of topic under Spans.  So the Tree is:\n\nTracing\n+ Proposed Solution\n+ + Spans\n+ + + Event Ingestion\n+ + + Tenant Event Processing\n...","commit_id":"30c046842459891d03654d6ceeaa46fa3f27bd57"},{"author":{"_account_id":5263,"name":"Jeremy Stanley","display_name":"fungi","email":"fungi@yuggoth.org","username":"fungi","status":"missing, presumed fed"},"change_message_id":"9b71d9b1607388d43a4efe5791090dd0d130c33b","unresolved":false,"context_lines":[{"line_number":143,"context_line":"these processing steps/queues should appear as their own child spans."},{"line_number":144,"context_line":"The spans should include event IDs (and potentially other information"},{"line_number":145,"context_line":"about the event such as change or pull request numbers) as metadata."},{"line_number":146,"context_line":""},{"line_number":147,"context_line":"Tenant Event Processing"},{"line_number":148,"context_line":"+++++++++++++++++++++++"},{"line_number":149,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"80f60c04_d0d5dcd8","line":146,"updated":"2022-08-02 16:02:38.000000000","message":"And this?","commit_id":"30c046842459891d03654d6ceeaa46fa3f27bd57"},{"author":{"_account_id":5263,"name":"Jeremy Stanley","display_name":"fungi","email":"fungi@yuggoth.org","username":"fungi","status":"missing, presumed fed"},"change_message_id":"9b71d9b1607388d43a4efe5791090dd0d130c33b","unresolved":false,"context_lines":[{"line_number":163,"context_line":""},{"line_number":164,"context_line":"Within the root span, there will be a span for each buildset (so that"},{"line_number":165,"context_line":"if a gate reset happens and a new buildset is created, users will see"},{"line_number":166,"context_line":"a series of buildset spans).  Within a buildset, there will be spans"},{"line_number":167,"context_line":"for all of the major processing steps, such as merge operations,"},{"line_number":168,"context_line":"layout calculating, freezing the job graph, and freezing jobs.  Each"},{"line_number":169,"context_line":"build will also merit a span (retried builds will get their own spans"}],"source_content_type":"text/x-rst","patch_set":1,"id":"8724b949_923e11f0","line":166,"updated":"2022-08-02 16:02:38.000000000","message":"Is the plan for node requests to be represented here as well (at least the scheduler-side of issuing the request and then eventually receiving an assignment or not, and any reissuing)? At a minimum, being able to suss out which launcher(s) handled a request without needing to dig in the scheduler log would be nice, though if the traces also cover nodepool itself this could be even more useful.","commit_id":"30c046842459891d03654d6ceeaa46fa3f27bd57"},{"author":{"_account_id":1,"name":"James E. Blair","email":"jim@acmegating.com","username":"corvus"},"change_message_id":"708876c59edf37eab135b175c1e5837f5e78036d","unresolved":false,"context_lines":[{"line_number":163,"context_line":""},{"line_number":164,"context_line":"Within the root span, there will be a span for each buildset (so that"},{"line_number":165,"context_line":"if a gate reset happens and a new buildset is created, users will see"},{"line_number":166,"context_line":"a series of buildset spans).  Within a buildset, there will be spans"},{"line_number":167,"context_line":"for all of the major processing steps, such as merge operations,"},{"line_number":168,"context_line":"layout calculating, freezing the job graph, and freezing jobs.  Each"},{"line_number":169,"context_line":"build will also merit a span (retried builds will get their own spans"}],"source_content_type":"text/x-rst","patch_set":1,"id":"3886e3cc_2cb7c062","line":166,"in_reply_to":"8724b949_923e11f0","updated":"2022-08-02 20:56:44.000000000","message":"Yes for the scheduler view of a node request as you describe.  I believe we should be able to associate the launcher that fulfilled the request (also, if we want, all the launchers that declined it) with the span.\n\nWe could also forward the span information to Nodepool so that it can open up its own spans, but I\u0027m not anticipating that right now in this spec.  At least partly because I have another spec I\u0027m working on that would fold Nodepool into Zuul.\n\nIf we retain Nodepool as a separate app, I expect us to extend tracing support to it eventually.  If we fold it into Zuul, we\u0027ll probably do the same.\n\nBut to keep things simple for now, let\u0027s just say this covers the \"scheduler view\" of a node request.","commit_id":"30c046842459891d03654d6ceeaa46fa3f27bd57"},{"author":{"_account_id":4146,"name":"Clark Boylan","email":"cboylan@sapwetik.org","username":"cboylan"},"change_message_id":"670f0d84d9998a79dc1a47b28f0cf8c91655fe7c","unresolved":true,"context_lines":[{"line_number":182,"context_line":"might start a buildset span while another ends it is problematic."},{"line_number":183,"context_line":""},{"line_number":184,"context_line":"Fortunately, the OpenTelemetry API only reports spans when they end,"},{"line_number":185,"context_line":"not when they start.  This means that we don\u0027t need to coordinate a"},{"line_number":186,"context_line":"\"start\" API call on one scheduler with an \"end\" API call on another."},{"line_number":187,"context_line":"We can simply emit the trace with its root span at the end.  However,"},{"line_number":188,"context_line":"any child spans emitted during that time need to know the trace ID"}],"source_content_type":"text/x-rst","patch_set":1,"id":"6a2fe777_9fdc452a","line":185,"updated":"2022-08-05 17:02:29.000000000","message":"Does this mean any incomplete spans (system crash or extremely slow processing) won\u0027t be visible?","commit_id":"30c046842459891d03654d6ceeaa46fa3f27bd57"},{"author":{"_account_id":1,"name":"James E. Blair","email":"jim@acmegating.com","username":"corvus"},"change_message_id":"a66d8348576b0e1ed5f49415ba1df31ed0a6fcd0","unresolved":false,"context_lines":[{"line_number":182,"context_line":"might start a buildset span while another ends it is problematic."},{"line_number":183,"context_line":""},{"line_number":184,"context_line":"Fortunately, the OpenTelemetry API only reports spans when they end,"},{"line_number":185,"context_line":"not when they start.  This means that we don\u0027t need to coordinate a"},{"line_number":186,"context_line":"\"start\" API call on one scheduler with an \"end\" API call on another."},{"line_number":187,"context_line":"We can simply emit the trace with its root span at the end.  However,"},{"line_number":188,"context_line":"any child spans emitted during that time need to know the trace ID"}],"source_content_type":"text/x-rst","patch_set":1,"id":"a51ad8d3_cb744634","line":185,"in_reply_to":"6a2fe777_9fdc452a","updated":"2022-08-05 17:10:32.000000000","message":"Yes, basically the protocol just sends everything at the end of a span.  So if we crash before the end of a span, it won\u0027t make it to the storage system.\n\nHowever, for us, that\u0027s only true for a span that starts and ends on the same system (imagine a merge job).  Since some of our spans start and end on different systems (imagine a queue item), those would be crash-resilient since we\u0027re storing data in ZK for them.\n\nChild spans with no parent (parent crashes but child doesn\u0027t) will be visible (they show up with warning messages in jaeger).","commit_id":"30c046842459891d03654d6ceeaa46fa3f27bd57"},{"author":{"_account_id":27582,"name":"Simon Westphahl","email":"simon.westphahl@bmw.de","username":"simon.westphahl"},"change_message_id":"353d07485ce8285c82763588499784506ea29fc9","unresolved":true,"context_lines":[{"line_number":190,"context_line":"and start timestamp on our starting scheduler for use by any child"},{"line_number":191,"context_line":"spans as well as the \"end span\" API call."},{"line_number":192,"context_line":""},{"line_number":193,"context_line":"The SDK does not support creating a span with a specific trace ID or"},{"line_number":194,"context_line":"start timestamp (most timestamps are automatic), but it has"},{"line_number":195,"context_line":"well-defined interfaces for spans and we can subclass the"},{"line_number":196,"context_line":"implementation to allow us to specify trace IDs and timestamps.  With"}],"source_content_type":"text/x-rst","patch_set":1,"id":"aa0249a1_4213bcf3","line":193,"updated":"2022-07-26 14:03:42.000000000","message":"Is this also true for the trace ID? Usually the trace ID is what connects all the spans and you can definitely propagate the trace context (which has the trace ID) and create a new span from that.\n\nI think a missing piece is that you can (de-)serialize a span, which we probably need.","commit_id":"30c046842459891d03654d6ceeaa46fa3f27bd57"},{"author":{"_account_id":1,"name":"James E. Blair","email":"jim@acmegating.com","username":"corvus"},"change_message_id":"fbe728e9bd48b6cc32b9aae0b24f03cb675466e6","unresolved":false,"context_lines":[{"line_number":190,"context_line":"and start timestamp on our starting scheduler for use by any child"},{"line_number":191,"context_line":"spans as well as the \"end span\" API call."},{"line_number":192,"context_line":""},{"line_number":193,"context_line":"The SDK does not support creating a span with a specific trace ID or"},{"line_number":194,"context_line":"start timestamp (most timestamps are automatic), but it has"},{"line_number":195,"context_line":"well-defined interfaces for spans and we can subclass the"},{"line_number":196,"context_line":"implementation to allow us to specify trace IDs and timestamps.  With"}],"source_content_type":"text/x-rst","patch_set":1,"id":"388389fb_3f3f7607","line":193,"in_reply_to":"aa0249a1_4213bcf3","updated":"2022-07-26 14:25:20.000000000","message":"Given a root span for a trace, you can serialize/deserialize its trace context so that other systems can make child traces from it.  But the deserialization doesn\u0027t give you the root span, it gives you the trace context for the root span, which is only useful for creating child spans, not for emitting the root span itself.  The issue is that our root spans start on one host and end on another, and there isn\u0027t a way to serialize the span itself.  Here\u0027s what we want to do:\n\n1) Start root span on scheduler A\n2) Send trace context to merger\n3) Merger emits child span\n4) End root span on scheduler B and emit root span\n\nThat would work if we only had one scheduler, because the root span could be kept in memory.  But with multiple schedulers, and no way to serialize/deserialize the root span, we can\u0027t transfer the root span from step 1 to step 4.  So we need to augment the SDK somehow.\n\nWe could implement span serialization ourselves, but we don\u0027t actually need to do that, all we really need is the trace id, root span id, and timestamp of when the root span started.  I guess it is a form of serialization, but it\u0027s verify lightweight (we will only store that info, and not everything else like the list of errors, logs, events, etc).","commit_id":"30c046842459891d03654d6ceeaa46fa3f27bd57"},{"author":{"_account_id":27582,"name":"Simon Westphahl","email":"simon.westphahl@bmw.de","username":"simon.westphahl"},"change_message_id":"626bdcdecaa5251e3074e0a2f95a1d4abc7d528e","unresolved":true,"context_lines":[{"line_number":198,"context_line":"information in ZooKeeper with whatever long-lived object it is"},{"line_number":199,"context_line":"associated with (such as a QueueItem) and then make it concrete on"},{"line_number":200,"context_line":"another host when we end it."},{"line_number":201,"context_line":""},{"line_number":202,"context_line":"Alternatives"},{"line_number":203,"context_line":"++++++++++++"},{"line_number":204,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"fba1cc24_d5b5cf4f","line":201,"updated":"2022-07-26 14:07:59.000000000","message":"Maybe something else worth exploring is how we want to configure this. When I wrote the initial prototype there was quite some config necessary depending on which exporter was used. But maybe that\u0027s a non-issue if we only support the OLTP exporter.","commit_id":"30c046842459891d03654d6ceeaa46fa3f27bd57"},{"author":{"_account_id":1,"name":"James E. Blair","email":"jim@acmegating.com","username":"corvus"},"change_message_id":"fbe728e9bd48b6cc32b9aae0b24f03cb675466e6","unresolved":false,"context_lines":[{"line_number":198,"context_line":"information in ZooKeeper with whatever long-lived object it is"},{"line_number":199,"context_line":"associated with (such as a QueueItem) and then make it concrete on"},{"line_number":200,"context_line":"another host when we end it."},{"line_number":201,"context_line":""},{"line_number":202,"context_line":"Alternatives"},{"line_number":203,"context_line":"++++++++++++"},{"line_number":204,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"862d45be_a63b89d4","line":201,"in_reply_to":"fba1cc24_d5b5cf4f","updated":"2022-07-26 14:25:20.000000000","message":"Yeah, with only the OLTP exporter, it\u0027s a pretty small set of config options (just what you\u0027d expect: host, port, maybe protocol).  I don\u0027t see us needing more than an ini section with a few options.","commit_id":"30c046842459891d03654d6ceeaa46fa3f27bd57"},{"author":{"_account_id":5263,"name":"Jeremy Stanley","display_name":"fungi","email":"fungi@yuggoth.org","username":"fungi","status":"missing, presumed fed"},"change_message_id":"9b71d9b1607388d43a4efe5791090dd0d130c33b","unresolved":false,"context_lines":[{"line_number":277,"context_line":""},{"line_number":278,"context_line":"No sensitive information (secrets, passwords, job variables, etc)"},{"line_number":279,"context_line":"should be included in tracing output.  All output should be suitable"},{"line_number":280,"context_line":"for an audience of Zuul users (that is, if someone has access to the"},{"line_number":281,"context_line":"Zuul dashboard, then tracing data should not have any more sensitive"},{"line_number":282,"context_line":"information than they already have access to).  For public-facing Zuul"},{"line_number":283,"context_line":"systems (such as OpenDev), the information should be suitable for"}],"source_content_type":"text/x-rst","patch_set":1,"id":"2d813751_932c8d0f","line":280,"updated":"2022-08-02 16:02:38.000000000","message":"Would this be segmented by tenant, in order to accommodate tenant-specific access controls and avoid leaking information about one to users who have access to another?","commit_id":"30c046842459891d03654d6ceeaa46fa3f27bd57"},{"author":{"_account_id":1,"name":"James E. Blair","email":"jim@acmegating.com","username":"corvus"},"change_message_id":"708876c59edf37eab135b175c1e5837f5e78036d","unresolved":false,"context_lines":[{"line_number":277,"context_line":""},{"line_number":278,"context_line":"No sensitive information (secrets, passwords, job variables, etc)"},{"line_number":279,"context_line":"should be included in tracing output.  All output should be suitable"},{"line_number":280,"context_line":"for an audience of Zuul users (that is, if someone has access to the"},{"line_number":281,"context_line":"Zuul dashboard, then tracing data should not have any more sensitive"},{"line_number":282,"context_line":"information than they already have access to).  For public-facing Zuul"},{"line_number":283,"context_line":"systems (such as OpenDev), the information should be suitable for"}],"source_content_type":"text/x-rst","patch_set":1,"id":"ce9c893e_f03e737e","line":280,"in_reply_to":"2d813751_932c8d0f","updated":"2022-08-02 20:56:44.000000000","message":"We can include the metadata about the tenant so that anyone with tooling capable of acting on that has the information, but I\u0027m unaware of the ability of any systems that present tracing data to limit it like that (maybe Kibana could?).  That\u0027s probably a good caveat to include if we add any deployment suggestions.","commit_id":"30c046842459891d03654d6ceeaa46fa3f27bd57"},{"author":{"_account_id":4146,"name":"Clark Boylan","email":"cboylan@sapwetik.org","username":"cboylan"},"change_message_id":"670f0d84d9998a79dc1a47b28f0cf8c91655fe7c","unresolved":true,"context_lines":[{"line_number":292,"context_line":"Protobuf applications include automatically generated code, we may"},{"line_number":293,"context_line":"encounter the occasional version inconsistency.  We may need to"},{"line_number":294,"context_line":"navigate package requirements more than normal due to this (especially"},{"line_number":295,"context_line":"if we have multiple packages that depend on protobuf)."},{"line_number":296,"context_line":""},{"line_number":297,"context_line":"For a contemporary example, the OpenTelemetry project is in the"},{"line_number":298,"context_line":"process of pinning to an older version of protobuf:"}],"source_content_type":"text/x-rst","patch_set":1,"id":"00ae8f83_1bd257d0","line":295,"updated":"2022-08-05 17:02:29.000000000","message":"Does this also affect how we package zuul? I\u0027m not sure PBR is used with any C compilation steps currently, but I think that would be necessary to build our protobuf definitions? Or are the sufficiently abstract that the existing libs define and build them and we just shove data into them?","commit_id":"30c046842459891d03654d6ceeaa46fa3f27bd57"},{"author":{"_account_id":1,"name":"James E. Blair","email":"jim@acmegating.com","username":"corvus"},"change_message_id":"a66d8348576b0e1ed5f49415ba1df31ed0a6fcd0","unresolved":false,"context_lines":[{"line_number":292,"context_line":"Protobuf applications include automatically generated code, we may"},{"line_number":293,"context_line":"encounter the occasional version inconsistency.  We may need to"},{"line_number":294,"context_line":"navigate package requirements more than normal due to this (especially"},{"line_number":295,"context_line":"if we have multiple packages that depend on protobuf)."},{"line_number":296,"context_line":""},{"line_number":297,"context_line":"For a contemporary example, the OpenTelemetry project is in the"},{"line_number":298,"context_line":"process of pinning to an older version of protobuf:"}],"source_content_type":"text/x-rst","patch_set":1,"id":"534bcb5f_33fd488b","line":295,"in_reply_to":"00ae8f83_1bd257d0","updated":"2022-08-05 17:10:32.000000000","message":"That should be included in the packages we depend on (ie, we won\u0027t need any custom protobuf stuff for zuul, only what\u0027s in the opentelemetry package).  So if that\u0027s already in a wheel, then we shouldn\u0027t notice it.  But absent a wheel, we might be performing a build step for opentelementry?  I\u0027m not sure about this, I didn\u0027t look too closely.","commit_id":"30c046842459891d03654d6ceeaa46fa3f27bd57"},{"author":{"_account_id":9311,"name":"Tristan Cacqueray","email":"tdecacqu@redhat.com","username":"tristanC"},"change_message_id":"7ee932f57e6cc438e2fcbcc81e727767f3f84847","unresolved":true,"context_lines":[{"line_number":311,"context_line":"  different hosts"},{"line_number":312,"context_line":"* Instrument event processing in each driver"},{"line_number":313,"context_line":"* Instrument event processing in scheduler"},{"line_number":314,"context_line":"* Instrument queue items and related spans"},{"line_number":315,"context_line":"* Document a simple Jaeger setup as a quickstart add-on (similar to"},{"line_number":316,"context_line":"  authz)"},{"line_number":317,"context_line":"* Optional: work with OpenDev to run a public Jaeger server for"}],"source_content_type":"text/x-rst","patch_set":1,"id":"39d8a092_7e0a44de","line":314,"updated":"2022-08-02 13:28:19.000000000","message":"It seems like opentelemetry-python metrics are compatible with prometheus client, and perhaps it would be preferable to migrate the existing gauge to use the same object for both use-case?","commit_id":"30c046842459891d03654d6ceeaa46fa3f27bd57"},{"author":{"_account_id":1,"name":"James E. Blair","email":"jim@acmegating.com","username":"corvus"},"change_message_id":"708876c59edf37eab135b175c1e5837f5e78036d","unresolved":false,"context_lines":[{"line_number":311,"context_line":"  different hosts"},{"line_number":312,"context_line":"* Instrument event processing in each driver"},{"line_number":313,"context_line":"* Instrument event processing in scheduler"},{"line_number":314,"context_line":"* Instrument queue items and related spans"},{"line_number":315,"context_line":"* Document a simple Jaeger setup as a quickstart add-on (similar to"},{"line_number":316,"context_line":"  authz)"},{"line_number":317,"context_line":"* Optional: work with OpenDev to run a public Jaeger server for"}],"source_content_type":"text/x-rst","patch_set":1,"id":"b3efdf30_6180a9f5","line":314,"in_reply_to":"39d8a092_7e0a44de","updated":"2022-08-02 20:56:44.000000000","message":"I agree there may be an opportunity for future consolidation.  But I don\u0027t want to encode that in this spec so that we keep all the functions separate for the moment.","commit_id":"30c046842459891d03654d6ceeaa46fa3f27bd57"}]}