From 4674a821d8bf6f17075165df8ece98b63aa86d3f Mon Sep 17 00:00:00 2001 From: Jeremy Wall Date: Fri, 16 Feb 2024 17:23:11 -0500 Subject: [PATCH] feat: multiple subplots per graph --- examples/example_dashboards.yaml | 38 ++++++--- src/dashboard.rs | 57 ++++++++----- src/query.rs | 70 ++++++++++++---- src/routes.rs | 29 +++---- static/lib.js | 138 +++++++++++++++++-------------- 5 files changed, 207 insertions(+), 125 deletions(-) diff --git a/examples/example_dashboards.yaml b/examples/example_dashboards.yaml index 676887d..c5f3320 100644 --- a/examples/example_dashboards.yaml +++ b/examples/example_dashboards.yaml @@ -2,29 +2,45 @@ - title: Test Dasbboard 1 graphs: - title: Node cpu - source: http://heimdall:9001 - query: 'sum by (instance)(irate(node_cpu_seconds_total{job="nodestats"}[5m]))' + d3_tick_format: "~s" + plots: + - source: http://heimdall:9001 + query: 'sum by (instance)(irate(node_cpu_seconds_total{job="nodestats"}[5m]))' + meta: + name_label: instance query_type: Range span: end: now duration: 1d step_duration: 10min - name_label: instance - title: Test Dasbboard 2 span: end: 2024-02-10T00:00:00.00Z duration: 2 days step_duration: 1 minute graphs: - - title: Node cpu sytem percent - source: http://heimdall:9001 - query: | - sum by (instance)(irate(node_cpu_seconds_total{mode="system",job="nodestats"}[5m])) / sum by (instance)(irate(node_cpu_seconds_total{job="nodestats"}[5m])) + - title: Node cpu percent d3_tick_format: "~%" query_type: Range - name_label: instance + plots: + - source: http://heimdall:9001 + query: | + sum by (instance)(irate(node_cpu_seconds_total{mode="system",job="nodestats"}[5m])) / sum by (instance)(irate(node_cpu_seconds_total{job="nodestats"}[5m])) + meta: + d3_tick_format: "~s" + name_label: instance + name_prefix: "System" + - source: http://heimdall:9001 + query: | + sum by (instance)(irate(node_cpu_seconds_total{mode="user",job="nodestats"}[5m])) / sum by (instance)(irate(node_cpu_seconds_total{job="nodestats"}[5m])) + meta: + #d3_tick_format: "~s" + name_label: instance + name_suffix: "User" - title: Node memory - source: http://heimdall:9001 - query: 'node_memory_MemFree_bytes{job="nodestats"}' query_type: Scalar - name_label: instance + plots: + - source: http://heimdall:9001 + query: 'node_memory_MemFree_bytes{job="nodestats"}' + meta: + name_label: instance diff --git a/src/dashboard.rs b/src/dashboard.rs index 4a2e859..db43b4a 100644 --- a/src/dashboard.rs +++ b/src/dashboard.rs @@ -19,10 +19,11 @@ use serde::Deserialize; use serde_yaml; use tracing::{debug, error}; -use crate::query::{QueryConn, QueryType}; +use crate::query::{QueryConn, QueryType, PlotMeta}; #[derive(Deserialize, Debug)] pub struct GraphSpan { + // serialized with https://datatracker.ietf.org/doc/html/rfc3339 and special handling for 'now' pub end: String, pub duration: String, pub step_duration: String, @@ -32,17 +33,21 @@ pub struct GraphSpan { pub struct Dashboard { pub title: String, pub graphs: Vec, - pub span: Option + pub span: Option, +} + +#[derive(Deserialize)] +pub struct SubPlot { + pub source: String, + pub query: String, + pub meta: PlotMeta, } #[derive(Deserialize)] pub struct Graph { pub title: String, - pub source: String, - pub query: String, - // serialized with https://datatracker.ietf.org/doc/html/rfc3339 + pub plots: Vec, pub span: Option, - pub name_label: String, pub query_type: QueryType, pub d3_tick_format: Option, } @@ -97,23 +102,31 @@ fn graph_span_to_tuple(span: &Option) -> Option<(DateTime, Durat } impl Graph { - pub fn get_query_connection<'conn, 'graph: 'conn>(&'graph self, graph_span: &'graph Option, query_span: &'graph Option) -> QueryConn<'conn> { - debug!( - query = self.query, - source = self.source, - "Getting query connection for graph" - ); - let mut conn = QueryConn::new(&self.source, &self.query, self.query_type.clone()); - // Query params take precendence over all other settings. Then graph settings take - // precedences and finally the dashboard settings take precendence - if let Some((end, duration, step_duration)) = graph_span_to_tuple(query_span) { - conn = conn.with_span(end, duration, step_duration); - } else if let Some((end, duration, step_duration)) = graph_span_to_tuple(&self.span) { - conn = conn.with_span(end, duration, step_duration); - } else if let Some((end, duration, step_duration)) = graph_span_to_tuple(graph_span) { - conn = conn.with_span(end, duration, step_duration); + pub fn get_query_connections<'conn, 'graph: 'conn>( + &'graph self, + graph_span: &'graph Option, + query_span: &'graph Option, + ) -> Vec> { + let mut conns = Vec::new(); + for plot in self.plots.iter() { + debug!( + query = plot.query, + source = plot.source, + "Getting query connection for graph" + ); + let mut conn = QueryConn::new(&plot.source, &plot.query, self.query_type.clone(), plot.meta.clone()); + // Query params take precendence over all other settings. Then graph settings take + // precedences and finally the dashboard settings take precendence + if let Some((end, duration, step_duration)) = graph_span_to_tuple(query_span) { + conn = conn.with_span(end, duration, step_duration); + } else if let Some((end, duration, step_duration)) = graph_span_to_tuple(&self.span) { + conn = conn.with_span(end, duration, step_duration); + } else if let Some((end, duration, step_duration)) = graph_span_to_tuple(graph_span) { + conn = conn.with_span(end, duration, step_duration); + } + conns.push(conn); } - conn + conns } } diff --git a/src/query.rs b/src/query.rs index 20cca44..f41844f 100644 --- a/src/query.rs +++ b/src/query.rs @@ -38,20 +38,31 @@ pub struct QueryConn<'conn> { query: &'conn str, span: Option, query_type: QueryType, + pub meta: PlotMeta, } impl<'conn> QueryConn<'conn> { - pub fn new<'a: 'conn>(source: &'a str, query: &'a str, query_type: QueryType) -> Self { + pub fn new<'a: 'conn>(source: &'a str, query: &'a str, query_type: QueryType, meta: PlotMeta) -> Self { Self { source, query, query_type, + meta, span: None, } } - pub fn with_span(mut self, end: DateTime, duration: chrono::Duration, step: chrono::Duration) -> Self { - self.span = Some(TimeSpan { end, duration, step_seconds: step.num_seconds() , }); + pub fn with_span( + mut self, + end: DateTime, + duration: chrono::Duration, + step: chrono::Duration, + ) -> Self { + self.span = Some(TimeSpan { + end, + duration, + step_seconds: step.num_seconds(), + }); self } @@ -64,25 +75,35 @@ impl<'conn> QueryConn<'conn> { step_seconds, }) = self.span { - let start = end - du; - debug!(?start, ?end, step_seconds, "Running Query with range values"); + let start = end - du; + debug!( + ?start, + ?end, + step_seconds, + "Running Query with range values" + ); (start.timestamp(), end.timestamp(), step_seconds as f64) } else { let end = Utc::now(); let start = end - chrono::Duration::minutes(10); - debug!(?start, ?end, step_seconds=30, "Running Query with range values"); + debug!( + ?start, + ?end, + step_seconds = 30, + "Running Query with range values" + ); (start.timestamp(), end.timestamp(), 30 as f64) }; //debug!(start, end, step_resolution, "Running Query with range values"); match self.query_type { QueryType::Range => { let results = client - .query_range(self.query, start, end, step_resolution) - .get() - .await?; + .query_range(self.query, start, end, step_resolution) + .get() + .await?; //debug!(?results, "range results"); Ok(results) - }, + } QueryType::Scalar => Ok(client.query(self.query).get().await?), } } @@ -94,19 +115,33 @@ pub struct DataPoint { value: f64, } +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct PlotMeta { + name_prefix: Option, + name_suffix: Option, + name_label: Option, + d3_tick_format: Option, +} + #[derive(Serialize, Deserialize)] pub enum QueryResult { - Series(Vec<(HashMap, Vec)>), - Scalar(Vec<(HashMap, DataPoint)>), + Series(Vec<(HashMap, PlotMeta, Vec)>), + Scalar(Vec<(HashMap, PlotMeta, DataPoint)>), } impl std::fmt::Debug for QueryResult { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { - QueryResult::Series(v) => { + QueryResult::Series(v) => { f.write_fmt(format_args!("Series trace count = {}", v.len()))?; - for (idx, (tags, trace)) in v.iter().enumerate() { - f.write_fmt(format_args!("; {}: meta {:?} datapoint count = {};", idx, tags, trace.len()))?; + for (idx, (tags, meta, trace)) in v.iter().enumerate() { + f.write_fmt(format_args!( + "; {}: tags {:?} meta: {:?} datapoint count = {};", + idx, + tags, + meta, + trace.len() + ))?; } } QueryResult::Scalar(v) => { @@ -117,7 +152,7 @@ impl std::fmt::Debug for QueryResult { } } -pub fn to_samples(data: Data) -> QueryResult { +pub fn to_samples(data: Data, meta: PlotMeta) -> QueryResult { match data { Data::Matrix(mut range) => QueryResult::Series( range @@ -126,6 +161,7 @@ pub fn to_samples(data: Data) -> QueryResult { let (metric, mut samples) = rv.into_inner(); ( metric, + meta.clone(), samples .drain(0..) .map(|s| DataPoint { @@ -144,6 +180,7 @@ pub fn to_samples(data: Data) -> QueryResult { let (metric, sample) = iv.into_inner(); ( metric, + meta.clone(), DataPoint { timestamp: sample.timestamp(), value: sample.value(), @@ -154,6 +191,7 @@ pub fn to_samples(data: Data) -> QueryResult { ), Data::Scalar(sample) => QueryResult::Scalar(vec![( HashMap::new(), + meta.clone(), DataPoint { timestamp: sample.timestamp(), value: sample.value(), diff --git a/src/routes.rs b/src/routes.rs index ea7cfc7..ae052bc 100644 --- a/src/routes.rs +++ b/src/routes.rs @@ -33,7 +33,7 @@ pub async fn graph_query( State(config): Config, Path((dash_idx, graph_idx)): Path<(usize, usize)>, Query(query): Query>, -) -> Json { +) -> Json> { debug!("Getting data for query"); let dash = config.get(dash_idx).expect("No such dashboard index"); let graph = dash @@ -45,7 +45,6 @@ pub async fn graph_query( && query.contains_key("duration") && query.contains_key("step_duration") { - // TODO(jwall): handle the now case. Some(GraphSpan { end: query["end"].clone(), duration: query["duration"].clone(), @@ -55,21 +54,23 @@ pub async fn graph_query( None } }; - let data = to_samples( - graph - .get_query_connection(&dash.span, &query_span) - .get_results() - .await - .expect("Unable to get query results") - .data() - .clone(), - ); + let connections = graph.get_query_connections(&dash.span, &query_span); + let mut data = Vec::new(); + for conn in connections { + data.push(to_samples( + conn.get_results() + .await + .expect("Unable to get query results") + .data() + .clone(), + conn.meta, + )); + } Json(data) } pub fn mk_api_routes(config: Arc>) -> Router { // Query routes - // TODO(zaphar): Allow passing the timespan in via query Router::new().route( "/dash/:dash_idx/graph/:graph_idx", get(graph_query).with_state(config), @@ -83,9 +84,9 @@ pub fn graph_component(dash_idx: usize, graph_idx: usize, graph: &Graph) -> Mark div { h2 { (graph.title) } @if graph.d3_tick_format.is_some() { - timeseries-graph uri=(graph_data_uri) id=(graph_id) label=(graph.name_label) d3-tick-format=(graph.d3_tick_format.as_ref().unwrap()) { } + timeseries-graph uri=(graph_data_uri) id=(graph_id) d3-tick-format=(graph.d3_tick_format.as_ref().unwrap()) { } } @else { - timeseries-graph uri=(graph_data_uri) id=(graph_id) label=(graph.name_label) { } + timeseries-graph uri=(graph_data_uri) id=(graph_id) { } } } ) diff --git a/static/lib.js b/static/lib.js index 37733da..692d597 100644 --- a/static/lib.js +++ b/static/lib.js @@ -18,7 +18,6 @@ class TimeseriesGraph extends HTMLElement { #height; #intervalId; #pollSeconds; - #label; #end; #duration; #step_duration; @@ -32,7 +31,7 @@ class TimeseriesGraph extends HTMLElement { this.#targetNode = this.appendChild(document.createElement("div")); } - static observedAttributes = ['uri', 'width', 'height', 'poll-seconds', 'end', 'duration', 'step-duration', 'd3-tick-format']; + static observedAttributes = ['uri', 'width', 'height', 'poll-seconds', 'end', 'duration', 'step-duration']; attributeChangedCallback(name, _oldValue, newValue) { switch (name) { @@ -48,9 +47,6 @@ class TimeseriesGraph extends HTMLElement { case 'poll-seconds': this.#pollSeconds = newValue; break; - case 'label': - this.#label = newValue; - break; case 'end': this.#end = newValue; break; @@ -74,7 +70,6 @@ class TimeseriesGraph extends HTMLElement { this.#width = this.getAttribute('width') || this.#width; this.#height = this.getAttribute('height') || this.#height; this.#pollSeconds = this.getAttribute('poll-seconds') || this.#pollSeconds; - this.#label = this.getAttribute('label') || null; this.#end = this.getAttribute('end') || null; this.#duration = this.getAttribute('duration') || null; this.#step_duration = this.getAttribute('step-duration') || null; @@ -134,62 +129,81 @@ class TimeseriesGraph extends HTMLElement { orientation: 'h' } }; - const layout = { + var layout = { displayModeBar: false, responsive: true, - yaxis: { - tickformat: this.#d3TickFormat, - //showticksuffix: 'all', - //ticksuffix: '%', - //exponentFormat: 'SI' - } }; - console.debug("layout", layout); - if (data.Series) { - // https://plotly.com/javascript/reference/scatter/ - var traces = []; - for (const pair of data.Series) { - const series = pair[1]; - const labels = pair[0]; - var trace = { - type: "scatter", - mode: "lines+text", - x: [], - y: [] - }; - if (labels[this.#label]) { - trace.name = labels[this.#label]; - }; - for (const point of series) { - trace.x.push(new Date(point.timestamp * 1000)); - trace.y.push(point.value); + var traces = []; + for (var subplot_idx in data) { + const subplot = data[subplot_idx]; + const subplotCount = Number(subplot_idx) + 1; + const yaxis = "y" + subplotCount + if (subplot.Series) { + // https://plotly.com/javascript/reference/scatter/ + for (const triple of subplot.Series) { + const labels = triple[0]; + const meta = triple[1]; + layout["yaxis" + subplotCount] = { + anchor: yaxis, + tickformat: meta["d3_tick_format"] || this.#d3TickFormat + }; + const series = triple[2]; + var trace = { + type: "scatter", + mode: "lines+text", + x: [], + y: [], + yaxis: yaxis, + yhoverformat: meta["d3_tick_format"], + }; + const namePrefix = meta["name_prefix"]; + const nameSuffix = meta["name_suffix"]; + const nameLabel = meta["name_label"]; + var name = ""; + if (namePrefix) { + name = namePrefix + "-"; + }; + if (nameLabel && labels[nameLabel]) { + name = name + labels[nameLabel]; + }; + if (nameSuffix) { + name = name + " - " + nameSuffix; + }; + if (name) { trace.name = name; } + for (const point of series) { + trace.x.push(new Date(point.timestamp * 1000)); + trace.y.push(point.value); + } + traces.push(trace); + } + } else if (subplot.Scalar) { + // https://plotly.com/javascript/reference/bar/ + for (const triple of subplot.Scalar) { + const labels = triple[0]; + const meta = triple[1]; + const series = triple[2]; + var trace = { + type: "bar", + x: [], + y: [], + yaxis: yaxis, + yhoverformat: meta["d3_tick_format"], + }; + let nameLabel = meta["name_label"]; + if (nameLabel && labels[nameLabel]) { + trace.name = labels[nameLabel]; + }; + if (nameLabel && labels[nameLabel]) { + trace.x.push(labels[nameLabel]); + }; + trace.y.push(series.value); + traces.push(trace); } - traces.push(trace); } - // https://plotly.com/javascript/plotlyjs-function-reference/#plotlyreact - Plotly.react(this.getTargetNode(), traces, layout, config); - } else if (data.Scalar) { - // https://plotly.com/javascript/reference/bar/ - var traces = []; - for (const pair of data.Scalar) { - const series = pair[1]; - const labels = pair[0]; - var trace = { - type: "bar", - x: [], - y: [] - }; - if (labels[this.#label]) { - trace.name = labels[this.#label]; - }; - if (labels[this.#label]) { - trace.x.push(labels[this.#label]); - }; - trace.y.push(series.value); - traces.push(trace); - } - Plotly.react(this.getTargetNode(), traces, layout, config); } + console.debug("traces: ", traces); + // https://plotly.com/javascript/plotlyjs-function-reference/#plotlyreact + Plotly.react(this.getTargetNode(), traces, layout, config); } } @@ -201,20 +215,20 @@ class SpanSelector extends HTMLElement { #durationInput = null; #stepDurationInput = null; #updateInput = null - + constructor() { super(); this.#targetNode = this.appendChild(document.createElement('div')); - + this.#targetNode.appendChild(document.createElement('span')).innerText = "end: "; this.#endInput = this.#targetNode.appendChild(document.createElement('input')); - + this.#targetNode.appendChild(document.createElement('span')).innerText = "duration: "; this.#durationInput = this.#targetNode.appendChild(document.createElement('input')); - + this.#targetNode.appendChild(document.createElement('span')).innerText = "step duration: "; this.#stepDurationInput = this.#targetNode.appendChild(document.createElement('input')); - + this.#updateInput = this.#targetNode.appendChild(document.createElement('button')); this.#updateInput.innerText = "Update"; } @@ -225,7 +239,7 @@ class SpanSelector extends HTMLElement { self.updateGraphs() }; } - + disconnectedCallback() { this.#updateInput.onclick = undefined; }