From 16ff43f4e2de3d094e7b0b399d5e0d00b419d6d7 Mon Sep 17 00:00:00 2001 From: Jeremy Wall Date: Mon, 26 Feb 2024 19:13:05 -0500 Subject: [PATCH 01/15] feat: Loki logql query client --- Cargo.lock | 166 ++++++++++++++++++++++++++++++++ Cargo.toml | 1 + src/dashboard.rs | 8 +- src/query/loki.rs | 127 ++++++++++++++++++++++++ src/query/mod.rs | 35 +++++++ src/{query.rs => query/prom.rs} | 39 +++----- src/routes.rs | 4 +- 7 files changed, 349 insertions(+), 31 deletions(-) create mode 100644 src/query/loki.rs create mode 100644 src/query/mod.rs rename src/{query.rs => query/prom.rs} (88%) diff --git a/Cargo.lock b/Cargo.lock index 0971659..14ae837 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -429,12 +429,33 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "fastrand" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" + [[package]] name = "fnv" version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "form_urlencoded" version = "1.2.1" @@ -579,6 +600,7 @@ dependencies = [ "maud", "parse_duration", "prometheus-http-query", + "reqwest", "serde", "serde_json", "serde_yaml", @@ -719,6 +741,19 @@ dependencies = [ "tokio-rustls", ] +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes", + "hyper 0.14.28", + "native-tls", + "tokio", + "tokio-native-tls", +] + [[package]] name = "hyper-util" version = "0.1.3" @@ -885,6 +920,24 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "native-tls" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "nu-ansi-term" version = "0.46.0" @@ -1002,6 +1055,50 @@ version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +[[package]] +name = "openssl" +version = "0.10.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15c9d69dd87a29568d4d017cfe8ec518706046a05184e5aea92d0af890b803c8" +dependencies = [ + "bitflags 2.4.2", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + +[[package]] +name = "openssl-sys" +version = "0.9.99" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e1bf214306098e4832460f797824c05d25aacdf896f64a985fb0fd992454ae" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "overload" version = "0.1.1" @@ -1063,6 +1160,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkg-config" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2900ede94e305130c13ddd391e0ab7cbaeb783945ae07a279c268cb05109c6cb" + [[package]] name = "polling" version = "3.3.2" @@ -1183,10 +1286,12 @@ dependencies = [ "http-body 0.4.6", "hyper 0.14.28", "hyper-rustls", + "hyper-tls", "ipnet", "js-sys", "log", "mime", + "native-tls", "once_cell", "percent-encoding", "pin-project-lite", @@ -1198,6 +1303,7 @@ dependencies = [ "sync_wrapper", "system-configuration", "tokio", + "tokio-native-tls", "tokio-rustls", "tower-service", "url", @@ -1284,6 +1390,15 @@ version = "1.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f98d2aa92eebf49b69786be48e4477826b256916e84a57ff2a4f21923b48eb4c" +[[package]] +name = "schannel" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" +dependencies = [ + "windows-sys 0.52.0", +] + [[package]] name = "sct" version = "0.7.1" @@ -1294,6 +1409,29 @@ dependencies = [ "untrusted", ] +[[package]] +name = "security-framework" +version = "2.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05b64fb303737d99b81884b2c63433e9ae28abebe5eb5045dcdd175dc2ecf4de" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e932934257d3b408ed8f30db49d85ea163bfe74961f017f405b025af298f0c7a" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "serde" version = "1.0.196" @@ -1444,6 +1582,18 @@ dependencies = [ "libc", ] +[[package]] +name = "tempfile" +version = "3.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a365e8cd18e44762ef95d87f284f4b5cd04107fec2ff3052bd6a3e6069669e67" +dependencies = [ + "cfg-if", + "fastrand", + "rustix", + "windows-sys 0.52.0", +] + [[package]] name = "thread_local" version = "1.1.7" @@ -1527,6 +1677,16 @@ dependencies = [ "syn", ] +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + [[package]] name = "tokio-rustls" version = "0.24.1" @@ -1717,6 +1877,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version_check" version = "0.9.4" diff --git a/Cargo.toml b/Cargo.toml index 743998a..b27c6f1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,3 +23,4 @@ tokio = { version = "1.36.0", features = ["net", "rt", "rt-multi-thread"] } tower-http = { version = "0.5.1", features = ["trace"] } tracing = "0.1.40" tracing-subscriber = "0.3.18" +reqwest = { version = "0.11.24", features = ["rustls-tls"] } diff --git a/src/dashboard.rs b/src/dashboard.rs index 76a0f32..9b816a3 100644 --- a/src/dashboard.rs +++ b/src/dashboard.rs @@ -20,7 +20,7 @@ use serde_yaml; use tracing::{debug, error}; use anyhow::Result; -use crate::query::{QueryConn, QueryType, QueryResult, to_samples}; +use crate::query::{PromQueryConn, QueryType, PromQueryResult, to_samples}; #[derive(Serialize, Deserialize, Debug, Clone)] pub struct PlotMeta { @@ -103,7 +103,7 @@ pub struct Graph { pub d3_tick_format: Option, } -pub async fn query_data(graph: &Graph, dash: &Dashboard, query_span: Option) -> Result> { +pub async fn query_data(graph: &Graph, dash: &Dashboard, query_span: Option) -> Result> { let connections = graph.get_query_connections(&dash.span, &query_span); let mut data = Vec::new(); for conn in connections { @@ -172,7 +172,7 @@ impl Graph { &'graph self, graph_span: &'graph Option, query_span: &'graph Option, - ) -> Vec> { + ) -> Vec> { let mut conns = Vec::new(); for plot in self.plots.iter() { debug!( @@ -180,7 +180,7 @@ impl Graph { source = plot.source, "Getting query connection for graph" ); - let mut conn = QueryConn::new(&plot.source, &plot.query, self.query_type.clone(), plot.meta.clone()); + let mut conn = PromQueryConn::new(&plot.source, &plot.query, self.query_type.clone(), plot.meta.clone()); // Query params take precendence over all other settings. Then graph settings take // precedences and finally the dashboard settings take precendence if let Some((end, duration, step_duration)) = graph_span_to_tuple(query_span) { diff --git a/src/query/loki.rs b/src/query/loki.rs new file mode 100644 index 0000000..aa18cea --- /dev/null +++ b/src/query/loki.rs @@ -0,0 +1,127 @@ +// Copyright 2024 Jeremy Wall +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +use std::collections::HashMap; + +use anyhow::Result; +use chrono::prelude::*; +use reqwest; +use serde::{Deserialize, Serialize}; + +use super::{QueryType, TimeSpan}; + +// TODO(jwall): Should I allow non stream returns? +#[derive(Serialize, Deserialize)] +pub enum ResultType { + /// Returned by query endpoints + #[serde(rename = "vector")] + Vector, + /// Returned by query_range endpoints + #[serde(rename = "matrix")] + Matrix, + /// Returned by query and query_range endpoints + #[serde(rename = "streams")] + Streams, +} + +#[derive(Serialize, Deserialize)] +pub struct LokiResult { + #[serde(alias = "metric")] + #[serde(alias = "stream")] + labels: Option>, + value: Option<(i64, String)>, + /// The only version that returns log lines + values: Option>, +} + +#[derive(Serialize, Deserialize)] +pub struct LokiData { + result_type: ResultType, + result: Vec, + //stats: // TODO +} + +#[derive(Serialize, Deserialize)] +pub struct LokiResponse { + status: String, + data: LokiData, +} + +pub struct LokiConn<'conn> { + url: &'conn str, + query: &'conn str, + span: Option, + query_type: QueryType, + limit: Option, +} + +const SCALAR_API_PATH: &'static str = "/loki/api/v1/query"; +const RANGE_API_PATH: &'static str = "/loki/api/v1/query_range"; + +impl<'conn> LokiConn<'conn> { + pub fn new<'a: 'conn>(url: &'a str, query: &'a str, query_type: QueryType) -> Self { + Self { + url, + query, + query_type, + span: None, + limit: None, + } + } + + pub fn with_limit(mut self, limit: usize) -> Self { + self.limit = Some(limit); + self + } + + pub fn with_span( + mut self, + end: DateTime, + duration: chrono::Duration, + step: chrono::Duration, + ) -> Self { + self.span = Some(TimeSpan { + end, + duration, + step_seconds: step.num_seconds(), + }); + self + } + + pub async fn get_results(&self) -> Result { + let url = match self.query_type { + QueryType::Scalar => format!("{}{}", self.url, SCALAR_API_PATH), + QueryType::Range => format!("{}{}", self.url, RANGE_API_PATH), + }; + let client = reqwest::Client::new(); + let mut req = client.get(url).query(&["query", self.query]); + if self.limit.is_some() { + req = req.query(&["limit", &self.limit.map(|u| u.to_string()).unwrap()]); + } + if let QueryType::Range = self.query_type { + let (start, end, step_resolution) = if let Some(span) = &self.span { + let start = span.end - span.duration; + (start.timestamp(), span.end.timestamp(), span.step_seconds as f64) + } else { + let end = Utc::now(); + let start = end - chrono::Duration::minutes(10); + (start.timestamp(), end.timestamp(), 30 as f64) + }; + req = req.query(&["end", &end.to_string()]); + req = req.query(&["since", &start.to_string()]); + req = req.query(&["step", &step_resolution.to_string()]); + } + + Ok(req.send().await?.json().await?) + } +} diff --git a/src/query/mod.rs b/src/query/mod.rs new file mode 100644 index 0000000..b348f1d --- /dev/null +++ b/src/query/mod.rs @@ -0,0 +1,35 @@ +// Copyright 2023 Jeremy Wall +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +use serde::Deserialize; +use chrono::prelude::*; + +mod loki; +mod prom; + +#[derive(Deserialize, Clone, Debug)] +pub enum QueryType { + Range, + Scalar, +} + +#[derive(Debug)] +pub struct TimeSpan { + pub end: DateTime, + pub duration: chrono::Duration, + pub step_seconds: i64, +} + + +pub use prom::*; + diff --git a/src/query.rs b/src/query/prom.rs similarity index 88% rename from src/query.rs rename to src/query/prom.rs index 695d94d..ba486bf 100644 --- a/src/query.rs +++ b/src/query/prom.rs @@ -1,6 +1,4 @@ -use std::collections::HashMap; - -// Copyright 2023 Jeremy Wall +// Copyright 2024 Jeremy Wall // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -13,6 +11,8 @@ use std::collections::HashMap; // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. +use std::collections::HashMap; + use chrono::prelude::*; use prometheus_http_query::{ response::{Data, PromqlResult}, @@ -23,21 +23,10 @@ use tracing::debug; use crate::dashboard::PlotMeta; -#[derive(Deserialize, Clone, Debug)] -pub enum QueryType { - Range, - Scalar, -} +use super::{QueryType, TimeSpan}; #[derive(Debug)] -pub struct TimeSpan { - pub end: DateTime, - pub duration: chrono::Duration, - pub step_seconds: i64, -} - -#[derive(Debug)] -pub struct QueryConn<'conn> { +pub struct PromQueryConn<'conn> { source: &'conn str, query: &'conn str, span: Option, @@ -45,7 +34,7 @@ pub struct QueryConn<'conn> { pub meta: PlotMeta, } -impl<'conn> QueryConn<'conn> { +impl<'conn> PromQueryConn<'conn> { pub fn new<'a: 'conn>(source: &'a str, query: &'a str, query_type: QueryType, meta: PlotMeta) -> Self { Self { source, @@ -120,15 +109,15 @@ pub struct DataPoint { } #[derive(Serialize, Deserialize)] -pub enum QueryResult { +pub enum PromQueryResult { Series(Vec<(HashMap, PlotMeta, Vec)>), Scalar(Vec<(HashMap, PlotMeta, DataPoint)>), } -impl std::fmt::Debug for QueryResult { +impl std::fmt::Debug for PromQueryResult { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { - QueryResult::Series(v) => { + PromQueryResult::Series(v) => { f.write_fmt(format_args!("Series trace count = {}", v.len()))?; for (idx, (tags, meta, trace)) in v.iter().enumerate() { f.write_fmt(format_args!( @@ -140,7 +129,7 @@ impl std::fmt::Debug for QueryResult { ))?; } } - QueryResult::Scalar(v) => { + PromQueryResult::Scalar(v) => { f.write_fmt(format_args!("{} traces", v.len()))?; } } @@ -148,9 +137,9 @@ impl std::fmt::Debug for QueryResult { } } -pub fn to_samples(data: Data, meta: PlotMeta) -> QueryResult { +pub fn to_samples(data: Data, meta: PlotMeta) -> PromQueryResult { match data { - Data::Matrix(mut range) => QueryResult::Series( + Data::Matrix(mut range) => PromQueryResult::Series( range .drain(0..) .map(|rv| { @@ -169,7 +158,7 @@ pub fn to_samples(data: Data, meta: PlotMeta) -> QueryResult { }) .collect(), ), - Data::Vector(mut vector) => QueryResult::Scalar( + Data::Vector(mut vector) => PromQueryResult::Scalar( vector .drain(0..) .map(|iv| { @@ -185,7 +174,7 @@ pub fn to_samples(data: Data, meta: PlotMeta) -> QueryResult { }) .collect(), ), - Data::Scalar(sample) => QueryResult::Scalar(vec![( + Data::Scalar(sample) => PromQueryResult::Scalar(vec![( HashMap::new(), meta.clone(), DataPoint { diff --git a/src/routes.rs b/src/routes.rs index 3b36328..81cbd8f 100644 --- a/src/routes.rs +++ b/src/routes.rs @@ -26,7 +26,7 @@ use serde::{Serialize, Deserialize}; use tracing::debug; use crate::dashboard::{Dashboard, Graph, GraphSpan, AxisDefinition, Orientation, query_data}; -use crate::query::QueryResult; +use crate::query::PromQueryResult; type Config = State>>; @@ -34,7 +34,7 @@ type Config = State>>; pub struct GraphPayload { pub legend_orientation: Option, pub yaxes: Vec, - pub plots: Vec, + pub plots: Vec, } pub async fn graph_query( From ec2394eaf790c077b538683f6425103446ffd01a Mon Sep 17 00:00:00 2001 From: Jeremy Wall Date: Tue, 27 Feb 2024 18:13:26 -0500 Subject: [PATCH 02/15] feat: logql queries in dashboard definitions --- examples/example_dashboards.yaml | 13 ++++ src/dashboard.rs | 64 ++++++++++++++++-- src/main.rs | 14 ++-- src/query/loki.rs | 107 ++++++++++++++++++++++++------- src/query/mod.rs | 65 ++++++++++++++++++- src/query/prom.rs | 53 ++++----------- src/routes.rs | 50 +++++++++++---- static/lib.js | 12 ++-- static/site.css | 2 +- 9 files changed, 284 insertions(+), 96 deletions(-) diff --git a/examples/example_dashboards.yaml b/examples/example_dashboards.yaml index 464218a..ff8da83 100644 --- a/examples/example_dashboards.yaml +++ b/examples/example_dashboards.yaml @@ -58,3 +58,16 @@ query: 'node_memory_MemFree_bytes{job="nodestats"}' meta: name_format: "`${labels.instance}`" +- title: Log Test Dashboard 1 + span: + end: now + duration: 1h + step_duration: 5min + logs: + - title: Systemd Service Logs + query_type: Range + yaxes: + - anchor: "y" # This axis is y + source: http://heimdall:3100 + query: | + {job="systemd-journal"} diff --git a/src/dashboard.rs b/src/dashboard.rs index 9b816a3..35eb634 100644 --- a/src/dashboard.rs +++ b/src/dashboard.rs @@ -20,7 +20,7 @@ use serde_yaml; use tracing::{debug, error}; use anyhow::Result; -use crate::query::{PromQueryConn, QueryType, PromQueryResult, to_samples}; +use crate::query::{PromQueryConn, QueryType, QueryResult, LokiConn, prom_to_samples, loki_to_sample}; #[derive(Serialize, Deserialize, Debug, Clone)] pub struct PlotMeta { @@ -73,7 +73,8 @@ pub struct GraphSpan { #[derive(Deserialize)] pub struct Dashboard { pub title: String, - pub graphs: Vec, + pub graphs: Option>, + pub logs: Option>, pub span: Option, } @@ -92,6 +93,8 @@ pub enum Orientation { Vertical, } +// NOTE(zapher): These two structs look repetitive but we haven't hit the rule of three yet. +// If we do then it might be time to restructure them a bit. #[derive(Deserialize)] pub struct Graph { pub title: String, @@ -103,11 +106,23 @@ pub struct Graph { pub d3_tick_format: Option, } -pub async fn query_data(graph: &Graph, dash: &Dashboard, query_span: Option) -> Result> { +#[derive(Deserialize)] +pub struct LogStream { + pub title: String, + pub legend_orientation: Option, + pub source: String, + pub yaxes: Vec, + pub query: String, + pub span: Option, + pub limit: Option, + pub query_type: QueryType, +} + +pub async fn prom_query_data(graph: &Graph, dash: &Dashboard, query_span: Option) -> Result> { let connections = graph.get_query_connections(&dash.span, &query_span); let mut data = Vec::new(); for conn in connections { - data.push(to_samples( + data.push(prom_to_samples( conn.get_results() .await? .data() @@ -118,6 +133,17 @@ pub async fn query_data(graph: &Graph, dash: &Dashboard, query_span: Option) -> Result { + let conn = stream.get_query_connection(&dash.span, &query_span); + let response = conn.get_results().await?; + if response.status == "success" { + Ok(loki_to_sample(response.data)) + } else { + // TODO(jwall): Better error handling than this + panic!("Loki query status: {}", response.status) + } +} + fn duration_from_string(duration_string: &str) -> Option { match parse_duration::parse(duration_string) { Ok(d) => match Duration::from_std(d) { @@ -178,7 +204,7 @@ impl Graph { debug!( query = plot.query, source = plot.source, - "Getting query connection for graph" + "Getting query connection for graph", ); let mut conn = PromQueryConn::new(&plot.source, &plot.query, self.query_type.clone(), plot.meta.clone()); // Query params take precendence over all other settings. Then graph settings take @@ -196,6 +222,34 @@ impl Graph { } } +impl LogStream { + pub fn get_query_connection<'conn, 'stream: 'conn>( + &'stream self, + graph_span: &'stream Option, + query_span: &'stream Option, + ) -> LokiConn<'conn> { + debug!( + query = self.query, + source = self.source, + "Getting query connection for log streams", + ); + let mut conn = LokiConn::new(&self.source, &self.query, self.query_type.clone()); + // Query params take precendence over all other settings. Then graph settings take + // precedences and finally the dashboard settings take precendence + if let Some((end, duration, step_duration)) = graph_span_to_tuple(query_span) { + conn = conn.with_span(end, duration, step_duration); + } else if let Some((end, duration, step_duration)) = graph_span_to_tuple(&self.span) { + conn = conn.with_span(end, duration, step_duration); + } else if let Some((end, duration, step_duration)) = graph_span_to_tuple(graph_span) { + conn = conn.with_span(end, duration, step_duration); + } + if let Some(limit) = self.limit { + conn = conn.with_limit(limit); + } + conn + } +} + pub fn read_dashboard_list(path: &Path) -> anyhow::Result> { let f = std::fs::File::open(path)?; Ok(serde_yaml::from_reader(f)?) diff --git a/src/main.rs b/src/main.rs index fac843b..829c7a4 100644 --- a/src/main.rs +++ b/src/main.rs @@ -15,7 +15,7 @@ use std::path::PathBuf; use anyhow; use axum::{self, extract::State, routing::*, Router}; use clap::{self, Parser, ValueEnum}; -use dashboard::{Dashboard, query_data}; +use dashboard::{Dashboard, prom_query_data}; use tokio::net::TcpListener; use tower_http::trace::TraceLayer; use tracing::{error, info}; @@ -49,12 +49,14 @@ struct Cli { } async fn validate(dash: &Dashboard) -> anyhow::Result<()> { - for graph in dash.graphs.iter() { - let data = query_data(graph, &dash, None).await; - if data.is_err() { - error!(err=?data, "Invalid dashboard query or queries"); + if let Some(ref graphs) = dash.graphs { + for graph in graphs.iter() { + let data = prom_query_data(graph, &dash, None).await; + if data.is_err() { + error!(err=?data, "Invalid dashboard query or queries"); + } + let _ = data?; } - let _ = data?; } return Ok(()); } diff --git a/src/query/loki.rs b/src/query/loki.rs index aa18cea..8152593 100644 --- a/src/query/loki.rs +++ b/src/query/loki.rs @@ -17,11 +17,12 @@ use anyhow::Result; use chrono::prelude::*; use reqwest; use serde::{Deserialize, Serialize}; +use tracing::{debug, error}; -use super::{QueryType, TimeSpan}; +use super::{LogLine, QueryResult, QueryType, TimeSpan}; // TODO(jwall): Should I allow non stream returns? -#[derive(Serialize, Deserialize)] +#[derive(Serialize, Deserialize, Debug)] pub enum ResultType { /// Returned by query endpoints #[serde(rename = "vector")] @@ -34,27 +35,81 @@ pub enum ResultType { Streams, } -#[derive(Serialize, Deserialize)] +// Note that the value and volue types return a pair where the first item is a string but +// will in actuality always be an f64 number. +#[derive(Serialize, Deserialize, Debug)] pub struct LokiResult { #[serde(alias = "metric")] #[serde(alias = "stream")] - labels: Option>, - value: Option<(i64, String)>, - /// The only version that returns log lines - values: Option>, + labels: HashMap, + /// Calculated Value returned by vector result types + value: Option<(String, String)>, + /// Stream of Log lines, Returned by matrix and stream result types + values: Option>, +} + +#[derive(Serialize, Deserialize)] +pub struct LokiResponse { + pub status: String, + pub data: LokiData, } #[derive(Serialize, Deserialize)] pub struct LokiData { + #[serde(rename = "resultType")] result_type: ResultType, result: Vec, //stats: // TODO } -#[derive(Serialize, Deserialize)] -pub struct LokiResponse { - status: String, - data: LokiData, +pub fn loki_to_sample(data: LokiData) -> QueryResult { + match data.result_type { + ResultType::Vector => { + let mut values = Vec::with_capacity(data.result.len()); + for result in data.result { + if let Some(value) = result.value { + values.push(( + result.labels, + LogLine { + timestamp: value.0.parse::().expect("Invalid f64 type"), + line: value.1, + }, + )); + } else { + error!( + ?result, + "Invalid LokiResult: No value field when result type is {:?}", + data.result_type, + ); + } + } + QueryResult::StreamInstant(values) + } + ResultType::Matrix | ResultType::Streams => { + let mut values = Vec::with_capacity(data.result.len()); + for result in data.result { + if let Some(value) = result.values { + values.push(( + result.labels, + value + .into_iter() + .map(|(timestamp, line)| LogLine { + timestamp: timestamp.parse::().expect("Invalid f64 type"), + line, + }) + .collect(), + )); + } else { + error!( + ?result, + "Invalid LokiResult: No values field when result type is {:?}", + data.result_type, + ); + } + } + QueryResult::Stream(values) + } + } } pub struct LokiConn<'conn> { @@ -98,30 +153,38 @@ impl<'conn> LokiConn<'conn> { self } - pub async fn get_results(&self) -> Result { + pub async fn get_results(&self) -> Result { let url = match self.query_type { QueryType::Scalar => format!("{}{}", self.url, SCALAR_API_PATH), QueryType::Range => format!("{}{}", self.url, RANGE_API_PATH), }; let client = reqwest::Client::new(); - let mut req = client.get(url).query(&["query", self.query]); + let mut req = client.get(url).query(&[("query", self.query)]); + debug!(?req, "Building loki reqwest client"); if self.limit.is_some() { - req = req.query(&["limit", &self.limit.map(|u| u.to_string()).unwrap()]); + debug!(?req, "adding limit"); + req = req.query(&[("limit", &self.limit.map(|u| u.to_string()).unwrap())]); } if let QueryType::Range = self.query_type { - let (start, end, step_resolution) = if let Some(span) = &self.span { - let start = span.end - span.duration; - (start.timestamp(), span.end.timestamp(), span.step_seconds as f64) + debug!(?req, "Configuring span query params"); + let (since, end, step_resolution) = if let Some(span) = &self.span { + ( + span.duration, + span.end.timestamp(), + span.step_seconds as f64, + ) } else { let end = Utc::now(); - let start = end - chrono::Duration::minutes(10); - (start.timestamp(), end.timestamp(), 30 as f64) + (chrono::Duration::minutes(10), end.timestamp(), 30 as f64) }; - req = req.query(&["end", &end.to_string()]); - req = req.query(&["since", &start.to_string()]); - req = req.query(&["step", &step_resolution.to_string()]); + req = req.query(&[ + ("end", &end.to_string()), + ("since", &format!("{}s", since.num_seconds())), + ("step", &step_resolution.to_string()), + ]); } + debug!(?req, "Sending request"); Ok(req.send().await?.json().await?) } } diff --git a/src/query/mod.rs b/src/query/mod.rs index b348f1d..59d2036 100644 --- a/src/query/mod.rs +++ b/src/query/mod.rs @@ -11,9 +11,13 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -use serde::Deserialize; +use std::collections::HashMap; + +use serde::{Serialize, Deserialize}; use chrono::prelude::*; +use crate::dashboard::PlotMeta; + mod loki; mod prom; @@ -31,5 +35,62 @@ pub struct TimeSpan { } -pub use prom::*; +#[derive(Serialize, Deserialize, Debug)] +pub struct DataPoint { + timestamp: f64, + value: f64, +} +#[derive(Serialize, Deserialize, Debug)] +pub struct LogLine { + timestamp: f64, + line: String, +} + +#[derive(Serialize, Deserialize)] +pub enum QueryResult { + Series(Vec<(HashMap, PlotMeta, Vec)>), + Scalar(Vec<(HashMap, PlotMeta, DataPoint)>), + StreamInstant(Vec<(HashMap, LogLine)>), + Stream(Vec<(HashMap, Vec)>), +} + +impl std::fmt::Debug for QueryResult { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + QueryResult::Series(v) => { + f.write_fmt(format_args!("Series trace count = {}", v.len()))?; + for (idx, (tags, meta, trace)) in v.iter().enumerate() { + f.write_fmt(format_args!( + "; {}: tags {:?} meta: {:?} datapoint count = {};", + idx, + tags, + meta, + trace.len() + ))?; + } + } + QueryResult::Scalar(v) => { + f.write_fmt(format_args!("{} traces", v.len()))?; + } + QueryResult::StreamInstant(v) => { + f.write_fmt(format_args!("{} traces", v.len()))?; + } + QueryResult::Stream(v) => { + f.write_fmt(format_args!("stream trace count = {}", v.len()))?; + for (idx, (tags, trace)) in v.iter().enumerate() { + f.write_fmt(format_args!( + "; {}: tags {:?} line count = {}", + idx, + tags, + trace.len() + ))? + } + } + } + Ok(()) + } +} + +pub use prom::*; +pub use loki::*; diff --git a/src/query/prom.rs b/src/query/prom.rs index ba486bf..fd0c84d 100644 --- a/src/query/prom.rs +++ b/src/query/prom.rs @@ -18,12 +18,11 @@ use prometheus_http_query::{ response::{Data, PromqlResult}, Client, }; -use serde::{Deserialize, Serialize}; use tracing::debug; use crate::dashboard::PlotMeta; -use super::{QueryType, TimeSpan}; +use super::{QueryType, TimeSpan, QueryResult, DataPoint}; #[derive(Debug)] pub struct PromQueryConn<'conn> { @@ -35,7 +34,12 @@ pub struct PromQueryConn<'conn> { } impl<'conn> PromQueryConn<'conn> { - pub fn new<'a: 'conn>(source: &'a str, query: &'a str, query_type: QueryType, meta: PlotMeta) -> Self { + pub fn new<'a: 'conn>( + source: &'a str, + query: &'a str, + query_type: QueryType, + meta: PlotMeta, + ) -> Self { Self { source, query, @@ -102,44 +106,9 @@ impl<'conn> PromQueryConn<'conn> { } } -#[derive(Serialize, Deserialize, Debug)] -pub struct DataPoint { - timestamp: f64, - value: f64, -} - -#[derive(Serialize, Deserialize)] -pub enum PromQueryResult { - Series(Vec<(HashMap, PlotMeta, Vec)>), - Scalar(Vec<(HashMap, PlotMeta, DataPoint)>), -} - -impl std::fmt::Debug for PromQueryResult { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - PromQueryResult::Series(v) => { - f.write_fmt(format_args!("Series trace count = {}", v.len()))?; - for (idx, (tags, meta, trace)) in v.iter().enumerate() { - f.write_fmt(format_args!( - "; {}: tags {:?} meta: {:?} datapoint count = {};", - idx, - tags, - meta, - trace.len() - ))?; - } - } - PromQueryResult::Scalar(v) => { - f.write_fmt(format_args!("{} traces", v.len()))?; - } - } - Ok(()) - } -} - -pub fn to_samples(data: Data, meta: PlotMeta) -> PromQueryResult { +pub fn prom_to_samples(data: Data, meta: PlotMeta) -> QueryResult { match data { - Data::Matrix(mut range) => PromQueryResult::Series( + Data::Matrix(mut range) => QueryResult::Series( range .drain(0..) .map(|rv| { @@ -158,7 +127,7 @@ pub fn to_samples(data: Data, meta: PlotMeta) -> PromQueryResult { }) .collect(), ), - Data::Vector(mut vector) => PromQueryResult::Scalar( + Data::Vector(mut vector) => QueryResult::Scalar( vector .drain(0..) .map(|iv| { @@ -174,7 +143,7 @@ pub fn to_samples(data: Data, meta: PlotMeta) -> PromQueryResult { }) .collect(), ), - Data::Scalar(sample) => PromQueryResult::Scalar(vec![( + Data::Scalar(sample) => QueryResult::Scalar(vec![( HashMap::new(), meta.clone(), DataPoint { diff --git a/src/routes.rs b/src/routes.rs index 81cbd8f..b3f3f6f 100644 --- a/src/routes.rs +++ b/src/routes.rs @@ -25,8 +25,8 @@ use maud::{html, Markup}; use serde::{Serialize, Deserialize}; use tracing::debug; -use crate::dashboard::{Dashboard, Graph, GraphSpan, AxisDefinition, Orientation, query_data}; -use crate::query::PromQueryResult; +use crate::dashboard::{Dashboard, Graph, GraphSpan, AxisDefinition, Orientation, prom_query_data, loki_query_data}; +use crate::query::QueryResult; type Config = State>>; @@ -34,7 +34,21 @@ type Config = State>>; pub struct GraphPayload { pub legend_orientation: Option, pub yaxes: Vec, - pub plots: Vec, + pub plots: Vec, +} + +// TODO(jwall): Should this be a completely different payload? +pub async fn loki_query( + State(config): Config, + Path((dash_idx, loki_idx)): Path<(usize, usize)>, + Query(query): Query>, +) -> Json { + let dash = config.get(dash_idx).expect(&format!("No such dashboard index {}", dash_idx)); + let log = dash.logs + .as_ref().expect("No logs in this dashboard") + .get(loki_idx).expect(&format!("No such log query {}", loki_idx)); + let plots = vec![loki_query_data(log, dash, query_to_graph_span(query)).await.expect("Unable to get log query results")]; + Json(GraphPayload{legend_orientation: None, yaxes: log.yaxes.clone(), plots}) } pub async fn graph_query( @@ -43,11 +57,17 @@ pub async fn graph_query( Query(query): Query>, ) -> Json { debug!("Getting data for query"); - let dash = config.get(dash_idx).expect("No such dashboard index"); + let dash = config.get(dash_idx).expect(&format!("No such dashboard index {}", dash_idx)); let graph = dash .graphs + .as_ref().expect("No graphs in this dashboard") .get(graph_idx) .expect(&format!("No such graph in dasboard {}", dash_idx)); + let plots = prom_query_data(graph, dash, query_to_graph_span(query)).await.expect("Unable to get query results"); + Json(GraphPayload{legend_orientation: graph.legend_orientation.clone(), yaxes: graph.yaxes.clone(), plots}) +} + +fn query_to_graph_span(query: HashMap) -> Option { let query_span = { if query.contains_key("end") && query.contains_key("duration") @@ -62,15 +82,19 @@ pub async fn graph_query( None } }; - let plots = query_data(graph, dash, query_span).await.expect("Unable to get query results"); - Json(GraphPayload{legend_orientation: graph.legend_orientation.clone(), yaxes: graph.yaxes.clone(), plots}) + query_span } pub fn mk_api_routes(config: Arc>) -> Router { // Query routes - Router::new().route( + Router::new() + .route( "/dash/:dash_idx/graph/:graph_idx", - get(graph_query).with_state(config), + get(graph_query).with_state(config.clone()), + ) + .route( + "/dash/:dash_idx/log/:log_idx", + get(loki_query).with_state(config), ) } @@ -82,9 +106,9 @@ pub fn graph_component(dash_idx: usize, graph_idx: usize, graph: &Graph) -> Mark div { h2 { (graph.title) " - " a href=(graph_embed_uri) { "embed url" } } @if graph.d3_tick_format.is_some() { - timeseries-graph uri=(graph_data_uri) id=(graph_id) d3-tick-format=(graph.d3_tick_format.as_ref().unwrap()) { } + graph-plot uri=(graph_data_uri) id=(graph_id) d3-tick-format=(graph.d3_tick_format.as_ref().unwrap()) { } } @else { - timeseries-graph uri=(graph_data_uri) id=(graph_id) { } + graph-plot uri=(graph_data_uri) id=(graph_id) { } } } ) @@ -96,8 +120,9 @@ pub async fn graph_ui( ) -> Markup { let graph = config .get(dash_idx) - .expect("No such dashboard") + .expect(&format!("No such dashboard {}", dash_idx)) .graphs + .as_ref().expect("No graphs in this dashboard") .get(graph_idx) .expect("No such graph"); graph_component(dash_idx, graph_idx, graph) @@ -109,9 +134,10 @@ pub async fn dash_ui(State(config): State, Path(dash_idx): Path) } fn dash_elements(config: State>>, dash_idx: usize) -> maud::PreEscaped { - let dash = config.get(dash_idx).expect("No such dashboard"); + let dash = config.get(dash_idx).expect(&format!("No such dashboard {}", dash_idx)); let graph_iter = dash .graphs + .as_ref().expect("No graphs in this dashboard") .iter() .enumerate() .collect::>(); diff --git a/static/lib.js b/static/lib.js index 739afa4..48c90ad 100644 --- a/static/lib.js +++ b/static/lib.js @@ -16,7 +16,7 @@ function getCssVariableValue(variableName) { return getComputedStyle(document.documentElement).getPropertyValue(variableName); } -class TimeseriesGraph extends HTMLElement { +class GraphPlot extends HTMLElement { #uri; #width; #height; @@ -94,7 +94,7 @@ class TimeseriesGraph extends HTMLElement { this.stopInterval() } - static elementName = "timeseries-graph"; + static elementName = "graph-plot"; getTargetNode() { return this.#targetNode; @@ -122,8 +122,8 @@ class TimeseriesGraph extends HTMLElement { } static registerElement() { - if (!customElements.get(TimeseriesGraph.elementName)) { - customElements.define(TimeseriesGraph.elementName, TimeseriesGraph); + if (!customElements.get(GraphPlot.elementName)) { + customElements.define(GraphPlot.elementName, GraphPlot); } } @@ -350,7 +350,7 @@ class TimeseriesGraph extends HTMLElement { } } -TimeseriesGraph.registerElement(); +GraphPlot.registerElement(); class SpanSelector extends HTMLElement { #targetNode = null; @@ -389,7 +389,7 @@ class SpanSelector extends HTMLElement { } updateGraphs() { - for (var node of document.getElementsByTagName(TimeseriesGraph.elementName)) { + for (var node of document.getElementsByTagName(GraphPlot.elementName)) { node.setAttribute('end', this.#endInput.value); node.setAttribute('duration', this.#durationInput.value); node.setAttribute('step-duration', this.#stepDurationInput.value); diff --git a/static/site.css b/static/site.css index b7e6922..f8cf0c5 100644 --- a/static/site.css +++ b/static/site.css @@ -84,7 +84,7 @@ body * { flex: 0 1 auto; } -timeseries-graph { +graph-plot { background-color: var(--paper-background-color); border-radius: 4px; display: flex; From 1de77b189d5b792af1f0b3689061368ac91552fc Mon Sep 17 00:00:00 2001 From: Jeremy Wall Date: Sat, 2 Mar 2024 10:13:37 -0500 Subject: [PATCH 03/15] fix: flake url warning --- flake.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index 7b103f1..643da5f 100644 --- a/flake.nix +++ b/flake.nix @@ -7,7 +7,7 @@ }; naersk.url = "github:nix-community/naersk"; flake-compat = { - url = github:edolstra/flake-compat; + url = "github:edolstra/flake-compat"; flake = false; }; flake-utils.url = "github:numtide/flake-utils"; From 464f5db99cae6eec52ee95af78dad30dd10dda32 Mon Sep 17 00:00:00 2001 From: Jeremy Wall Date: Sat, 2 Mar 2024 10:14:36 -0500 Subject: [PATCH 04/15] maint: formatting fixes --- src/dashboard.rs | 32 +++++++++++++++------- src/main.rs | 10 ++++--- src/query/mod.rs | 5 ++-- src/query/prom.rs | 2 +- src/routes.rs | 70 ++++++++++++++++++++++++++++++++--------------- static/lib.js | 1 - 6 files changed, 79 insertions(+), 41 deletions(-) diff --git a/src/dashboard.rs b/src/dashboard.rs index 35eb634..65749f1 100644 --- a/src/dashboard.rs +++ b/src/dashboard.rs @@ -13,14 +13,16 @@ // limitations under the License. use std::path::Path; +use anyhow::Result; use chrono::prelude::*; use chrono::Duration; -use serde::{Serialize, Deserialize}; +use serde::{Deserialize, Serialize}; use serde_yaml; use tracing::{debug, error}; -use anyhow::Result; -use crate::query::{PromQueryConn, QueryType, QueryResult, LokiConn, prom_to_samples, loki_to_sample}; +use crate::query::{ + loki_to_sample, prom_to_samples, LokiConn, PromQueryConn, QueryResult, QueryType, +}; #[derive(Serialize, Deserialize, Debug, Clone)] pub struct PlotMeta { @@ -118,22 +120,27 @@ pub struct LogStream { pub query_type: QueryType, } -pub async fn prom_query_data(graph: &Graph, dash: &Dashboard, query_span: Option) -> Result> { +pub async fn prom_query_data( + graph: &Graph, + dash: &Dashboard, + query_span: Option, +) -> Result> { let connections = graph.get_query_connections(&dash.span, &query_span); let mut data = Vec::new(); for conn in connections { data.push(prom_to_samples( - conn.get_results() - .await? - .data() - .clone(), + conn.get_results().await?.data().clone(), conn.meta, )); } Ok(data) } -pub async fn loki_query_data(stream: &LogStream, dash: &Dashboard, query_span: Option) -> Result { +pub async fn loki_query_data( + stream: &LogStream, + dash: &Dashboard, + query_span: Option, +) -> Result { let conn = stream.get_query_connection(&dash.span, &query_span); let response = conn.get_results().await?; if response.status == "success" { @@ -206,7 +213,12 @@ impl Graph { source = plot.source, "Getting query connection for graph", ); - let mut conn = PromQueryConn::new(&plot.source, &plot.query, self.query_type.clone(), plot.meta.clone()); + let mut conn = PromQueryConn::new( + &plot.source, + &plot.query, + self.query_type.clone(), + plot.meta.clone(), + ); // Query params take precendence over all other settings. Then graph settings take // precedences and finally the dashboard settings take precendence if let Some((end, duration, step_duration)) = graph_span_to_tuple(query_span) { diff --git a/src/main.rs b/src/main.rs index 829c7a4..495eedc 100644 --- a/src/main.rs +++ b/src/main.rs @@ -11,15 +11,15 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -use std::path::PathBuf; use anyhow; use axum::{self, extract::State, routing::*, Router}; use clap::{self, Parser, ValueEnum}; -use dashboard::{Dashboard, prom_query_data}; +use dashboard::{prom_query_data, Dashboard}; +use std::path::PathBuf; use tokio::net::TcpListener; use tower_http::trace::TraceLayer; -use tracing::{error, info}; use tracing::Level; +use tracing::{error, info}; use tracing_subscriber::FmtSubscriber; mod dashboard; @@ -101,7 +101,9 @@ async fn main() -> anyhow::Result<()> { .layer(TraceLayer::new_for_http()) .with_state(State(config.clone())); let socket_addr = args.listen.unwrap_or("127.0.0.1:3000".to_string()); - let listener = TcpListener::bind(socket_addr).await.expect("Unable to bind listener to address"); + let listener = TcpListener::bind(socket_addr) + .await + .expect("Unable to bind listener to address"); axum::serve(listener, router).await?; Ok(()) } diff --git a/src/query/mod.rs b/src/query/mod.rs index 59d2036..596df70 100644 --- a/src/query/mod.rs +++ b/src/query/mod.rs @@ -13,8 +13,8 @@ // limitations under the License. use std::collections::HashMap; -use serde::{Serialize, Deserialize}; use chrono::prelude::*; +use serde::{Deserialize, Serialize}; use crate::dashboard::PlotMeta; @@ -34,7 +34,6 @@ pub struct TimeSpan { pub step_seconds: i64, } - #[derive(Serialize, Deserialize, Debug)] pub struct DataPoint { timestamp: f64, @@ -92,5 +91,5 @@ impl std::fmt::Debug for QueryResult { } } -pub use prom::*; pub use loki::*; +pub use prom::*; diff --git a/src/query/prom.rs b/src/query/prom.rs index fd0c84d..ace9c88 100644 --- a/src/query/prom.rs +++ b/src/query/prom.rs @@ -22,7 +22,7 @@ use tracing::debug; use crate::dashboard::PlotMeta; -use super::{QueryType, TimeSpan, QueryResult, DataPoint}; +use super::{DataPoint, QueryResult, QueryType, TimeSpan}; #[derive(Debug)] pub struct PromQueryConn<'conn> { diff --git a/src/routes.rs b/src/routes.rs index b3f3f6f..c04d20a 100644 --- a/src/routes.rs +++ b/src/routes.rs @@ -22,10 +22,12 @@ use axum::{ // https://maud.lambda.xyz/getting-started.html use maud::{html, Markup}; -use serde::{Serialize, Deserialize}; +use serde::{Deserialize, Serialize}; use tracing::debug; -use crate::dashboard::{Dashboard, Graph, GraphSpan, AxisDefinition, Orientation, prom_query_data, loki_query_data}; +use crate::dashboard::{ + loki_query_data, prom_query_data, AxisDefinition, Dashboard, Graph, GraphSpan, Orientation, +}; use crate::query::QueryResult; type Config = State>>; @@ -43,12 +45,23 @@ pub async fn loki_query( Path((dash_idx, loki_idx)): Path<(usize, usize)>, Query(query): Query>, ) -> Json { - let dash = config.get(dash_idx).expect(&format!("No such dashboard index {}", dash_idx)); - let log = dash.logs - .as_ref().expect("No logs in this dashboard") - .get(loki_idx).expect(&format!("No such log query {}", loki_idx)); - let plots = vec![loki_query_data(log, dash, query_to_graph_span(query)).await.expect("Unable to get log query results")]; - Json(GraphPayload{legend_orientation: None, yaxes: log.yaxes.clone(), plots}) + let dash = config + .get(dash_idx) + .expect(&format!("No such dashboard index {}", dash_idx)); + let log = dash + .logs + .as_ref() + .expect("No logs in this dashboard") + .get(loki_idx) + .expect(&format!("No such log query {}", loki_idx)); + let plots = vec![loki_query_data(log, dash, query_to_graph_span(query)) + .await + .expect("Unable to get log query results")]; + Json(GraphPayload { + legend_orientation: None, + yaxes: log.yaxes.clone(), + plots, + }) } pub async fn graph_query( @@ -57,14 +70,23 @@ pub async fn graph_query( Query(query): Query>, ) -> Json { debug!("Getting data for query"); - let dash = config.get(dash_idx).expect(&format!("No such dashboard index {}", dash_idx)); + let dash = config + .get(dash_idx) + .expect(&format!("No such dashboard index {}", dash_idx)); let graph = dash .graphs - .as_ref().expect("No graphs in this dashboard") + .as_ref() + .expect("No graphs in this dashboard") .get(graph_idx) .expect(&format!("No such graph in dasboard {}", dash_idx)); - let plots = prom_query_data(graph, dash, query_to_graph_span(query)).await.expect("Unable to get query results"); - Json(GraphPayload{legend_orientation: graph.legend_orientation.clone(), yaxes: graph.yaxes.clone(), plots}) + let plots = prom_query_data(graph, dash, query_to_graph_span(query)) + .await + .expect("Unable to get query results"); + Json(GraphPayload { + legend_orientation: graph.legend_orientation.clone(), + yaxes: graph.yaxes.clone(), + plots, + }) } fn query_to_graph_span(query: HashMap) -> Option { @@ -89,13 +111,13 @@ pub fn mk_api_routes(config: Arc>) -> Router { // Query routes Router::new() .route( - "/dash/:dash_idx/graph/:graph_idx", - get(graph_query).with_state(config.clone()), - ) + "/dash/:dash_idx/graph/:graph_idx", + get(graph_query).with_state(config.clone()), + ) .route( - "/dash/:dash_idx/log/:log_idx", - get(loki_query).with_state(config), - ) + "/dash/:dash_idx/log/:log_idx", + get(loki_query).with_state(config), + ) } pub fn graph_component(dash_idx: usize, graph_idx: usize, graph: &Graph) -> Markup { @@ -105,7 +127,7 @@ pub fn graph_component(dash_idx: usize, graph_idx: usize, graph: &Graph) -> Mark html!( div { h2 { (graph.title) " - " a href=(graph_embed_uri) { "embed url" } } - @if graph.d3_tick_format.is_some() { + @if graph.d3_tick_format.is_some() { graph-plot uri=(graph_data_uri) id=(graph_id) d3-tick-format=(graph.d3_tick_format.as_ref().unwrap()) { } } @else { graph-plot uri=(graph_data_uri) id=(graph_id) { } @@ -122,7 +144,8 @@ pub async fn graph_ui( .get(dash_idx) .expect(&format!("No such dashboard {}", dash_idx)) .graphs - .as_ref().expect("No graphs in this dashboard") + .as_ref() + .expect("No graphs in this dashboard") .get(graph_idx) .expect("No such graph"); graph_component(dash_idx, graph_idx, graph) @@ -134,10 +157,13 @@ pub async fn dash_ui(State(config): State, Path(dash_idx): Path) } fn dash_elements(config: State>>, dash_idx: usize) -> maud::PreEscaped { - let dash = config.get(dash_idx).expect(&format!("No such dashboard {}", dash_idx)); + let dash = config + .get(dash_idx) + .expect(&format!("No such dashboard {}", dash_idx)); let graph_iter = dash .graphs - .as_ref().expect("No graphs in this dashboard") + .as_ref() + .expect("No graphs in this dashboard") .iter() .enumerate() .collect::>(); diff --git a/static/lib.js b/static/lib.js index 48c90ad..e692448 100644 --- a/static/lib.js +++ b/static/lib.js @@ -86,7 +86,6 @@ class GraphPlot extends HTMLElement { this.#duration = this.getAttribute('duration') || null; this.#step_duration = this.getAttribute('step-duration') || null; this.#d3TickFormat = this.getAttribute('d3-tick-format') || this.#d3TickFormat; - var self = this; this.reset(); } From fb48c6900c8c10599247d7fc6d08ecfde70ec4bd Mon Sep 17 00:00:00 2001 From: Jeremy Wall Date: Sun, 3 Mar 2024 15:23:31 -0500 Subject: [PATCH 05/15] maint: JSDoc type annotations and tsserver configuration --- jsconfig.json | 11 ++++ static/lib.js | 147 ++++++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 141 insertions(+), 17 deletions(-) create mode 100644 jsconfig.json diff --git a/jsconfig.json b/jsconfig.json new file mode 100644 index 0000000..679d8ed --- /dev/null +++ b/jsconfig.json @@ -0,0 +1,11 @@ +{ + "compilerOptions": { + "target": "es2022", + "noImplicitThis": true, + "checkJs": true, + "allowJs": true + }, + "include": [ + "static/*.js" + ] +} diff --git a/static/lib.js b/static/lib.js index e692448..2cbce54 100644 --- a/static/lib.js +++ b/static/lib.js @@ -12,25 +12,79 @@ // See the License for the specific language governing permissions and // limitations under the License. +/** + * @typedef PlotList + * @type {object} + * @property {?Array} Series + * @property {?Array} Scalar + * @property {?Array} StreamInstant + * @property {?Array} Stream + */ + +/** + * @typedef QueryData + * @type {object} + * @property {object} yaxes + * @property {?string} legend_orientation + * @property {Array} plots + */ + +/** + * @typedef PlotTrace + * @type {object} + * @property {string=} name + * @property type {string} + * @property {string=} mode + * @property {Array} x + * @property {Array} y + * @peroperty {string=} xaxis + * @peroperty {string=} yaxis +*/ + +/** + * Get's a css variable's value from the document. + * @param {string} variableName - Name of the variable to get `--var-name` + * @returns string + */ function getCssVariableValue(variableName) { return getComputedStyle(document.documentElement).getPropertyValue(variableName); } +/** + * Custom element for showing a plotly graph. + * + * @extends HTMLElement + */ class GraphPlot extends HTMLElement { + /** @type {?string} */ #uri; + /** @type {?number} */ #width; + /** @type {?number} */ #height; + /** @type {?number} */ #intervalId; + /** @type {?number} */ #pollSeconds; + /** @type {?string} */ #end; + /** @type {?number} */ #duration; + /** @type {?string} */ #step_duration; + /** @type {?string} */ #d3TickFormat = "~s"; + /** @type {?HTMLDivElement} */ #targetNode = null; + /** @type {?HTMLElement} */ #menuContainer = null; + /** @type {Object} */ #filterSelectElements = {}; + /** @type {Object>} */ #filterLabels = {}; + /** @type {Object>} */ #filteredLabelSets = {}; + constructor() { super(); this.#width = 800; @@ -45,25 +99,32 @@ class GraphPlot extends HTMLElement { static observedAttributes = ['uri', 'width', 'height', 'poll-seconds', 'end', 'duration', 'step-duration', 'd3-tick-format']; + /** + * Callback for attributes changes. + * + * @param {string} name - The name of the attribute. + * @param {?string} _oldValue - The old value for the attribute + * @param {?string} newValue - The new value for the attribute + */ attributeChangedCallback(name, _oldValue, newValue) { switch (name) { case 'uri': this.#uri = newValue; break; case 'width': - this.#width = newValue; + this.#width = Number(newValue); break; case 'height': - this.#height = newValue; + this.#height = Number(newValue); break; case 'poll-seconds': - this.#pollSeconds = newValue; + this.#pollSeconds = Number(newValue); break; case 'end': this.#end = newValue; break; case 'duration': - this.#duration = newValue; + this.#duration = Number(newValue); break; case 'step-duration': this.#step_duration = newValue; @@ -79,11 +140,11 @@ class GraphPlot extends HTMLElement { connectedCallback() { this.#uri = this.getAttribute('uri') || this.#uri; - this.#width = this.getAttribute('width') || this.#width; - this.#height = this.getAttribute('height') || this.#height; - this.#pollSeconds = this.getAttribute('poll-seconds') || this.#pollSeconds; + this.#width = Number(this.getAttribute('width') || this.#width); + this.#height = Number(this.getAttribute('height') || this.#height); + this.#pollSeconds = Number(this.getAttribute('poll-seconds') || this.#pollSeconds); this.#end = this.getAttribute('end') || null; - this.#duration = this.getAttribute('duration') || null; + this.#duration = Number(this.getAttribute('duration')) || null; this.#step_duration = this.getAttribute('step-duration') || null; this.#d3TickFormat = this.getAttribute('d3-tick-format') || this.#d3TickFormat; this.reset(); @@ -95,10 +156,17 @@ class GraphPlot extends HTMLElement { static elementName = "graph-plot"; + /* + * Get's the target node for placing the plotly graph. + * + * @returns {?HTMLDivElement} + */ getTargetNode() { return this.#targetNode; } + /** + */ stopInterval() { if (this.#intervalId) { clearInterval(this.#intervalId); @@ -106,6 +174,10 @@ class GraphPlot extends HTMLElement { } } + /** + * Resets the entire graph and then restarts polling. + * @param {boolean=} updateOnly + */ reset(updateOnly) { var self = this; self.stopInterval() @@ -120,12 +192,18 @@ class GraphPlot extends HTMLElement { }); } + /** Registers the custom element if it doesn't already exist */ static registerElement() { if (!customElements.get(GraphPlot.elementName)) { customElements.define(GraphPlot.elementName, GraphPlot); } } + /** + * Returns the uri formatted with any query strings if necessary. + * + * @returns {string} + */ getUri() { if (this.#end && this.#duration && this.#step_duration) { return this.#uri + "?end=" + this.#end + "&duration=" + this.#duration + "&step_duration=" + this.#step_duration; @@ -134,6 +212,11 @@ class GraphPlot extends HTMLElement { } } + /** + * Returns the data from an api call. + * + * @return {Promise} + */ async fetchData() { // TODO(zaphar): Can we do some massaging on these // to get the full set of labels and possible values? @@ -142,6 +225,12 @@ class GraphPlot extends HTMLElement { return data; } + /** + * Formats the name for the plot trace. + * @param {{name_format: ?string}} meta + * @param {Map} labels + * @return string + */ formatName(meta, labels) { var name = ""; const formatter = meta.name_format @@ -157,6 +246,9 @@ class GraphPlot extends HTMLElement { return name; } + /** + * @param {Object} labels + */ populateFilterData(labels) { for (var key in labels) { const label = this.#filterLabels[key]; @@ -170,13 +262,18 @@ class GraphPlot extends HTMLElement { } } + /** + * @param {string} key + * @returns {HTMLDivElement} + */ buildSelectElement(key) { // TODO(jwall): Should we have a select all? var id = key + "-select" + Math.random(); const element = document.createElement("div"); const select = document.createElement("select"); select.setAttribute("name", id); - select.setAttribute("multiple", true); + // TODO(jwall): This is how you set boolean attributes. Use the attribute named... :-( + select.setAttribute("multiple", "multiple"); const optElement = document.createElement("option"); const optValue = "Select " + key; optElement.innerText = optValue; @@ -184,7 +281,7 @@ class GraphPlot extends HTMLElement { for (var opt of this.#filterLabels[key]) { const optElement = document.createElement("option"); optElement.setAttribute("value", opt); - optElement.setAttribute("selected", true); + optElement.setAttribute("selected", "selected"); optElement.innerText = opt; select.appendChild(optElement); } @@ -193,7 +290,7 @@ class GraphPlot extends HTMLElement { select.onchange = function(evt) { evt.stopPropagation(); var filteredValues = []; - for (var opt of evt.target.selectedOptions) { + for (var opt of /** @type {HTMLSelectElement} */(evt.target).selectedOptions) { filteredValues.push(opt.getAttribute("value")); } self.#filteredLabelSets[key] = filteredValues; @@ -217,6 +314,9 @@ class GraphPlot extends HTMLElement { this.#menuContainer.replaceChildren(...children); } + /** + * @param {QueryData} graph + */ getLabelsForData(graph) { const data = graph.plots; for (var subplot of data) { @@ -247,6 +347,11 @@ class GraphPlot extends HTMLElement { }; } + /** + * Update the graph with new data. + * + * @param {?QueryData=} maybeGraph + */ async updateGraph(maybeGraph) { var graph = maybeGraph; if (!graph) { @@ -278,10 +383,9 @@ class GraphPlot extends HTMLElement { yaxis.gridColor = getCssVariableValue("--accent-color"); layout[nextYaxis()] = yaxis; } - var traces = []; + var traces = /** @type {Array} */ ([]); for (var subplot_idx in data) { const subplot = data[subplot_idx]; - const subplotCount = Number(subplot_idx) + 1; var nextYaxis = this.yaxisNameGenerator(); if (subplot.Series) { // https://plotly.com/javascript/reference/scatter/ @@ -297,7 +401,7 @@ class GraphPlot extends HTMLElement { var yaxis = meta.yaxis || "y"; // https://plotly.com/javascript/reference/layout/yaxis/ const series = triple[2]; - var trace = { + const trace = { type: "scatter", mode: "lines+text", x: [], @@ -330,32 +434,39 @@ class GraphPlot extends HTMLElement { } const meta = triple[1]; const series = triple[2]; - var trace = { + const trace = /** @type PlotTrace */({ type: "bar", x: [], y: [], yhoverformat: meta["d3_tick_format"], - }; + }); var name = this.formatName(meta, labels); if (name) { trace.name = name; } trace.y.push(series.value); trace.x.push(trace.name); traces.push(trace); } - } + } // TODO(zaphar): subplot.Stream // log lines!!! } // https://plotly.com/javascript/plotlyjs-function-reference/#plotlyreact + // @ts-ignore Plotly.react(this.getTargetNode(), traces, layout, null); } } GraphPlot.registerElement(); +/** Custom Element for selecting a timespan for the dashboard. */ class SpanSelector extends HTMLElement { + /** @type {HTMLElement} */ #targetNode = null; + /** @type {HTMLInputElement} */ #endInput = null; + /** @type {HTMLInputElement} */ #durationInput = null; + /** @type {HTMLInputElement} */ #stepDurationInput = null; + /** @type {HTMLButtonElement} */ #updateInput = null constructor() { @@ -387,6 +498,7 @@ class SpanSelector extends HTMLElement { this.#updateInput.onclick = undefined; } + /** Updates all the graphs on the dashboard with the new timespan. */ updateGraphs() { for (var node of document.getElementsByTagName(GraphPlot.elementName)) { node.setAttribute('end', this.#endInput.value); @@ -397,6 +509,7 @@ class SpanSelector extends HTMLElement { static elementName = "span-selector"; + /** Register the element if it doesn't exist */ static registerElement() { if (!customElements.get(SpanSelector.elementName)) { customElements.define(SpanSelector.elementName, SpanSelector); From 17e6ae81a0bbc0392930127d77b30fd918570381 Mon Sep 17 00:00:00 2001 From: Jeremy Wall Date: Sun, 3 Mar 2024 18:15:41 -0500 Subject: [PATCH 06/15] Turn our lib into an es6 module --- src/routes.rs | 2 +- static/lib.js | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/routes.rs b/src/routes.rs index c04d20a..097b6f9 100644 --- a/src/routes.rs +++ b/src/routes.rs @@ -191,7 +191,7 @@ pub fn mk_ui_routes(config: Arc>) -> Router { fn graph_lib_prelude() -> Markup { html! { script src="/js/plotly.js" { } - script defer src="/js/lib.js" { } + script type="module" defer src="/js/lib.js" { } link rel="stylesheet" href="/static/site.css" { } } } diff --git a/static/lib.js b/static/lib.js index 2cbce54..ec2d447 100644 --- a/static/lib.js +++ b/static/lib.js @@ -55,7 +55,7 @@ function getCssVariableValue(variableName) { * * @extends HTMLElement */ -class GraphPlot extends HTMLElement { +export class GraphPlot extends HTMLElement { /** @type {?string} */ #uri; /** @type {?number} */ @@ -457,7 +457,7 @@ class GraphPlot extends HTMLElement { GraphPlot.registerElement(); /** Custom Element for selecting a timespan for the dashboard. */ -class SpanSelector extends HTMLElement { +export class SpanSelector extends HTMLElement { /** @type {HTMLElement} */ #targetNode = null; /** @type {HTMLInputElement} */ @@ -518,3 +518,4 @@ class SpanSelector extends HTMLElement { } SpanSelector.registerElement(); + From f69ea6d6faaf45ec183521902dd573a046b6cb87 Mon Sep 17 00:00:00 2001 From: Jeremy Wall Date: Mon, 4 Mar 2024 20:52:36 -0500 Subject: [PATCH 07/15] ui: Show log results in our dashboards --- src/query/loki.rs | 6 ++-- src/routes.rs | 44 ++++++++++++++++++++----- static/lib.js | 84 ++++++++++++++++++++++++++++++++++++++++------- 3 files changed, 113 insertions(+), 21 deletions(-) diff --git a/src/query/loki.rs b/src/query/loki.rs index 8152593..43e4c87 100644 --- a/src/query/loki.rs +++ b/src/query/loki.rs @@ -22,7 +22,7 @@ use tracing::{debug, error}; use super::{LogLine, QueryResult, QueryType, TimeSpan}; // TODO(jwall): Should I allow non stream returns? -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, PartialEq)] pub enum ResultType { /// Returned by query endpoints #[serde(rename = "vector")] @@ -85,8 +85,10 @@ pub fn loki_to_sample(data: LokiData) -> QueryResult { } QueryResult::StreamInstant(values) } + // Stream types are nanoseconds. // Matrix types are seconds ResultType::Matrix | ResultType::Streams => { let mut values = Vec::with_capacity(data.result.len()); + let multiple = (if data.result_type == ResultType::Matrix { 1000000 } else { 1 }) as f64; for result in data.result { if let Some(value) = result.values { values.push(( @@ -94,7 +96,7 @@ pub fn loki_to_sample(data: LokiData) -> QueryResult { value .into_iter() .map(|(timestamp, line)| LogLine { - timestamp: timestamp.parse::().expect("Invalid f64 type"), + timestamp: multiple * timestamp.parse::().expect("Invalid f64 type"), line, }) .collect(), diff --git a/src/routes.rs b/src/routes.rs index 097b6f9..f5106dd 100644 --- a/src/routes.rs +++ b/src/routes.rs @@ -26,7 +26,7 @@ use serde::{Deserialize, Serialize}; use tracing::debug; use crate::dashboard::{ - loki_query_data, prom_query_data, AxisDefinition, Dashboard, Graph, GraphSpan, Orientation, + loki_query_data, prom_query_data, AxisDefinition, Dashboard, Graph, GraphSpan, Orientation, LogStream, }; use crate::query::QueryResult; @@ -120,6 +120,18 @@ pub fn mk_api_routes(config: Arc>) -> Router { ) } +pub fn log_component(dash_idx: usize, log_idx: usize, log: &LogStream) -> Markup { + let log_id = format!("log-{}-{}", dash_idx, log_idx); + let log_data_uri = format!("/api/dash/{}/log/{}", dash_idx, log_idx); + let log_embed_uri = format!("/embed/dash/{}/log/{}", dash_idx, log_idx); + html! { + div { + h2 { (log.title) " - " a href=(log_embed_uri) { "embed url" } } + graph-plot uri=(log_data_uri) id=(log_id) { } + } + } +} + pub fn graph_component(dash_idx: usize, graph_idx: usize, graph: &Graph) -> Markup { let graph_id = format!("graph-{}-{}", dash_idx, graph_idx); let graph_data_uri = format!("/api/dash/{}/graph/{}", dash_idx, graph_idx); @@ -160,19 +172,35 @@ fn dash_elements(config: State>>, dash_idx: usize) -> maud::P let dash = config .get(dash_idx) .expect(&format!("No such dashboard {}", dash_idx)); - let graph_iter = dash + let graph_components = if let Some(graphs) = dash .graphs - .as_ref() - .expect("No graphs in this dashboard") - .iter() + .as_ref() { + let graph_iter = graphs.iter() .enumerate() .collect::>(); + Some(html! { + @for (idx, graph) in &graph_iter { + (graph_component(dash_idx, *idx, *graph)) + } + }) + } else { + None + }; + let log_components = if let Some(logs) = dash.logs.as_ref() { + let log_iter = logs.iter().enumerate().collect::>(); + Some(html! { + @for (idx, log) in &log_iter { + (log_component(dash_idx, *idx, *log)) + } + }) + } else { + None + }; html!( h1 { (dash.title) } span-selector class="row-flex" {} - @for (idx, graph) in &graph_iter { - (graph_component(dash_idx, *idx, *graph)) - } + @if graph_components.is_some() { (graph_components.unwrap()) } + @if log_components.is_some() { (log_components.unwrap()) } ) } diff --git a/static/lib.js b/static/lib.js index ec2d447..80dd4d1 100644 --- a/static/lib.js +++ b/static/lib.js @@ -15,10 +15,10 @@ /** * @typedef PlotList * @type {object} - * @property {?Array} Series - * @property {?Array} Scalar - * @property {?Array} StreamInstant - * @property {?Array} Stream + * @property {Array=} Series + * @property {Array=} Scalar + * @property {Array<{timestamp: string, line: string}>=} StreamInstant - Timestamps are in seconds + * @property {Array<{timestamp: string, line: string}>=} Stream - Timestamps are in nanoseconds */ /** @@ -29,16 +29,43 @@ * @property {Array} plots */ +/** + * @typedef HeaderOrCell + * @type {object} + * @property {array} values + * @property {string=} fill + * @property {{width: number, color: string}=} line + * @property {{family: string, size: number, color: string }=} font + */ + /** - * @typedef PlotTrace + * @typedef TableTrace * @type {object} * @property {string=} name * @property type {string} * @property {string=} mode + * @property {HeaderOrCell} headers + * @property {HeaderOrCell} cells - An Array of columns for the table. + * @property {string=} xaxis + * @property {string=} yaxis +*/ + +/** + * @typedef GraphTrace + * @type {object} + * @property {string=} name + * @property {string=} fill + * @property type {string} + * @property {string=} mode * @property {Array} x * @property {Array} y - * @peroperty {string=} xaxis - * @peroperty {string=} yaxis + * @property {string=} xaxis + * @property {string=} yaxis +*/ + +/** + * @typedef PlotTrace + * @type {(TableTrace|GraphTrace)} */ /** @@ -401,7 +428,7 @@ export class GraphPlot extends HTMLElement { var yaxis = meta.yaxis || "y"; // https://plotly.com/javascript/reference/layout/yaxis/ const series = triple[2]; - const trace = { + const trace = /** @type GraphTrace */({ type: "scatter", mode: "lines+text", x: [], @@ -410,7 +437,7 @@ export class GraphPlot extends HTMLElement { xaxis: "x", yaxis: yaxis, //yhoverformat: yaxis.tickformat, - }; + }); if (meta.fill) { trace.fill = meta.fill; } @@ -434,7 +461,7 @@ export class GraphPlot extends HTMLElement { } const meta = triple[1]; const series = triple[2]; - const trace = /** @type PlotTrace */({ + const trace = /** @type GraphTrace */({ type: "bar", x: [], y: [], @@ -446,7 +473,42 @@ export class GraphPlot extends HTMLElement { trace.x.push(trace.name); traces.push(trace); } - } // TODO(zaphar): subplot.Stream // log lines!!! + } else if (subplot.Stream) { + // TODO(zaphar): subplot.Stream // log lines!!! + const trace = /** @type TableTrace */({ + type: "table", + headers: { + align: "left", + values: ["Timestamp", "Log"] + }, + cells: { + align: "left", + values: [] + }, + }); + const dateColumn = []; + const logColumn = []; + + loopStream: for (const pair of subplot.Stream) { + const labels = pair[0]; + for (var label in labels) { + var show = this.#filteredLabelSets[label]; + if (show && !show.includes(labels[label])) { + continue loopStream; + } + } + const lines = pair[1]; + // TODO(jwall): Headers + for (const line of lines) { + // For streams the timestamps are in nanoseconds + dateColumn.push(new Date(line.timestamp / 1000000)); + logColumn.push(line.line); + } + } + trace.cells.values.push(dateColumn); + trace.cells.values.push(logColumn); + traces.push(trace); + } } // https://plotly.com/javascript/plotlyjs-function-reference/#plotlyreact // @ts-ignore From 474bfe4f1dbee345bfe32a280744c586574df890 Mon Sep 17 00:00:00 2001 From: Jeremy Wall Date: Mon, 4 Mar 2024 20:57:36 -0500 Subject: [PATCH 08/15] ui: Show the labels for filtering on logs --- static/lib.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/static/lib.js b/static/lib.js index 80dd4d1..c2f5515 100644 --- a/static/lib.js +++ b/static/lib.js @@ -359,6 +359,12 @@ export class GraphPlot extends HTMLElement { this.populateFilterData(labels); } } + if (subplot.Stream) { + for (const pair of subplot.Stream) { + const labels = pair[0]; + this.populateFilterData(labels); + } + } } } From a7a6c99099749406f5dd1833653af2f1ba022b54 Mon Sep 17 00:00:00 2001 From: Jeremy Wall Date: Mon, 4 Mar 2024 21:05:37 -0500 Subject: [PATCH 09/15] ui: log table styling --- static/lib.js | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/static/lib.js b/static/lib.js index c2f5515..3760ff4 100644 --- a/static/lib.js +++ b/static/lib.js @@ -33,7 +33,7 @@ * @typedef HeaderOrCell * @type {object} * @property {array} values - * @property {string=} fill + * @property {{color: string}=} fill * @property {{width: number, color: string}=} line * @property {{family: string, size: number, color: string }=} font */ @@ -42,7 +42,7 @@ * @typedef TableTrace * @type {object} * @property {string=} name - * @property type {string} + * @property {string} type * @property {string=} mode * @property {HeaderOrCell} headers * @property {HeaderOrCell} cells - An Array of columns for the table. @@ -483,13 +483,16 @@ export class GraphPlot extends HTMLElement { // TODO(zaphar): subplot.Stream // log lines!!! const trace = /** @type TableTrace */({ type: "table", + // TODO(zaphar): Column width? headers: { align: "left", - values: ["Timestamp", "Log"] + values: ["Timestamp", "Log"], + fill: { color: layout.xaxis.gridColor } }, cells: { align: "left", - values: [] + values: [], + fill: { color: layout.paper_bgcolor } }, }); const dateColumn = []; From 01cd54afa443a0cd1e9d9fe3e8a0d5b73dfc27c0 Mon Sep 17 00:00:00 2001 From: Jeremy Wall Date: Tue, 5 Mar 2024 20:11:01 -0500 Subject: [PATCH 10/15] ui: Show labels for the log line. --- static/lib.js | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/static/lib.js b/static/lib.js index 3760ff4..eb22ead 100644 --- a/static/lib.js +++ b/static/lib.js @@ -36,6 +36,7 @@ * @property {{color: string}=} fill * @property {{width: number, color: string}=} line * @property {{family: string, size: number, color: string }=} font + * @property {Array=} columnwidth */ /** @@ -480,13 +481,14 @@ export class GraphPlot extends HTMLElement { traces.push(trace); } } else if (subplot.Stream) { - // TODO(zaphar): subplot.Stream // log lines!!! + // TODO(jwall): It's possible that this should actually be a separate custom + // element. const trace = /** @type TableTrace */({ type: "table", - // TODO(zaphar): Column width? + columnwidth: [10, 20, 70], headers: { align: "left", - values: ["Timestamp", "Log"], + values: ["Timestamp","Label", "Log"], fill: { color: layout.xaxis.gridColor } }, cells: { @@ -496,25 +498,32 @@ export class GraphPlot extends HTMLElement { }, }); const dateColumn = []; + const metaColumn = []; const logColumn = []; loopStream: for (const pair of subplot.Stream) { const labels = pair[0]; + var labelList = []; for (var label in labels) { var show = this.#filteredLabelSets[label]; if (show && !show.includes(labels[label])) { continue loopStream; } + labelList.push(`${label}:${labels[label]}`); } + const labelsName = labelList.join("
"); const lines = pair[1]; // TODO(jwall): Headers for (const line of lines) { // For streams the timestamps are in nanoseconds + // TODO(zaphar): We should improve the timstamp formatting a bit dateColumn.push(new Date(line.timestamp / 1000000)); + metaColumn.push(labelsName); logColumn.push(line.line); } } trace.cells.values.push(dateColumn); + trace.cells.values.push(metaColumn); trace.cells.values.push(logColumn); traces.push(trace); } From 6a503947d8edfee23cae5ef4432977f3976cf421 Mon Sep 17 00:00:00 2001 From: Jeremy Wall Date: Tue, 5 Mar 2024 20:32:04 -0500 Subject: [PATCH 11/15] ui: respect ansi colors in our log lines --- static/lib.js | 42 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 39 insertions(+), 3 deletions(-) diff --git a/static/lib.js b/static/lib.js index eb22ead..0b51d23 100644 --- a/static/lib.js +++ b/static/lib.js @@ -69,6 +69,41 @@ * @type {(TableTrace|GraphTrace)} */ +/** + * Map ansi terminal codes to html color codes. + * @param {string} line + */ +function ansiToHtml(line) { + const ansiToHtmlMap = { + // Map ANSI color codes to HTML color names or hex values + // We don't necessarily handle all the colors but this is enough to start. + "30": "black", + "31": "red", + "32": "green", + "33": "yellow", + "34": "blue", + "35": "magenta", + "36": "cyan", + "37": "white", + "39": "initial" + }; + + // NOTE(zaphar): Yes this is gross and I should really do a better parser but I'm lazy. + // Replace ANSI codes with HTML span elements styled with the corresponding color + return line.replace(/\x1b\[([0-9;]*)m/g, (match, p1) => { + const parts = p1.split(';'); // ANSI codes can be compounded, e.g., "1;31" for bold red + let styles = ''; + for (let part of parts) { + if (ansiToHtmlMap[part]) { + // If the code is a color, map it to a CSS color + styles += `color: ${ansiToHtmlMap[part]};`; + } + // TODO(zaphar): Add more conditions here to handle other styles like bold or underline? + } + return styles ? `` : ''; + }) + ''; +} + /** * Get's a css variable's value from the document. * @param {string} variableName - Name of the variable to get `--var-name` @@ -485,7 +520,7 @@ export class GraphPlot extends HTMLElement { // element. const trace = /** @type TableTrace */({ type: "table", - columnwidth: [10, 20, 70], + columnwidth: [15, 20, 70], headers: { align: "left", values: ["Timestamp","Label", "Log"], @@ -517,9 +552,10 @@ export class GraphPlot extends HTMLElement { for (const line of lines) { // For streams the timestamps are in nanoseconds // TODO(zaphar): We should improve the timstamp formatting a bit - dateColumn.push(new Date(line.timestamp / 1000000)); + let timestamp = new Date(line.timestamp / 1000000); + dateColumn.push(timestamp.toISOString()); metaColumn.push(labelsName); - logColumn.push(line.line); + logColumn.push(ansiToHtml(line.line)); } } trace.cells.values.push(dateColumn); From c830d1530db9dc547b15c5bcf508fff9533eadc0 Mon Sep 17 00:00:00 2001 From: Jeremy Wall Date: Tue, 5 Mar 2024 20:39:03 -0500 Subject: [PATCH 12/15] ui: better background color for the plots --- static/lib.js | 4 ++-- static/site.css | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/static/lib.js b/static/lib.js index 0b51d23..7ae17e0 100644 --- a/static/lib.js +++ b/static/lib.js @@ -431,7 +431,7 @@ export class GraphPlot extends HTMLElement { var layout = { displayModeBar: false, responsive: true, - plot_bgcolor: getCssVariableValue('--paper-background-color').trim(), + plot_bgcolor: getCssVariableValue('--plot-background-color').trim(), paper_bgcolor: getCssVariableValue('--paper-background-color').trim(), font: { color: getCssVariableValue('--text-color').trim() @@ -529,7 +529,7 @@ export class GraphPlot extends HTMLElement { cells: { align: "left", values: [], - fill: { color: layout.paper_bgcolor } + fill: { color: layout.plot_bgcolor } }, }); const dateColumn = []; diff --git a/static/site.css b/static/site.css index f8cf0c5..9127089 100644 --- a/static/site.css +++ b/static/site.css @@ -3,6 +3,7 @@ --background-color: #FFFFFF; /* Light background */ --text-color: #333333; /* Dark text for contrast */ --paper-background-color: #F0F0F0; + --plot-background-color: #F0F0F0; --accent-color: #6200EE; /* For buttons and interactive elements */ /* Graph colors */ @@ -27,6 +28,7 @@ /* Solarized Dark Base Colors */ --background-color: #002b36; /* base03 */ --paper-background-color: #003c4a; + --plot-background-color: rgb(24, 34, 21); --text-color: #839496; /* base0 */ --accent-color: #268bd2; /* blue */ From e44d2087c81b77996fbf4bc839a33d53fa2af997 Mon Sep 17 00:00:00 2001 From: Jeremy Wall Date: Tue, 5 Mar 2024 20:44:10 -0500 Subject: [PATCH 13/15] feat: you can embed log tables now. --- src/main.rs | 4 ++++ src/routes.rs | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/src/main.rs b/src/main.rs index 495eedc..cfa31d7 100644 --- a/src/main.rs +++ b/src/main.rs @@ -96,6 +96,10 @@ async fn main() -> anyhow::Result<()> { "/embed/dash/:dash_idx/graph/:graph_idx", get(routes::graph_embed).with_state(State(config.clone())), ) + .route( + "/embed/dash/:dash_idx/log/:graph_idx", + get(routes::log_embed).with_state(State(config.clone())), + ) .route("/dash/:dash_idx", get(routes::dashboard_direct)) .route("/", get(routes::index).with_state(State(config.clone()))) .layer(TraceLayer::new_for_http()) diff --git a/src/routes.rs b/src/routes.rs index f5106dd..6348728 100644 --- a/src/routes.rs +++ b/src/routes.rs @@ -163,6 +163,21 @@ pub async fn graph_ui( graph_component(dash_idx, graph_idx, graph) } +pub async fn log_ui( + State(config): State, + Path((dash_idx, log_idx)): Path<(usize, usize)>, +) -> Markup { + let log = config + .get(dash_idx) + .expect(&format!("No such dashboard {}", dash_idx)) + .logs + .as_ref() + .expect("No graphs in this dashboard") + .get(log_idx) + .expect("No such graph"); + log_component(dash_idx, log_idx, log) +} + pub async fn dash_ui(State(config): State, Path(dash_idx): Path) -> Markup { // TODO(zaphar): Should do better http error reporting here. dash_elements(config, dash_idx) @@ -241,6 +256,23 @@ pub async fn graph_embed( } } +pub async fn log_embed( + State(config): State, + Path((dash_idx, log_idx)): Path<(usize, usize)>, +) -> Markup { + html! { + html { + head { + title { ("Heracles - Prometheus Unshackled") } + } + body { + (graph_lib_prelude()) + (log_ui(State(config.clone()), Path((dash_idx, log_idx))).await) + } + } + } +} + async fn index_html(config: Config, dash_idx: Option) -> Markup { html! { html { From 4427cd414f60c4347beb76aee18e7eef0d36dbba Mon Sep 17 00:00:00 2001 From: Jeremy Wall Date: Tue, 5 Mar 2024 21:10:21 -0500 Subject: [PATCH 14/15] ui: fix some color scheme stuff --- static/lib.js | 12 ++++++++---- static/site.css | 26 +------------------------- 2 files changed, 9 insertions(+), 29 deletions(-) diff --git a/static/lib.js b/static/lib.js index 7ae17e0..15fc471 100644 --- a/static/lib.js +++ b/static/lib.js @@ -34,8 +34,11 @@ * @type {object} * @property {array} values * @property {{color: string}=} fill + * @property {object=} font + * @property {string=} font.family + * @property {number=} font.size + * @property {string=} font.color * @property {{width: number, color: string}=} line - * @property {{family: string, size: number, color: string }=} font * @property {Array=} columnwidth */ @@ -437,7 +440,7 @@ export class GraphPlot extends HTMLElement { color: getCssVariableValue('--text-color').trim() }, xaxis: { - gridcolor: getCssVariableValue("--accent-color") + gridcolor: getCssVariableValue("--grid-line-color") }, legend: { orientation: 'v' @@ -449,7 +452,7 @@ export class GraphPlot extends HTMLElement { var nextYaxis = this.yaxisNameGenerator(); for (const yaxis of yaxes) { yaxis.tickformat = yaxis.tickformat || this.#d3TickFormat; - yaxis.gridColor = getCssVariableValue("--accent-color"); + yaxis.gridColor = getCssVariableValue("--grid-line-color"); layout[nextYaxis()] = yaxis; } var traces = /** @type {Array} */ ([]); @@ -524,7 +527,8 @@ export class GraphPlot extends HTMLElement { headers: { align: "left", values: ["Timestamp","Label", "Log"], - fill: { color: layout.xaxis.gridColor } + fill: { color: layout.xaxis.paper_bgcolor }, + font: { color: getCssVariableValue('--text-color').trim() } }, cells: { align: "left", diff --git a/static/site.css b/static/site.css index 9127089..3754316 100644 --- a/static/site.css +++ b/static/site.css @@ -3,24 +3,12 @@ --background-color: #FFFFFF; /* Light background */ --text-color: #333333; /* Dark text for contrast */ --paper-background-color: #F0F0F0; - --plot-background-color: #F0F0F0; + --plot-background-color: #FFFFFF; --accent-color: #6200EE; /* For buttons and interactive elements */ - /* Graph colors */ - --graph-color-1: #007BFF; /* Blue */ - --graph-color-2: #28A745; /* Green */ - --graph-color-3: #DC3545; /* Red */ - --graph-color-4: #FFC107; /* Yellow */ - --graph-color-5: #17A2B8; /* Cyan */ - --graph-color-6: #6C757D; /* Gray */ - /* Axis and grid lines */ --axis-color: #CCCCCC; --grid-line-color: #EEEEEE; - - /* Tooltip background */ - --tooltip-background-color: #FFFFFF; - --tooltip-text-color: #000000; } @media (prefers-color-scheme: dark) { @@ -32,21 +20,9 @@ --text-color: #839496; /* base0 */ --accent-color: #268bd2; /* blue */ - /* Graph colors - Solarized Accent Colors */ - --graph-color-1: #b58900; /* yellow */ - --graph-color-2: #cb4b16; /* orange */ - --graph-color-3: #dc322f; /* red */ - --graph-color-4: #d33682; /* magenta */ - --graph-color-5: #6c71c4; /* violet */ - --graph-color-6: #2aa198; /* cyan */ - /* Axis and grid lines */ --axis-color: #586e75; /* base01 */ --grid-line-color: #073642; /* base02 */ - - /* Tooltip background */ - --tooltip-background-color: #002b36; /* base03 */ - --tooltip-text-color: #839496; /* base0 */ } } From 1b0547d93ca0e2f20493461bd071267462d62aa7 Mon Sep 17 00:00:00 2001 From: Jeremy Wall Date: Tue, 5 Mar 2024 21:13:15 -0500 Subject: [PATCH 15/15] fix: header field was mistakenly plural --- static/lib.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/static/lib.js b/static/lib.js index 15fc471..55c42e0 100644 --- a/static/lib.js +++ b/static/lib.js @@ -48,7 +48,7 @@ * @property {string=} name * @property {string} type * @property {string=} mode - * @property {HeaderOrCell} headers + * @property {HeaderOrCell} header * @property {HeaderOrCell} cells - An Array of columns for the table. * @property {string=} xaxis * @property {string=} yaxis @@ -519,14 +519,15 @@ export class GraphPlot extends HTMLElement { traces.push(trace); } } else if (subplot.Stream) { + // TODO(jwall): It would be nice if scroll behavior would handle replots better. // TODO(jwall): It's possible that this should actually be a separate custom // element. const trace = /** @type TableTrace */({ type: "table", columnwidth: [15, 20, 70], - headers: { + header: { align: "left", - values: ["Timestamp","Label", "Log"], + values: ["Timestamp","Labels", "Log"], fill: { color: layout.xaxis.paper_bgcolor }, font: { color: getCssVariableValue('--text-color').trim() } },