Use the metrics facade

This commit is contained in:
Jeremy Wall 2023-03-14 11:53:51 -04:00
parent f609dcfef3
commit 5341153a86
5 changed files with 536 additions and 168 deletions

464
Cargo.lock generated
View File

@ -2,6 +2,17 @@
# It is not intended for manual editing.
version = 3
[[package]]
name = "ahash"
version = "0.7.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47"
dependencies = [
"getrandom",
"once_cell",
"version_check",
]
[[package]]
name = "ansi_term"
version = "0.12.1"
@ -41,12 +52,24 @@ version = "1.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
[[package]]
name = "bumpalo"
version = "3.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535"
[[package]]
name = "byteorder"
version = "1.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ae44d1a3d5a19df61dd0c8beb138458ac2a53a7ac09eba97d55592540004306b"
[[package]]
name = "bytes"
version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be"
[[package]]
name = "cfg-if"
version = "1.0.0"
@ -59,6 +82,28 @@ version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7477065d45a8fe57167bf3cf8bcd3729b54cfcb81cca49bda2d038ea89ae82ca"
[[package]]
name = "crossbeam-epoch"
version = "0.9.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "46bd5f3f85273295a9d14aedfb86f6aadbff6d8f5295c4a9edb08e819dcf5695"
dependencies = [
"autocfg",
"cfg-if",
"crossbeam-utils",
"memoffset",
"scopeguard",
]
[[package]]
name = "crossbeam-utils"
version = "0.8.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3c063cd8cc95f5c377ed0d4b49a4b21f632396ff690e8470c29b3359b346984b"
dependencies = [
"cfg-if",
]
[[package]]
name = "ctor"
version = "0.1.18"
@ -71,13 +116,14 @@ dependencies = [
[[package]]
name = "durnitisp"
version = "0.2.2"
version = "0.2.3"
dependencies = [
"anyhow",
"gflags",
"icmp-socket",
"metrics",
"metrics-exporter-prometheus",
"nursery",
"prometheus",
"resolve",
"socket2 0.5.1",
"tiny_http",
@ -97,6 +143,50 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba"
[[package]]
name = "futures-channel"
version = "0.3.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "164713a5a0dcc3e7b4b1ed7d3b433cabc18025386f9339346e8daf15963cf7ac"
dependencies = [
"futures-core",
]
[[package]]
name = "futures-core"
version = "0.3.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "86d7a0c1aa76363dac491de0ee99faf6941128376f1cf96f07db7603b7de69dd"
[[package]]
name = "futures-task"
version = "0.3.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fd65540d33b37b16542a0438c12e6aeead10d4ac5d05bd3f805b8f35ab592879"
[[package]]
name = "futures-util"
version = "0.3.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3ef6b17e481503ec85211fed8f39d1970f128935ca1f814cd32ac4a6842e84ab"
dependencies = [
"futures-core",
"futures-task",
"pin-project-lite",
"pin-utils",
]
[[package]]
name = "getrandom"
version = "0.2.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31"
dependencies = [
"cfg-if",
"libc",
"wasi 0.11.0+wasi-snapshot-preview1",
]
[[package]]
name = "gflags"
version = "0.3.7"
@ -131,12 +221,81 @@ dependencies = [
"syn",
]
[[package]]
name = "hashbrown"
version = "0.12.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
dependencies = [
"ahash",
]
[[package]]
name = "hermit-abi"
version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7"
dependencies = [
"libc",
]
[[package]]
name = "http"
version = "0.2.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482"
dependencies = [
"bytes",
"fnv",
"itoa",
]
[[package]]
name = "http-body"
version = "0.4.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1"
dependencies = [
"bytes",
"http",
"pin-project-lite",
]
[[package]]
name = "httparse"
version = "1.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904"
[[package]]
name = "httpdate"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421"
[[package]]
name = "hyper"
version = "0.14.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "abfba89e19b959ca163c7752ba59d737c1ceea53a5d31a149c805446fc958064"
dependencies = [
"bytes",
"futures-channel",
"futures-core",
"futures-util",
"http",
"http-body",
"httparse",
"httpdate",
"itoa",
"pin-project-lite",
"socket2 0.4.4",
"tokio",
"tower-service",
"tracing",
"want",
]
[[package]]
name = "icmp-socket"
version = "0.2.0"
@ -158,6 +317,16 @@ dependencies = [
"unicode-normalization",
]
[[package]]
name = "indexmap"
version = "1.9.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1885e79c1fc4b10f0e172c475f458b7f7b93061064d98c3293e98c5ba0c8b399"
dependencies = [
"autocfg",
"hashbrown",
]
[[package]]
name = "inventory"
version = "0.1.10"
@ -180,6 +349,27 @@ dependencies = [
"syn",
]
[[package]]
name = "ipnet"
version = "2.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "30e22bd8629359895450b59ea7a776c850561b96a3b1d31321c1949d9e6c9146"
[[package]]
name = "itoa"
version = "1.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6"
[[package]]
name = "js-sys"
version = "0.3.49"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dc15e39392125075f60c95ba416f5381ff6c3a948ff02ab12464715adf56c821"
dependencies = [
"wasm-bindgen",
]
[[package]]
name = "lazy_static"
version = "1.4.0"
@ -220,6 +410,15 @@ dependencies = [
"cfg-if",
]
[[package]]
name = "mach"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b823e83b2affd8f40a9ee8c29dbc56404c1e34cd2710921f2801e2cf29527afa"
dependencies = [
"libc",
]
[[package]]
name = "matches"
version = "0.1.8"
@ -227,10 +426,93 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08"
[[package]]
name = "memchr"
version = "2.3.4"
name = "memoffset"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525"
checksum = "d61c719bcfbcf5d62b3a09efa6088de8c54bc0bfcd3ea7ae39fcc186108b8de1"
dependencies = [
"autocfg",
]
[[package]]
name = "metrics"
version = "0.20.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7b9b8653cec6897f73b519a43fba5ee3d50f62fe9af80b428accdcc093b4a849"
dependencies = [
"ahash",
"metrics-macros",
"portable-atomic",
]
[[package]]
name = "metrics-exporter-prometheus"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8603921e1f54ef386189335f288441af761e0fc61bcb552168d9cedfe63ebc70"
dependencies = [
"hyper",
"indexmap",
"ipnet",
"metrics",
"metrics-util",
"parking_lot",
"portable-atomic",
"quanta",
"thiserror",
"tokio",
"tracing",
]
[[package]]
name = "metrics-macros"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "731f8ecebd9f3a4aa847dfe75455e4757a45da40a7793d2f0b1f9b6ed18b23f3"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "metrics-util"
version = "0.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f7d24dc2dbae22bff6f1f9326ffce828c9f07ef9cc1e8002e5279f845432a30a"
dependencies = [
"crossbeam-epoch",
"crossbeam-utils",
"hashbrown",
"metrics",
"num_cpus",
"parking_lot",
"portable-atomic",
"quanta",
"sketches-ddsketch",
]
[[package]]
name = "mio"
version = "0.8.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5b9d9a46eff5b4ff64b45a9e316a6d1e0bc719ef429cbec4dc630684212bfdf9"
dependencies = [
"libc",
"log 0.4.14",
"wasi 0.11.0+wasi-snapshot-preview1",
"windows-sys 0.45.0",
]
[[package]]
name = "num_cpus"
version = "1.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b"
dependencies = [
"hermit-abi",
"libc",
]
[[package]]
name = "nursery"
@ -273,6 +555,18 @@ version = "0.2.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116"
[[package]]
name = "pin-utils"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
[[package]]
name = "portable-atomic"
version = "0.3.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "26f6a7b87c2e435a3241addceeeff740ff8b7e76b74c13bf9acb17fa454ea00b"
[[package]]
name = "proc-macro2"
version = "1.0.24"
@ -283,26 +577,21 @@ dependencies = [
]
[[package]]
name = "prometheus"
version = "0.13.1"
name = "quanta"
version = "0.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cface98dfa6d645ea4c789839f176e4b072265d085bfcc48eaa8d137f58d3c39"
checksum = "b7e31331286705f455e56cca62e0e717158474ff02b7936c1fa596d983f4ae27"
dependencies = [
"cfg-if",
"fnv",
"lazy_static",
"memchr",
"parking_lot",
"protobuf",
"thiserror",
"crossbeam-utils",
"libc",
"mach",
"once_cell",
"raw-cpuid",
"wasi 0.10.2+wasi-snapshot-preview1",
"web-sys",
"winapi",
]
[[package]]
name = "protobuf"
version = "2.20.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "86473d5f16580f10b131a0bf0afb68f8e029d1835d33a00f37281b05694e5312"
[[package]]
name = "quote"
version = "1.0.8"
@ -350,6 +639,15 @@ version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc"
[[package]]
name = "raw-cpuid"
version = "10.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6c297679cb867470fa8c9f67dbba74a78d78e3e98d7cf2b08d6d71540f797332"
dependencies = [
"bitflags",
]
[[package]]
name = "rdrand"
version = "0.4.0"
@ -415,6 +713,12 @@ dependencies = [
"lazy_static",
]
[[package]]
name = "sketches-ddsketch"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ceb945e54128e09c43d8e4f1277851bd5044c6fc540bbaa2ad888f60b3da9ae7"
[[package]]
name = "smallvec"
version = "1.6.1"
@ -508,6 +812,26 @@ version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c"
[[package]]
name = "tokio"
version = "1.26.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "03201d01c3c27a29c8a5cee5b55a93ddae1ccf6f08f65365c2c918f8c1b76f64"
dependencies = [
"autocfg",
"libc",
"mio",
"pin-project-lite",
"socket2 0.4.4",
"windows-sys 0.45.0",
]
[[package]]
name = "tower-service"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52"
[[package]]
name = "tracing"
version = "0.1.35"
@ -566,6 +890,12 @@ dependencies = [
"tracing-log",
]
[[package]]
name = "try-lock"
version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed"
[[package]]
name = "unicode-bidi"
version = "0.3.4"
@ -596,6 +926,98 @@ version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d"
[[package]]
name = "version_check"
version = "0.9.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
[[package]]
name = "want"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0"
dependencies = [
"log 0.4.14",
"try-lock",
]
[[package]]
name = "wasi"
version = "0.10.2+wasi-snapshot-preview1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6"
[[package]]
name = "wasi"
version = "0.11.0+wasi-snapshot-preview1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
[[package]]
name = "wasm-bindgen"
version = "0.2.72"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8fe8f61dba8e5d645a4d8132dc7a0a66861ed5e1045d2c0ed940fab33bac0fbe"
dependencies = [
"cfg-if",
"wasm-bindgen-macro",
]
[[package]]
name = "wasm-bindgen-backend"
version = "0.2.72"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "046ceba58ff062da072c7cb4ba5b22a37f00a302483f7e2a6cdc18fedbdc1fd3"
dependencies = [
"bumpalo",
"lazy_static",
"log 0.4.14",
"proc-macro2",
"quote",
"syn",
"wasm-bindgen-shared",
]
[[package]]
name = "wasm-bindgen-macro"
version = "0.2.72"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ef9aa01d36cda046f797c57959ff5f3c615c9cc63997a8d545831ec7976819b"
dependencies = [
"quote",
"wasm-bindgen-macro-support",
]
[[package]]
name = "wasm-bindgen-macro-support"
version = "0.2.72"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "96eb45c1b2ee33545a813a92dbb53856418bf7eb54ab34f7f7ff1448a5b3735d"
dependencies = [
"proc-macro2",
"quote",
"syn",
"wasm-bindgen-backend",
"wasm-bindgen-shared",
]
[[package]]
name = "wasm-bindgen-shared"
version = "0.2.72"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b7148f4696fb4960a346eaa60bbfb42a1ac4ebba21f750f75fc1375b098d5ffa"
[[package]]
name = "web-sys"
version = "0.3.49"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "59fe19d70f5dacc03f6e46777213facae5ac3801575d56ca6cbd4c93dcd12310"
dependencies = [
"js-sys",
"wasm-bindgen",
]
[[package]]
name = "winapi"
version = "0.3.9"

View File

@ -1,6 +1,6 @@
[package]
name = "durnitisp"
version = "0.2.2"
version = "0.2.3"
authors = ["Jeremy Wall <jeremy@marzhillstudios.com>"]
edition = "2018"
@ -12,7 +12,8 @@ tracing-subscriber = "0.3.14"
anyhow = "1"
gflags = "^0.3"
nursery = "^0.0.1"
prometheus = "0.13.1"
metrics = "0.20.1"
metrics-exporter-prometheus = "0.11.0"
tiny_http = "0.12.0"
socket2 = "0.5.1"
icmp-socket = "0.2.0"

View File

@ -25,8 +25,8 @@ use icmp_socket::{
packet::{Icmpv4Message, Icmpv6Message, WithEchoRequest},
IcmpSocket, IcmpSocket4, IcmpSocket6, Icmpv4Packet, Icmpv6Packet,
};
use metrics::{gauge, histogram, increment_counter, Label};
use nursery::{thread, Nursery};
use prometheus::{CounterVec, GaugeVec};
use tracing::{debug, error, info, instrument, warn};
gflags::define! {
@ -62,11 +62,16 @@ fn resolve_host_address(host: &str) -> String {
struct State<AddrType> {
destinations: HashMap<u16, (String, AddrType)>, // domain, address
// TODO(jwall): This should be a time tracker by both identifier and sequence
time_tracker: BTreeMap<u16, BTreeMap<u16, Instant>>,
destination_counter: BTreeMap<u16, u16>,
latency_guage: GaugeVec,
ping_counter: CounterVec,
// TODO(jwall): Add histogram for latency as well.
}
fn make_ping_count_labels(domain_name: &str, result: &str) -> Vec<Label> {
vec![
Label::new("domain", domain_name.to_owned()),
Label::new("result", result.to_owned()),
]
}
impl<AddrType: std::fmt::Display> State<AddrType> {
@ -84,13 +89,18 @@ impl<AddrType: std::fmt::Display> State<AddrType> {
seq = sequence,
"Reply",
);
self.ping_counter
.with(&prometheus::labels! {"result" => "ok", "domain" => domain_name})
.inc();
increment_counter!("ping_counter", make_ping_count_labels(domain_name, "ok"),);
if elapsed as i32 != 0 {
self.latency_guage
.with(&prometheus::labels! {"domain" => domain_name.as_str()})
.set(elapsed);
gauge!(
"ping_latency",
elapsed,
vec![Label::new("domain", domain_name.to_owned()),],
);
histogram!(
"ping_latency_hist_ms",
elapsed,
vec![Label::new("domain", domain_name.to_owned()),],
);
}
self.time_tracker
.get_mut(&identifier)
@ -114,9 +124,10 @@ impl<AddrType: std::fmt::Display> State<AddrType> {
seq = sequence,
"Dropped"
);
self.ping_counter
.with(&prometheus::labels! {"result" => "dropped", "domain" => domain_name})
.inc();
increment_counter!(
"ping_counter",
make_ping_count_labels(domain_name, "dropped"),
);
for_delete.push(*k);
}
}
@ -175,9 +186,10 @@ impl<'a> PacketHandler<Icmpv6Packet, Ipv6Addr> for &'a mut State<Ipv6Addr> {
},
}) => {
if let Some((domain_name, _addr)) = self.destinations.get(&identifier) {
self.ping_counter
.with(&prometheus::labels! {"result" => "unreachable", "domain" => domain_name})
.inc();
increment_counter!(
"ping_counter",
make_ping_count_labels(domain_name, "unreachable")
);
return true;
}
}
@ -201,17 +213,12 @@ impl<'a> PacketHandler<Icmpv6Packet, Ipv6Addr> for &'a mut State<Ipv6Addr> {
checksum: _,
message:
Icmpv6Message::EchoRequest {
identifier,
identifier: _,
sequence: _,
payload: _,
},
}) => {
if let Some((domain_name, _addr)) = self.destinations.get(&identifier) {
self.ping_counter
.with(&prometheus::labels! {"result" => "parameter_problem", "domain" => domain_name})
.inc();
return true;
}
// TODO log but otherwise ignore this.
}
Err(e) => {
// We ignore these as well but log it.
@ -292,10 +299,7 @@ where
);
match self.send_to_destination(dest, identifier, sequence) {
Err(e) => {
state
.ping_counter
.with(&prometheus::labels! {"result" => "err", "type" => "send"})
.inc();
increment_counter!("ping_counter", make_ping_count_labels(domain_name, "err"),);
error!(
domain=domain_name, %dest, err=?e,
"Error sending. Trying again later",
@ -388,11 +392,7 @@ where
}
Err(e) => {
error!(err = ?e, "Error receiving packet");
handler
.get_mut_state()
.ping_counter
.with(&prometheus::labels! {"result" => "err", "domain" => "unknown"})
.inc();
increment_counter!("ping_counter", make_ping_count_labels("unknown", "err"),);
return;
}
}
@ -428,12 +428,7 @@ impl Multi {
}
#[instrument(name = "ICMP", skip_all)]
pub fn schedule_echo_server(
domain_names: &Vec<&str>,
ping_latency_guage: GaugeVec,
ping_counter: CounterVec,
parent: &mut Nursery,
) {
pub fn schedule_echo_server(domain_names: &Vec<&str>, parent: &mut Nursery) {
let resolved: Vec<(String, IpAddr)> = domain_names
.iter()
.map(|domain_name| {
@ -472,8 +467,6 @@ pub fn schedule_echo_server(
destinations: v4_destinations,
time_tracker: BTreeMap::new(),
destination_counter: BTreeMap::new(),
latency_guage: ping_latency_guage.clone(),
ping_counter: ping_counter.clone(),
};
let mut v6_destinations = HashMap::new();
let mut v6_id_counter = 42;
@ -493,8 +486,6 @@ pub fn schedule_echo_server(
destinations: v6_destinations,
time_tracker: BTreeMap::new(),
destination_counter: BTreeMap::new(),
latency_guage: ping_latency_guage,
ping_counter,
};
let v6_pinger = PingerImpl {
sock: IcmpSocket6::new().expect("Failed to open Icmpv6 Socket"),

View File

@ -15,10 +15,9 @@ use std::convert::Into;
use std::sync::Arc;
use gflags;
use metrics_exporter_prometheus;
use nursery::thread;
use nursery::{Nursery, Waitable};
use prometheus::{self, GaugeVec};
use prometheus::{CounterVec, Encoder, IntGaugeVec, Opts, Registry, TextEncoder};
use tiny_http;
use tracing::{debug, error, info, instrument, Level};
use tracing_subscriber::FmtSubscriber;
@ -93,49 +92,11 @@ fn main() -> anyhow::Result<()> {
let ping_hosts: Vec<&str> = PINGHOSTS.flag.split(",").collect();
let builder = metrics_exporter_prometheus::PrometheusBuilder::new();
let prom_handle = builder
.install_recorder()
.expect("Failed to install prometheus exporter");
// Create a Registry and register metrics.
let r = Registry::new();
let stun_counter_vec = CounterVec::new(
Opts::new(
"stun_attempt_counter",
"Counter for the good, bad, and total attempts to connect to stun server.",
),
&["result", "domain"],
)
.unwrap();
let stun_success_vec = IntGaugeVec::new(
Opts::new("stun_success", "Stun probe successes"),
&["domain"],
)
.unwrap();
let stun_latency_vec = IntGaugeVec::new(
Opts::new(
"stun_attempt_latency_ms",
"Latency guage in millis per stun domain.",
),
&["domain"],
)
.unwrap();
let ping_latency_vec = GaugeVec::new(
Opts::new("ping_latency", "ICMP Ping latency in milliseconds"),
&["domain"],
)
.unwrap();
let ping_counter_vec = CounterVec::new(
Opts::new("ping_counter", "Ping Request Counter"),
&["result", "domain"],
)
.unwrap();
r.register(Box::new(stun_counter_vec.clone()))
.expect("Failed to register stun connection counter");
r.register(Box::new(stun_latency_vec.clone()))
.expect("Failed to register stun latency guage");
r.register(Box::new(stun_success_vec.clone()))
.expect("Failed to register stun success gauge");
r.register(Box::new(ping_latency_vec.clone()))
.expect("Failed to register ping latency guage");
r.register(Box::new(ping_counter_vec.clone()))
.expect("Failed to register ping counter");
let stun_socket_addrs = util::resolve_socket_addrs(&stun_servers).unwrap();
let stun_servers = Arc::new(stun_servers);
let ping_hosts = Arc::new(ping_hosts);
@ -165,13 +126,8 @@ fn main() -> anyhow::Result<()> {
info!("Waiting for request");
match server.recv() {
Ok(req) => {
let mut buffer = vec![];
// Gather the metrics.
let encoder = TextEncoder::new();
let metric_families = r.gather();
encoder.encode(&metric_families, &mut buffer).unwrap();
let response = tiny_http::Response::from_data(buffer).with_status_code(200);
let response = tiny_http::Response::from_data(prom_handle.render())
.with_status_code(200);
if let Err(e) = req.respond(response) {
error!(err = ?e, "Error responding to request");
}
@ -185,27 +141,15 @@ fn main() -> anyhow::Result<()> {
parent.adopt(Box::new(render_thread));
}
{
let ping_latency_vec = ping_latency_vec.clone();
let ping_counter_vec = ping_counter_vec.clone();
icmp::schedule_echo_server(&ping_hosts, ping_latency_vec, ping_counter_vec, &mut parent);
icmp::schedule_echo_server(&ping_hosts, &mut parent);
}
// Then we attempt to start connections to each stun server.
for (i, s) in stun_socket_addrs.iter().enumerate() {
let stun_servers_copy = stun_servers.clone();
let stun_counter_vec_copy = stun_counter_vec.clone();
let stun_latency_vec_copy = stun_latency_vec.clone();
let stun_success_vec_copy = stun_success_vec.clone();
if let Some(s) = s.clone() {
let domain_name = *stun_servers_copy.get(i).unwrap();
let connect_thread = thread::Pending::new(move || {
stun::start_listen_thread(
domain_name,
s.into(),
stun_counter_vec_copy,
stun_latency_vec_copy,
stun_success_vec_copy,
)
});
let connect_thread =
thread::Pending::new(move || stun::start_listen_thread(domain_name, s.into()));
parent.schedule(Box::new(connect_thread));
// Spread the probe threads out so they're somewhat uniformly distributed.
std::thread::sleep(std::time::Duration::from_micros(

View File

@ -13,7 +13,7 @@
// limitations under the License.
use gflags;
use prometheus::{CounterVec, IntGaugeVec};
use metrics::{gauge, increment_counter, Label};
use std::convert::From;
use std::io;
use std::net::{SocketAddr, UdpSocket};
@ -70,18 +70,18 @@ fn attempt_stun_connect(addr: SocketAddr) -> Result<SystemTime, ConnectError> {
Ok(SystemTime::now())
}
fn make_count_labels(domain_name: &str, result: &str) -> Vec<Label> {
vec![
Label::new("domain", domain_name.to_owned()),
Label::new("result", result.to_owned()),
]
}
#[instrument(
name = "STUN",
fields(domain=domain_name, socket=%s),
skip(stun_counter_vec_copy, stun_latency_vec_copy, stun_success_vec_copy)
)]
pub fn start_listen_thread(
domain_name: &str,
s: SocketAddr,
stun_counter_vec_copy: CounterVec,
stun_latency_vec_copy: IntGaugeVec,
stun_success_vec_copy: IntGaugeVec,
) {
pub fn start_listen_thread(domain_name: &str, s: SocketAddr) {
debug!("starting thread");
loop {
let now = SystemTime::now();
@ -94,16 +94,17 @@ pub fn start_listen_thread(
millis = finish_time.duration_since(now).unwrap().as_millis(),
conn_type = "Stun connection",
);
stun_counter_vec_copy
.with(&prometheus::labels! {"result" => "ok", "domain" => domain_name})
.inc();
stun_latency_vec_copy
.with(&prometheus::labels! {"domain" => domain_name})
// Technically this could be lossy but we'll chance it anyway.
.set(finish_time.duration_since(now).unwrap().as_millis() as i64);
stun_success_vec_copy
.with(&prometheus::labels! {"domain" => domain_name})
.set(1);
increment_counter!("stun_attempt_counter", make_count_labels(domain_name, "ok"));
gauge!(
"stun_attempt_latency_ms",
finish_time.duration_since(now).unwrap().as_millis() as f64,
vec![Label::new("domain", domain_name.to_owned())]
);
gauge!(
"stun_success",
1 as f64,
vec![Label::new("domain", domain_name.to_owned())]
);
}
Err(ConnectError::Timeout(finish_time)) => {
info!(
@ -112,24 +113,30 @@ pub fn start_listen_thread(
millis = finish_time.duration_since(now).unwrap().as_millis(),
conn_type = "Stun connection",
);
stun_counter_vec_copy
.with(&prometheus::labels! {"result" => "timeout", "domain" => domain_name})
.inc();
stun_success_vec_copy
.with(&prometheus::labels! {"domain" => domain_name})
.set(0);
increment_counter!(
"stun_attempt_counter",
make_count_labels(domain_name, "timeout")
);
gauge!(
"stun_success",
0 as f64,
vec![Label::new("domain", domain_name.to_owned())]
);
}
Err(ConnectError::Err(e)) => {
error!(
timeout=true, success=false, err = ?e,
conn_type="Stun connection",
);
stun_counter_vec_copy
.with(&prometheus::labels! {"result" => "err", "domain" => domain_name})
.inc();
stun_success_vec_copy
.with(&prometheus::labels! {"domain" => domain_name})
.set(0);
increment_counter!(
"stun_attempt_counter",
make_count_labels(domain_name, "err")
);
gauge!(
"stun_success",
0 as f64,
vec![Label::new("domain", domain_name.to_owned())]
);
}
Err(ConnectError::Incomplete) => {
error!(
@ -138,12 +145,15 @@ pub fn start_listen_thread(
err = "Incomplete",
conn_type = "Stun connection",
);
stun_counter_vec_copy
.with(&prometheus::labels! {"result" => "incomplete", "domain" => domain_name})
.inc();
stun_success_vec_copy
.with(&prometheus::labels! {"domain" => domain_name})
.set(0);
increment_counter!(
"stun_attempt_counter",
make_count_labels(domain_name, "incomplete")
);
gauge!(
"stun_success",
0 as f64,
vec![Label::new("domain", domain_name.to_owned())]
);
}
}