feat: vector sending logs to victoria-logs

A better log query interface

nix/base-system/darwin-configuration.nix

@@ -161,6 +161,65 @@
         }
     ];
 
+    services.victoria-logs.enable = true;
+    services.vector.enable = true;
+    services.vector.settings = {
+        data_dir = "/var/lib/vector";
+        sources = {
+            prometheus = {
+                type = "file";
+                include = [
+                    "${config.launchd.user.agents.prometheus.serviceConfig.StandardOutPath}"
+                    "${config.launchd.user.agents.prometheus.serviceConfig.StandardErrorPath}"
+                ];
+            };
+            heracles = {
+                type = "file";
+                include = [
+                    "${config.launchd.user.agents.heracles.serviceConfig.StandardOutPath}"
+                    "${config.launchd.user.agents.heracles.serviceConfig.StandardErrorPath}"
+                ];
+            };
+            # TODO(zaphar): We should remap durnitisp output to get strip the tty control characters.
+            durnitisp = {
+                type = "file";
+                include = [
+                    "${config.launchd.daemons.durnitisp.serviceConfig.StandardOutPath}"
+                    "${config.launchd.daemons.durnitisp.serviceConfig.StandardErrorPath}"
+                ];
+            };
+            vector = {
+                type = "file";
+                include = [
+                    "${config.launchd.daemons.vector.serviceConfig.StandardOutPath}"
+                    "${config.launchd.daemons.vector.serviceConfig.StandardErrorPath}"
+                ];
+            };
+        };
+        sinks = {
+            victoria = {
+                type = "elasticsearch";
+                mode = "bulk";
+                endpoints = [
+                    "http://${config.services.victoria-logs.listenAddr}/insert/elasticsearch/"
+                ];
+                inputs = [
+                    "prometheus"
+                    "heracles"
+                    "durnitisp"
+                    "vector"
+                ];
+                api_version = "v8";
+                healthcheck.enabled = false;
+                query = {
+                    _msg_field = "message";
+                    _time_field = "timestamp";
+                    _stream_fields = "host,file";
+                };
+            };
+        };
+    };
+
   # TODO launchd.user.agents.prometheus;
   # Use a custom configuration.nix location.
   # $ darwin-rebuild switch -I darwin-config=$HOME/.config/nixpkgs/darwin/configuration.nix

nix/base-system/flake.lock

@@ -527,16 +527,16 @@
     },
     "nixpkgs-darwin": {
       "locked": {
-        "lastModified": 1715818734,
-        "narHash": "sha256-WvAJWCwPj/6quKcsgsvQYyZRxV8ho/yUzj0HZQ34DVU=",
+        "lastModified": 1717100421,
+        "narHash": "sha256-T+0Q1QHBDCoa4yBJrY7cG3vDEhqm4PwOLmNI6mzEwVc=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "95742536dc6debb5a8b8b78b27001c38f369f1e7",
+        "rev": "75000c2cf4422c8a1776284314921ac1289c02c9",
         "type": "github"
       },
       "original": {
         "owner": "nixos",
-        "ref": "nixpkgs-23.11-darwin",
+        "ref": "nixpkgs-24.05-darwin",
         "repo": "nixpkgs",
         "type": "github"
       }
@@ -633,16 +633,16 @@
     },
     "nixpkgs_7": {
       "locked": {
-        "lastModified": 1701282334,
-        "narHash": "sha256-MxCVrXY6v4QmfTwIysjjaX0XUhqBbxTWWB4HXtDYsdk=",
+        "lastModified": 1717179513,
+        "narHash": "sha256-vboIEwIQojofItm2xGCdZCzW96U85l9nDW3ifMuAIdM=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "057f9aecfb71c4437d2b27d3323df7f93c010b7e",
+        "rev": "63dacb46bf939521bdc93981b4cbb7ecb58427a0",
         "type": "github"
       },
       "original": {
         "owner": "nixos",
-        "ref": "23.11",
+        "ref": "24.05",
         "repo": "nixpkgs",
         "type": "github"
       }

nix/base-system/flake.nix

@@ -1,9 +1,9 @@
 {
   inputs = {
     # Default to sane nixpkgs versions
-    nixpkgs.url = "github:nixos/nixpkgs/23.11";
+    nixpkgs.url = "github:nixos/nixpkgs/24.05";
     unstable.url = "nixpkgs";
-    nixpkgs-darwin.url = "github:nixos/nixpkgs/nixpkgs-23.11-darwin";
+    nixpkgs-darwin.url = "github:nixos/nixpkgs/nixpkgs-24.05-darwin";
     #lean4-flake = { 
     #  url = "github:leanprover/lean4/v4.4.0";
     #  inputs.nixpkgs.follows = "nixpkgs";
@@ -156,7 +156,6 @@ EOF";
                 cmp-nvim-lsp-signature-help
                 nvim-dap # Debug Adapter Protocol support
                 #nvim-dap-ui
-                #neogit
                 hoon-vim
                 nvim-treesitter
                 lean-nvim
@@ -214,8 +213,8 @@ EOF";
             nil-pkg
 			lua-language-server
             rust-analyzer
-            omnisharp-roslyn
-            dbeaver
+            # TODO(jwall): This is no longer provided for darwin in nixpkgs
+            #dbeaver-bin
         ]);
       };
     };
@@ -274,12 +273,10 @@ EOF";
             # TODO(jwall): This appears to be broken due to: https://github.com/NixOS/nixpkgs/issues/166205
             # Should be fixed by: https://github.com/NixOS/nixpkgs/pull/282624
             #isabelle
-            # TODO(jwall): This appears to be broken due to: https://github.com/NixOS/nixpkgs/issues/166205
             # Should be fixed by: https://github.com/NixOS/nixpkgs/pull/282624
-            (lean4.override { stdenv = stdenv.override { allowedRequisites = null; cc = llvmPackages_11.clang; }; })
+            #(lean4.override { stdenv = stdenv.override { allowedRequisites = null; cc = llvmPackages_11.clang; }; })
+            lean4
             terraform # TODO(jeremy): Replace with opentofu when that is an option.
-            consul
-            consul-template
             nomad
             oha
             nodejs
@@ -314,20 +311,16 @@ EOF";
             alloy6
             tlaplus
             jdk
-            d2-lang
+            # TODO(zaphar): d2 is broken in latest nixpkgs
+            #d2-lang
             plantuml-c4
             nssTools
             nomad
             mkcert
             kubo
-            #dotnet-sdk_7
-            #dotnet-sdk_8
             #netcoredbg
             powershell
-            unstablePkgs.ocaml
-            unstablePkgs.opam
-            unstablePkgs.ocamlformat
-            # TODO(jwall): I include these initially for nvim telescope
+            # NOTE(jwall): I include these initially for nvim telescope
             ripgrep
             fd
             # TODO add sonic-pi here if it supports the arch
@@ -355,6 +348,8 @@ EOF";
         (systemModule system)
         (vimModule system)
         ./modules/darwin-monitor.nix
+        ./modules/victoria-logs.nix
+        ./modules/vector.nix
         ./darwin-configuration.nix
       ];
     };

nix/base-system/modules/vector.nix

@@ -0,0 +1,34 @@
+{pkgs, lib, config, ...}:
+with lib;
+{
+    options.services.vector = {
+        enable = mkEnableOption "Enable the vector agent";
+        settings = mkOption {
+            description = "Settings for the vector agent";
+            default = {
+                data_dir = "/var/lib/vector";
+            };
+            defaultText = "{}";
+        };
+    };
+
+    config = {
+        environment.etc."vector.yaml" = mkIf config.services.vector.enable {
+            text = (generators.toYAML {} config.services.vector.settings);
+        };
+        
+        launchd.daemons.vector = mkIf config.services.vector.enable {
+            serviceConfig = {
+                ProgramArguments = [
+                    "${pkgs.vector}/bin/vector"
+                    "--watch-config"
+                    "--config=/etc/${config.environment.etc."vector.yaml".target}"
+                ];
+                KeepAlive = true;
+                RunAtLoad = true;
+                StandardOutPath = "/var/log/vector.out.log";
+                StandardErrorPath = "/var/log/vector.err.log";
+            };
+        };
+    };
+}

nix/base-system/modules/victoria-logs.nix

@@ -0,0 +1,33 @@
+{pkgs, lib, config, ...}:
+with lib;
+{
+    options.services = {
+        victoria-logs = {
+           enable = mkEnableOption "Enable the VictoriaLogs service";
+           dataPath = mkOption {
+            description = "Logging directory path for victoria-logs service";
+            default = "/Users/Zaphar/opt/victoria-logs";
+           };
+           listenAddr = mkOption {
+            description = "Socket Address to listen on";
+            default = "127.0.0.1:9428";
+           };
+        };
+    };
+
+    config = {
+        launchd.user.agents.victoria-logs = mkIf config.services.victoria-logs.enable {
+            serviceConfig = {
+                ProgramArguments = [
+                    "${pkgs.victoriametrics}/bin/victoria-logs"
+                    "-storageDataPath=${config.services.victoria-logs.dataPath}/data"
+                    "-httpListenAddr=${config.services.victoria-logs.listenAddr}"
+                ];
+                KeepAlive = true;
+                RunAtLoad = true;
+                StandardOutPath = "${config.services.victoria-logs.dataPath}/victoria-logs.out.log";
+                StandardErrorPath = "${config.services.victoria-logs.dataPath}/victoria-logs.err.log";
+            };
+        };
+    };
+}