Merge branch 'monitoring'

nixos/configuration.nix

@@ -5,15 +5,17 @@
     ./networking.nix
     ./users.nix
     ./time.nix
+
+    ./databases/postgres.nix
+    ./databases/mysql.nix
+    ./databases/redis.nix
+
     ./programs/nginx.nix
     ./programs/bash.nix
     ./programs/acme.nix
-    ./programs/postgres.nix
     ./programs/mastodon.nix
-    ./programs/redis.nix
     ./programs/nextcloud.nix
     ./programs/jellyfin.nix
-    ./programs/mysql.nix
     ./programs/photoprism.nix
     ./programs/synapse.nix
     ./programs/fail2ban.nix
@@ -26,6 +28,9 @@
     ./programs/deluge.nix
     ./programs/prosody.nix
     ./programs/yggdrasil.nix
+
+    ./monitoring/grafana.nix
+    ./monitoring/prometheus.nix
   ];
 
   nix = {

nixos/databases/postgres.nix

@@ -3,75 +3,70 @@
   config.services.postgresql = {
     enable = true;
     package = pkgs.postgresql_15;
-    ensureDatabases = [ "mastodon" "matrix-synapse" "nextcloud" "maddy" "plausible" "microboard" "freshrss" "prosody" ];
+    ensureDatabases = [
+      "mastodon"
+      "matrix-synapse"
+      "nextcloud"
+      "maddy"
+      "plausible"
+      "microboard"
+      "freshrss"
+      "prosody"
+      "grafana"
+      "postgres-exporter"
+    ];
     ensureUsers = [
       {
         name = "mastodon";
-        ensurePermissions = {
+        ensureDBOwnership = true;
-          "DATABASE mastodon" = "ALL PRIVILEGES";
-        };
         ensureClauses.login = true;
       }
       {
         name = "nextcloud";
-        ensurePermissions = {
+        ensureDBOwnership = true;
-          "DATABASE nextcloud" = "ALL PRIVILEGES";
-        };
         ensureClauses.login = true;
       }
       {
         name = "matrix-synapse";
-        ensurePermissions = {
+        ensureDBOwnership = true;
-          "DATABASE \"matrix-synapse\"" = "ALL PRIVILEGES";
-        };
         ensureClauses.login = true;
       }
       {
         name = "maddy";
-        ensurePermissions = {
+        ensureDBOwnership = true;
-          "DATABASE maddy" = "ALL PRIVILEGES";
-        };
         ensureClauses.login = true;
       }
       {
         name = "plausible";
-        ensurePermissions = {
+        ensureDBOwnership = true;
-          "DATABASE plausible" = "ALL PRIVILEGES";
-        };
         ensureClauses.login = true;
       }
       {
         name = "microboard";
-        ensurePermissions = {
+        ensureDBOwnership = true;
-          "DATABASE microboard" = "ALL PRIVILEGES";
-        };
         ensureClauses.login = true;
       }
       {
         name = "freshrss";
-        ensurePermissions = {
+        ensureDBOwnership = true;
-          "DATABASE freshrss" = "ALL PRIVILEGES";
-        };
         ensureClauses.login = true;
       }
       {
         name = "prosody";
-        ensurePermissions = {
+        ensureDBOwnership = true;
-          "DATABASE prosody" = "ALL PRIVILEGES";
+        ensureClauses.login = true;
-        };
+      }
+      {
+        name = "grafana";
+        ensureDBOwnership = true;
+        ensureClauses.login = true;
+      }
+      {
+        name = "postgres-exporter";
+        ensureDBOwnership = true;
         ensureClauses.login = true;
       }
     ];
-    initialScript = pkgs.writeText "pg-init.sql" ''
-      ALTER DATABASE nextcloud OWNER TO nextcloud;
-      ALTER DATABASE mastodon OWNER TO mastodon;
-      ALTER DATABASE "matrix-synapse" OWNER TO "matrix-synapse";
-      ALTER DATABASE maddy OWNER TO maddy;
-      ALTER DATABASE plausible OWNER TO plausible;
-      ALTER DATABASE microboard OWNER TO microboard;
-      ALTER DATABASE freshrss OWNER TO freshrss;
-      ALTER DATABASE prosody OWNER TO prosody;
-    '';
     identMap = ''
        # ArbitraryMapName systemUser DBUser
        superuser_map      root       postgres

nixos/monitoring/grafana.nix

@@ -0,0 +1,27 @@
+{ config, ... }: {
+  services.grafana = {
+    enable = true;
+    settings = {
+      server = {
+        http_addr = "127.0.0.1";
+        http_port = 55010;
+        domain = "grafana.comfycamp.space";
+      };
+      database = {
+        user = "grafana";
+        type = "postgres";
+        name = "grafana";
+        host = "/var/run/postgresql";
+      };
+    };
+  };
+
+  services.nginx.virtualHosts.${config.services.grafana.settings.server.domain} = {
+    useACMEHost = "comfycamp.space";
+    forceSSL = true;
+    locations."/" = {
+      proxyPass = "http://127.0.0.1:${toString config.services.grafana.settings.server.http_port}";
+      proxyWebsockets = true;
+    };
+  };
+}

nixos/monitoring/prometheus.nix

@@ -0,0 +1,41 @@
+{ config, ... }: {
+  services.prometheus = {
+    enable = true;
+    port = 55011;
+
+    exporters = {
+      node = {
+        enable = true;
+        enabledCollectors = [ "systemd" ];
+        port = 55012;
+      };
+      postgres = {
+        enable = true;
+        port = 55014;
+        dataSourceName = "user=postgres-exporter database=postgres-exporter host=/run/postgresql sslmode=disable";
+      };
+    };
+    
+    scrapeConfigs = [
+      {
+        job_name = "node";
+        static_configs = [{
+          targets = [ "127.0.0.1:${toString config.services.prometheus.exporters.node.port}" ];
+        }];
+      }
+      {
+        job_name = "synapse";
+        metrics_path = "/_synapse/metrics";
+        static_configs = [{
+          targets = [ "127.0.0.1:55013" ];
+        }];
+      }
+      {
+        job_name = "postgres";
+        static_configs = [{
+          targets = [ "127.0.0.1:${toString config.services.prometheus.exporters.postgres.port}" ];
+        }];
+      }
+    ];
+  };
+}

nixos/programs/comfycamp.nix

@@ -2,7 +2,23 @@
 {
   virtualisation.oci-containers.containers.comfycamp = {
     autoStart = true;
-    image = "ghcr.io/ordinary-dev/comfycamp:v0.6.0";
+    image = "ghcr.io/ordinary-dev/comfycamp:v0.8.0";
     ports = ["55007:80"];
   };
+
+  services.nginx.virtualHosts."[201:80ed:6eeb:aea4:cdc0:c836:2831:f2dd]" = {
+    locations."/".proxyPass = "http://127.0.0.1:55007";
+  };
+
+  services.nginx.virtualHosts."comfycamp.space" = {
+    useACMEHost = "comfycamp.space";
+    forceSSL = true;
+    locations."/".proxyPass = "http://127.0.0.1:55007";
+  };
+
+  services.nginx.virtualHosts."www.comfycamp.space" = {
+    useACMEHost = "comfycamp.space";
+    forceSSL = true;
+    locations."/".return = "301 https://comfycamp.space$request_uri";
+  };
 }

nixos/programs/freshrss.nix

@@ -13,4 +13,9 @@
     passwordFile = "/var/lib/secrets/freshrss/password.txt";
     virtualHost = "freshrss.comfycamp.space";
   };
+
+  services.nginx.virtualHosts."freshrss.comfycamp.space" = {
+    useACMEHost = "comfycamp.space";
+    forceSSL = true;
+  };
 }

nixos/programs/jellyfin.nix

@@ -3,4 +3,15 @@
   services.jellyfin = {
     enable = true;
   };
+
+  services.nginx.virtualHosts."jf.comfycamp.space" = {
+    useACMEHost = "comfycamp.space";
+    forceSSL = true;
+    locations = {
+      "/" = {
+        proxyPass = "http://127.0.0.1:8096";
+        proxyWebsockets = true;
+      };
+    };
+  };
 }

nixos/programs/maddy.nix

@@ -25,4 +25,10 @@
       "driver postgres"
     ] options.services.maddy.config.default;
   };
+
+  services.nginx.virtualHosts."mta-sts.comfycamp.space" = {
+    useACMEHost = "comfycamp.space";
+    forceSSL = true;
+    root = "/var/lib/mta-sts";
+  };
 }

nixos/programs/mastodon.nix

@@ -78,4 +78,31 @@
   systemd.services.mastodon-sidekiq-all = {
     serviceConfig.ReadWritePaths = "/hdd/mastodon-public-system";
   };
+
+  services.nginx.virtualHosts."m.comfycamp.space" = {
+    useACMEHost = "comfycamp.space";
+    forceSSL = true;
+
+    root = "${config.services.mastodon.package}/public/";
+
+    locations = {
+      "/system/" = {
+        alias = "/var/lib/mastodon/public-system/";
+      };
+
+      "/" = {
+        tryFiles = "$uri @proxy";
+      };
+
+      "@proxy" = {
+        proxyPass = "http://unix:/run/mastodon-web/web.socket";
+        proxyWebsockets = true;
+      };
+
+      "/api/v1/streaming/" = {
+        proxyPass = "http://unix:/run/mastodon-streaming/streaming.socket";
+        proxyWebsockets = true;
+      };
+    };
+  };
 }

nixos/programs/microboard.nix

@@ -15,4 +15,14 @@ in
       "/run/postgresql:/run/postgresql"
     ];
   };
+  
+  services.nginx.virtualHosts."0ch.space" = {
+    useACMEHost = "0ch.space";
+    forceSSL = true;
+    locations = {
+      "/" = {
+        proxyPass = "http://127.0.0.1:55006";
+      };
+    };
+  };
 }

nixos/programs/nextcloud.nix

@@ -20,4 +20,9 @@
       createLocally = false;
     };
   };
+
+  services.nginx.virtualHosts."nc.comfycamp.space" = {
+    useACMEHost = "comfycamp.space";
+    forceSSL = true;
+  };
 }

nixos/programs/nginx.nix

@@ -1,14 +1,4 @@
-{ config, ... }:
+{ config, ... }: {
-let
-  # Required stuff for synapse
-  clientConfig."m.homeserver".base_url = "https://matrix.comfycamp.space";
-  serverConfig."m.server" = "matrix.comfycamp.space:443";
-  mkWellKnown = data: ''
-    add_header Content-Type application/json;
-    add_header Access-Control-Allow-Origin *;
-    return 200 '${builtins.toJSON data}';
-  '';
-in {
   services.nginx = {
     enable = true;
     recommendedTlsSettings = true;
@@ -23,149 +13,14 @@ in {
           https   "max-age=31536000; includeSubdomains; preload";
       }
       add_header Strict-Transport-Security $hsts_header;
-      access_log off;
+      
+      # Log 4xx and 5xx errors.
+      map $status $loggable {
+          ~^[23]  0;
+          default 1;
+      }
+      access_log /var/log/nginx/access.log combined if=$loggable;
     '';
-
-    virtualHosts = {
-      "[201:80ed:6eeb:aea4:cdc0:c836:2831:f2dd]" = {
-        locations = {
-          "/".proxyPass = "http://127.0.0.1:55007";
-        };
-      };
-
-      "comfycamp.space" = {
-        useACMEHost = "comfycamp.space";
-        forceSSL = true;
-        locations = {
-          "/".proxyPass = "http://127.0.0.1:55007";
-          # This section is not needed if the server_name of matrix-synapse is equal to
-          # the domain (i.e. example.org from @foo:example.org) and the federation port
-          # is 8448.
-          # Further reference can be found in the docs about delegation under
-          # https://matrix-org.github.io/synapse/latest/delegate.html
-          "/.well-known/matrix/server".extraConfig = mkWellKnown serverConfig;
-          # This is usually needed for homeserver discovery (from e.g. other Matrix clients).
-          # Further reference can be found in the upstream docs at
-          # https://spec.matrix.org/latest/client-server-api/#getwell-knownmatrixclient
-          "/.well-known/matrix/client".extraConfig = mkWellKnown clientConfig;
-        };
-      };
-
-      # Phoenix
-      "ph.comfycamp.space" = {
-        useACMEHost = "comfycamp.space";
-        forceSSL = true;
-        locations."/".proxyPass = "http://127.0.0.1:55009";
-      };
-
-      # Nextcloud
-      "nc.comfycamp.space" = {
-        useACMEHost = "comfycamp.space";
-        forceSSL = true;
-      };
-
-      # Jellyfin
-      "jf.comfycamp.space" = {
-        useACMEHost = "comfycamp.space";
-        forceSSL = true;
-        locations = {
-          "/" = {
-            proxyPass = "http://127.0.0.1:8096";
-            proxyWebsockets = true;
-          };
-        };
-      };
-
-      # Plausible
-      "plausible.comfycamp.space" = {
-        useACMEHost = "comfycamp.space";
-        forceSSL = true;
-        locations = {
-          "/" = {
-            proxyPass = "http://127.0.0.1:55005";
-          };
-        };
-      };
-      
-      # Microboard
-      "0ch.space" = {
-        useACMEHost = "0ch.space";
-        forceSSL = true;
-        locations = {
-          "/" = {
-            proxyPass = "http://127.0.0.1:55006";
-          };
-        };
-      };
-
-      # Mail: MTA-STS
-      "mta-sts.comfycamp.space" = {
-        useACMEHost = "comfycamp.space";
-        forceSSL = true;
-        root = "/var/lib/mta-sts";
-      };
-
-      "matrix.comfycamp.space" = {
-        useACMEHost = "comfycamp.space";
-        forceSSL = true;
-        locations = {
-          "/".extraConfig = ''
-            return 404;
-          '';
-          # Forward all Matrix API calls to the synapse Matrix homeserver. A trailing slash
-          # *must not* be used here.
-          "/_matrix".proxyPass = "http://127.0.0.1:8008";
-          # Forward requests for e.g. SSO and password-resets.
-          "/_synapse/client".proxyPass = "http://127.0.0.1:8008";
-        };
-      };
-
-      # Photoprism
-      "pp.comfycamp.space" = {
-        useACMEHost = "comfycamp.space";
-        forceSSL = true;
-        locations = {
-          "/" = {
-            proxyPass = "http://127.0.0.1:55004";
-            proxyWebsockets = true;
-          };
-        };
-      };
-
-      # Freshrss
-      "freshrss.comfycamp.space" = {
-        useACMEHost = "comfycamp.space";
-        forceSSL = true;
-      };
-
-      # Mastodon
-      "m.comfycamp.space" = {
-        useACMEHost = "comfycamp.space";
-        forceSSL = true;
-
-        root = "${config.services.mastodon.package}/public/";
-
-        locations = {
-          "/system/" = {
-            alias = "/var/lib/mastodon/public-system/";
-          };
-
-          "/" = {
-            tryFiles = "$uri @proxy";
-          };
-
-          "@proxy" = {
-            proxyPass = "http://unix:/run/mastodon-web/web.socket";
-            proxyWebsockets = true;
-          };
-
-          "/api/v1/streaming/" = {
-            proxyPass = "http://unix:/run/mastodon-streaming/streaming.socket";
-            proxyWebsockets = true;
-          };
-        };
-      };
-    };
   };
 
   users.users.nginx.extraGroups = [ "acme" ];

nixos/programs/phoenix.nix

@@ -12,4 +12,10 @@
       "/var/lib/phoenix:/var/lib/phoenix"
     ];
   };
+
+  services.nginx.virtualHosts."ph.comfycamp.space" = {
+    useACMEHost = "comfycamp.space";
+    forceSSL = true;
+    locations."/".proxyPass = "http://127.0.0.1:55009";
+  };
 }

nixos/programs/photoprism.nix

@@ -21,4 +21,15 @@
       PHOTOPRISM_DETECT_NSFW = "false";
     };
   };
+
+  services.nginx.virtualHosts."pp.comfycamp.space" = {
+    useACMEHost = "comfycamp.space";
+    forceSSL = true;
+    locations = {
+      "/" = {
+        proxyPass = "http://127.0.0.1:55004";
+        proxyWebsockets = true;
+      };
+    };
+  };
 }

nixos/programs/synapse.nix

@@ -1,5 +1,14 @@
 { config, ... }:
-{
+let
+  # Required stuff for synapse
+  clientConfig."m.homeserver".base_url = "https://matrix.comfycamp.space";
+  serverConfig."m.server" = "matrix.comfycamp.space:443";
+  mkWellKnown = data: ''
+    add_header Content-Type application/json;
+    add_header Access-Control-Allow-Origin *;
+    return 200 '${builtins.toJSON data}';
+  '';
+in {
   services.matrix-synapse = {
     enable = true;
     settings = {
@@ -12,9 +21,11 @@
           database = "matrix-synapse";
         };
       };
+      enable_metrics = true;
       enable_registration = true;
       report_stats = true;
-      listeners = [{
+      listeners = [
+        {
           bind_addresses = [ "127.0.0.1" ];
           port = 8008;
           type = "http";
@@ -24,11 +35,48 @@
             names = [ "client" "federation" ];
             compress = false;
           }];
+        }
+        {
+          bind_addresses = [ "127.0.0.1" ];
+          port = 55013;
+          type = "metrics";
+          tls = false;
+          resources = [{
+            names = [ "metrics" ];
+            compress = false;
           }];
+        }
+      ];
       signing_key_path = "/var/lib/matrix-synapse/matrix.comfycamp.space.signing.key";
     };
     extraConfigFiles = [
       "/var/lib/secrets/matrix-synapse/config.yml"
     ];
   };
+
+  services.nginx.virtualHosts."matrix.comfycamp.space" = {
+    useACMEHost = "comfycamp.space";
+    forceSSL = true;
+    locations = {
+      "/".extraConfig = ''
+        return 404;
+      '';
+      # Forward all Matrix API calls to the synapse Matrix homeserver. A trailing slash
+      # *must not* be used here.
+      "/_matrix".proxyPass = "http://127.0.0.1:8008";
+      # Forward requests for e.g. SSO and password-resets.
+      "/_synapse/client".proxyPass = "http://127.0.0.1:8008";
+    };
+  };
+
+  # This section is not needed if the server_name of matrix-synapse is equal to
+  # the domain (i.e. example.org from @foo:example.org) and the federation port
+  # is 8448.
+  # Further reference can be found in the docs about delegation under
+  # https://matrix-org.github.io/synapse/latest/delegate.html
+  services.nginx.virtualHosts."comfycamp.space".locations."/.well-known/matrix/server".extraConfig = mkWellKnown serverConfig;
+  # This is usually needed for homeserver discovery (from e.g. other Matrix clients).
+  # Further reference can be found in the upstream docs at
+  # https://spec.matrix.org/latest/client-server-api/#getwell-knownmatrixclient
+  services.nginx.virtualHosts."comfycamp.space".locations."/.well-known/matrix/client".extraConfig = mkWellKnown clientConfig;
 }