o11y

lotus-land-story/Rakefile

@@ -10,6 +10,7 @@ namespace :pip do
     sh "pip-sync requirements.txt"
   end
 
+  desc ""
   task :upgrade do
     out, _ = Open3.capture2e("pip-compile --upgrade --resolver=backtracking requirements.in")
     File.write("requirements.txt", out)
@@ -21,10 +22,12 @@ namespace :pip do
   end
 end
 
+desc ""
 task terraform: "terraform.tfvars" do
   sh "terraform apply"
 end
 
+desc ""
 task ansible: "vars.yml" do
   sh "ansible-playbook main.yml"
 end

lotus-land-story/main.tf

@@ -52,13 +52,21 @@ data "linode_domain" "domain" {
   domain = var.domain
 }
 
-resource "linode_domain_record" "rww" {
+resource "linode_domain_record" "subdomains" {
   domain_id   = data.linode_domain.domain.id
-    name = "rss"
   record_type = "A"
   target      = resource.linode_instance.lotus_land_story.ip_address
+
+  for_each = toset(["rss", "prometheus", "grafana"])
+  name = each.key
 }
 
+resource "linode_domain_record" "prometheus" {
+  domain_id   = data.linode_domain.domain.id
+  name        = "prometheus"
+  record_type = "A"
+  target      = resource.linode_instance.lotus_land_story.ip_address
+}
 
 output "lotus_land_story_ip" {
   value = resource.linode_instance.lotus_land_story.ip_address

lotus-land-story/main.yml

@@ -79,23 +79,40 @@
               - docker-buildx-plugin
               - docker-compose-plugin
             update_cache: true
-
-    - name: Set up postgres
-      become: true
-      become_user: postgres
-      block:
-
         - name: Get docker0 IP address
           ansible.builtin.shell: ip -4 -o addr show docker0 | awk '{print $4}' # noqa: risky-shell-pipe
           vars:
             executable: /usr/bin/bash
           register: docker_ip
           changed_when: docker_ip.rc != 0
+        - name: Set docker ip fact
+          ansible.builtin.set_fact:
+            docker_ip:
+              cidr: "{{ docker_ip.stdout }}"
+              address: "{{ docker_ip.stdout | ansible.utils.ipaddr('address') }}"
+        - name: Create Docker volume location
+          ansible.builtin.file:
+            path: /mnt/lotus-land-story/docker
+            state: directory
+            mode: "0755"
+        - name: Docker config
+          ansible.builtin.template:
+            src: templates/daemon.json
+            dest: /etc/docker/daemon.json
+            mode: "0644"
+          notify: Restart docker
+
+
+    - name: Set up postgres
+      become: true
+      become_user: postgres
+      block:
+
         - name: Listen on docker0 interface
           ansible.builtin.lineinfile:
             dest: "/etc/postgresql/13/main/conf.d/listen.conf"
             regexp: '^#?listen_addresses='
-            line: "listen_addresses='localhost,{{ docker_ip.stdout | ansible.utils.ipaddr('address') }}'"
+            line: "listen_addresses='localhost,{{ docker_ip.address }}'"
             state: present
             create: true
             mode: "0644"
@@ -132,6 +149,48 @@
             db: miniflux
           notify: Restart postgres
 
+        - name: Set up postgres-exporter
+          block:
+            - name: Get postgres roles
+              community.postgresql.postgresql_info:
+                filter: roles
+              register: postgres_info
+            - name: Add postgres permissions for postgres-exporter
+              community.postgresql.postgresql_query:
+                query: |
+                  CREATE USER prometheus;
+                  ALTER USER prometheus SET SEARCH_PATH TO prometheus,pg_catalog;
+
+                  CREATE SCHEMA prometheus AUTHORIZATION prometheus;
+
+                  CREATE FUNCTION prometheus.f_select_pg_stat_activity()
+                  RETURNS setof pg_catalog.pg_stat_activity
+                  LANGUAGE sql
+                  SECURITY DEFINER
+                  AS $$
+                    SELECT * from pg_catalog.pg_stat_activity;
+                  $$;
+
+                  CREATE FUNCTION prometheus.f_select_pg_stat_replication()
+                  RETURNS setof pg_catalog.pg_stat_replication
+                  LANGUAGE sql
+                  SECURITY DEFINER
+                  AS $$
+                    SELECT * from pg_catalog.pg_stat_replication;
+                  $$;
+
+                  CREATE VIEW prometheus.pg_stat_replication
+                  AS
+                    SELECT * FROM prometheus.f_select_pg_stat_replication();
+
+                  CREATE VIEW prometheus.pg_stat_activity
+                  AS
+                    SELECT * FROM prometheus.f_select_pg_stat_activity();
+
+                  GRANT SELECT ON prometheus.pg_stat_replication TO prometheus;
+                  GRANT SELECT ON prometheus.pg_stat_activity TO prometheus;
+              when: "'prometheus' not in postgres_info.roles"
+
     - name: Run miniflux
       community.docker.docker_compose:
         project_name: miniflux
@@ -149,9 +208,134 @@
                 - ADMIN_USERNAME=alpha
                 - ADMIN_PASSWORD={{ miniflux_password }}
                 - BASE_URL=https://rss.{{ domain }}
+              restart: unless-stopped
               extra_hosts:
                 - "host.docker.internal:host-gateway"
 
+    - name: Install host exporters
+      ansible.builtin.apt:
+        pkg:
+          - prometheus-node-exporter
+          - prometheus-postgres-exporter
+        state: present
+    # /usr/share/doc/prometheus-postgres-exporter/README.Debian
+    - name: Configure postgres-exporter
+      ansible.builtin.lineinfile:
+        dest: /etc/default/prometheus-postgres-exporter
+        regexp: '^DATA_SOURCE_NAME='
+        line: "DATA_SOURCE_NAME='user=prometheus host=/run/postgresql dbname=postgres'"
+        state: present
+    - name: Create Prometheus dir
+      ansible.builtin.file:
+        path: /mnt/lotus-land-story/prometheus
+        state: directory
+        mode: "0755"
+    - name: Prometheus config
+      ansible.builtin.copy:
+        dest: /mnt/lotus-land-story/prometheus/prometheus.yml
+        content: |
+          global:
+            # Attach these labels to any time series or alerts when communicating with
+            # external systems (federation, remote storage, Alertmanager).
+            external_labels:
+              monitor: 'codelab-monitor'
+
+          scrape_configs:
+            - job_name: 'prometheus'
+              static_configs:
+                - targets: ['localhost:9090']
+
+            - job_name: node
+              static_configs:
+                - targets: ['host.docker.internal:9100']
+
+            - job_name: 'docker'
+              static_configs:
+                - targets: ['host.docker.internal:9323']
+
+            - job_name: caddy
+              static_configs:
+                - targets: ['host.docker.internal:2019']
+
+            - job_name: 'grafana'
+              static_configs:
+                - targets: ['host.docker.internal:3000']
+
+            - job_name: 'postgres'
+              static_configs:
+                - targets: ['host.docker.internal:9187']
+        mode: "0644"
+    - name: Run Prometheus
+      community.docker.docker_compose:
+        project_name: prometheus
+        remove_orphans: true
+        definition:
+          version: "3.3"
+          services:
+            prometheus:
+              image: prom/prometheus:latest
+              ports:
+                - "9090:9090"
+              volumes:
+                - /mnt/lotus-land-story/prometheus:/etc/prometheus
+              restart: unless-stopped
+              extra_hosts:
+                - "host.docker.internal:host-gateway"
+
+    - name: Create Grafana dir
+      ansible.builtin.file:
+        path: /mnt/lotus-land-story/grafana/provisioning/{{ item }}
+        state: directory
+        mode: "0755"
+      loop:
+        - datasources
+    - name: Configure grafana
+      ansible.builtin.copy:
+        dest: /mnt/lotus-land-story/grafana/grafana.ini
+        content: |
+          [metrics]
+          enabled             = true
+          disable_total_stats = false
+        mode: "0644"
+    - name: Provision Grafana
+      ansible.builtin.copy:
+        dest: /mnt/lotus-land-story/grafana/provisioning/datasources/prometheus.yml
+        content: |
+          apiVersion: 1
+
+          datasources:
+            - name: Prometheus
+              type: prometheus
+              # Access mode - proxy (server in the UI) or direct (browser in the UI).
+              access: proxy
+              url: http://host.docker.internal:9090
+              jsonData:
+                httpMethod: POST
+                manageAlerts: true
+                prometheusType: Prometheus
+                prometheusVersion: 2.37.0
+        mode: "0644"
+    - name: Run Grafana
+      community.docker.docker_compose:
+        project_name: grafana
+        remove_orphans: true
+        definition:
+          version: "3.3"
+          services:
+            grafana:
+              image: grafana/grafana-oss:latest
+              ports:
+                - "3000:3000"
+              volumes:
+                - /mnt/lotus-land-story/grafana/grafana.ini:/etc/grafana/grafana.ini
+                - /mnt/lotus-land-story/grafana/provisioning:/etc/grafana/provisioning
+                - grafana:/var/lib/grafana
+              restart: unless-stopped
+              extra_hosts:
+                - "host.docker.internal:host-gateway"
+          volumes:
+            grafana:
+
     - name: Make /mnt/lotus-land-story/caddy
       ansible.builtin.file:
         path: /mnt/lotus-land-story/{{ item }}
@@ -160,7 +344,6 @@
       loop:
         - caddy
         - caddy/data
-        - caddy/config
     - name: Set up Caddyfile
       ansible.builtin.template:
         src: templates/Caddyfile
@@ -178,12 +361,16 @@
               ports:
                 - "80:80"
                 - "443:443"
+                - "2019:2019"
               volumes:
                 - /mnt/lotus-land-story/caddy/Caddyfile:/etc/caddy/Caddyfile
                 - /mnt/lotus-land-story/caddy/data:/data
-                - /mnt/lotus-land-story/caddy/config:/config
+                - caddy-config:/config
+              restart: unless-stopped
               extra_hosts:
                 - "host.docker.internal:host-gateway"
+          volumes:
+            caddy-config:
 
   handlers:
     - name: Restart postgres
@@ -191,4 +378,9 @@
         name: postgresql
         state: restarted
 
+    - name: Restart docker
+      ansible.builtin.service:
+        name: docker
+        state: restarted
+
 # vim: ft=yaml.ansible

lotus-land-story/templates/Caddyfile

@@ -1,2 +1,15 @@
-rss.{{ domain }}
+:2019 {
-reverse_proxy host.docker.internal:8080
+	metrics
+}
+
+rss.{{ domain }} {
+	reverse_proxy host.docker.internal:8080
+}
+
+prometheus.{{ domain }} {
+	reverse_proxy host.docker.internal:9090
+}
+
+grafana.{{ domain }} {
+	reverse_proxy host.docker.internal:3000
+}

lotus-land-story/templates/daemon.json

@@ -0,0 +1,4 @@
+{
+  "data-root": "/mnt/lotus-land-story/docker",
+  "metrics-addr": "{{ docker_ip.address }}:9323"
+}