From 7690c4596dd425d4365ecc16b6d4f092225b76cc Mon Sep 17 00:00:00 2001 From: Alpha Chen Date: Tue, 4 Apr 2023 18:45:28 -0700 Subject: [PATCH] o11y --- lotus-land-story/Rakefile | 3 + lotus-land-story/main.tf | 18 ++- lotus-land-story/main.yml | 210 +++++++++++++++++++++++-- lotus-land-story/templates/Caddyfile | 17 +- lotus-land-story/templates/daemon.json | 4 + 5 files changed, 236 insertions(+), 16 deletions(-) create mode 100644 lotus-land-story/templates/daemon.json diff --git a/lotus-land-story/Rakefile b/lotus-land-story/Rakefile index da4c765..27c9d49 100644 --- a/lotus-land-story/Rakefile +++ b/lotus-land-story/Rakefile @@ -10,6 +10,7 @@ namespace :pip do sh "pip-sync requirements.txt" end + desc "" task :upgrade do out, _ = Open3.capture2e("pip-compile --upgrade --resolver=backtracking requirements.in") File.write("requirements.txt", out) @@ -21,10 +22,12 @@ namespace :pip do end end +desc "" task terraform: "terraform.tfvars" do sh "terraform apply" end +desc "" task ansible: "vars.yml" do sh "ansible-playbook main.yml" end diff --git a/lotus-land-story/main.tf b/lotus-land-story/main.tf index 1cab02c..cfc9bd7 100644 --- a/lotus-land-story/main.tf +++ b/lotus-land-story/main.tf @@ -52,13 +52,21 @@ data "linode_domain" "domain" { domain = var.domain } -resource "linode_domain_record" "rww" { - domain_id = data.linode_domain.domain.id - name = "rss" - record_type = "A" - target = resource.linode_instance.lotus_land_story.ip_address +resource "linode_domain_record" "subdomains" { + domain_id = data.linode_domain.domain.id + record_type = "A" + target = resource.linode_instance.lotus_land_story.ip_address + + for_each = toset(["rss", "prometheus", "grafana"]) + name = each.key } +resource "linode_domain_record" "prometheus" { + domain_id = data.linode_domain.domain.id + name = "prometheus" + record_type = "A" + target = resource.linode_instance.lotus_land_story.ip_address +} output "lotus_land_story_ip" { value = resource.linode_instance.lotus_land_story.ip_address diff --git a/lotus-land-story/main.yml b/lotus-land-story/main.yml index 1b51e11..fd34eed 100644 --- a/lotus-land-story/main.yml +++ b/lotus-land-story/main.yml @@ -79,23 +79,40 @@ - docker-buildx-plugin - docker-compose-plugin update_cache: true - - - name: Set up postgres - become: true - become_user: postgres - block: - - name: Get docker0 IP address ansible.builtin.shell: ip -4 -o addr show docker0 | awk '{print $4}' # noqa: risky-shell-pipe vars: executable: /usr/bin/bash register: docker_ip changed_when: docker_ip.rc != 0 + - name: Set docker ip fact + ansible.builtin.set_fact: + docker_ip: + cidr: "{{ docker_ip.stdout }}" + address: "{{ docker_ip.stdout | ansible.utils.ipaddr('address') }}" + - name: Create Docker volume location + ansible.builtin.file: + path: /mnt/lotus-land-story/docker + state: directory + mode: "0755" + - name: Docker config + ansible.builtin.template: + src: templates/daemon.json + dest: /etc/docker/daemon.json + mode: "0644" + notify: Restart docker + + + - name: Set up postgres + become: true + become_user: postgres + block: + - name: Listen on docker0 interface ansible.builtin.lineinfile: dest: "/etc/postgresql/13/main/conf.d/listen.conf" regexp: '^#?listen_addresses=' - line: "listen_addresses='localhost,{{ docker_ip.stdout | ansible.utils.ipaddr('address') }}'" + line: "listen_addresses='localhost,{{ docker_ip.address }}'" state: present create: true mode: "0644" @@ -132,6 +149,48 @@ db: miniflux notify: Restart postgres + - name: Set up postgres-exporter + block: + - name: Get postgres roles + community.postgresql.postgresql_info: + filter: roles + register: postgres_info + - name: Add postgres permissions for postgres-exporter + community.postgresql.postgresql_query: + query: | + CREATE USER prometheus; + ALTER USER prometheus SET SEARCH_PATH TO prometheus,pg_catalog; + + CREATE SCHEMA prometheus AUTHORIZATION prometheus; + + CREATE FUNCTION prometheus.f_select_pg_stat_activity() + RETURNS setof pg_catalog.pg_stat_activity + LANGUAGE sql + SECURITY DEFINER + AS $$ + SELECT * from pg_catalog.pg_stat_activity; + $$; + + CREATE FUNCTION prometheus.f_select_pg_stat_replication() + RETURNS setof pg_catalog.pg_stat_replication + LANGUAGE sql + SECURITY DEFINER + AS $$ + SELECT * from pg_catalog.pg_stat_replication; + $$; + + CREATE VIEW prometheus.pg_stat_replication + AS + SELECT * FROM prometheus.f_select_pg_stat_replication(); + + CREATE VIEW prometheus.pg_stat_activity + AS + SELECT * FROM prometheus.f_select_pg_stat_activity(); + + GRANT SELECT ON prometheus.pg_stat_replication TO prometheus; + GRANT SELECT ON prometheus.pg_stat_activity TO prometheus; + when: "'prometheus' not in postgres_info.roles" + - name: Run miniflux community.docker.docker_compose: project_name: miniflux @@ -149,8 +208,133 @@ - ADMIN_USERNAME=alpha - ADMIN_PASSWORD={{ miniflux_password }} - BASE_URL=https://rss.{{ domain }} + restart: unless-stopped + extra_hosts: + - "host.docker.internal:host-gateway" + + - name: Install host exporters + ansible.builtin.apt: + pkg: + - prometheus-node-exporter + - prometheus-postgres-exporter + state: present + # /usr/share/doc/prometheus-postgres-exporter/README.Debian + - name: Configure postgres-exporter + ansible.builtin.lineinfile: + dest: /etc/default/prometheus-postgres-exporter + regexp: '^DATA_SOURCE_NAME=' + line: "DATA_SOURCE_NAME='user=prometheus host=/run/postgresql dbname=postgres'" + state: present + - name: Create Prometheus dir + ansible.builtin.file: + path: /mnt/lotus-land-story/prometheus + state: directory + mode: "0755" + - name: Prometheus config + ansible.builtin.copy: + dest: /mnt/lotus-land-story/prometheus/prometheus.yml + content: | + global: + # Attach these labels to any time series or alerts when communicating with + # external systems (federation, remote storage, Alertmanager). + external_labels: + monitor: 'codelab-monitor' + + scrape_configs: + - job_name: 'prometheus' + static_configs: + - targets: ['localhost:9090'] + + - job_name: node + static_configs: + - targets: ['host.docker.internal:9100'] + + - job_name: 'docker' + static_configs: + - targets: ['host.docker.internal:9323'] + + - job_name: caddy + static_configs: + - targets: ['host.docker.internal:2019'] + + - job_name: 'grafana' + static_configs: + - targets: ['host.docker.internal:3000'] + + - job_name: 'postgres' + static_configs: + - targets: ['host.docker.internal:9187'] + mode: "0644" + - name: Run Prometheus + community.docker.docker_compose: + project_name: prometheus + remove_orphans: true + definition: + version: "3.3" + services: + prometheus: + image: prom/prometheus:latest + ports: + - "9090:9090" + volumes: + - /mnt/lotus-land-story/prometheus:/etc/prometheus + restart: unless-stopped + extra_hosts: + - "host.docker.internal:host-gateway" + + - name: Create Grafana dir + ansible.builtin.file: + path: /mnt/lotus-land-story/grafana/provisioning/{{ item }} + state: directory + mode: "0755" + loop: + - datasources + - name: Configure grafana + ansible.builtin.copy: + dest: /mnt/lotus-land-story/grafana/grafana.ini + content: | + [metrics] + enabled = true + disable_total_stats = false + mode: "0644" + - name: Provision Grafana + ansible.builtin.copy: + dest: /mnt/lotus-land-story/grafana/provisioning/datasources/prometheus.yml + content: | + apiVersion: 1 + + datasources: + - name: Prometheus + type: prometheus + # Access mode - proxy (server in the UI) or direct (browser in the UI). + access: proxy + url: http://host.docker.internal:9090 + jsonData: + httpMethod: POST + manageAlerts: true + prometheusType: Prometheus + prometheusVersion: 2.37.0 + mode: "0644" + - name: Run Grafana + community.docker.docker_compose: + project_name: grafana + remove_orphans: true + definition: + version: "3.3" + services: + grafana: + image: grafana/grafana-oss:latest + ports: + - "3000:3000" + volumes: + - /mnt/lotus-land-story/grafana/grafana.ini:/etc/grafana/grafana.ini + - /mnt/lotus-land-story/grafana/provisioning:/etc/grafana/provisioning + - grafana:/var/lib/grafana + restart: unless-stopped extra_hosts: - "host.docker.internal:host-gateway" + volumes: + grafana: - name: Make /mnt/lotus-land-story/caddy ansible.builtin.file: @@ -160,7 +344,6 @@ loop: - caddy - caddy/data - - caddy/config - name: Set up Caddyfile ansible.builtin.template: src: templates/Caddyfile @@ -178,12 +361,16 @@ ports: - "80:80" - "443:443" + - "2019:2019" volumes: - /mnt/lotus-land-story/caddy/Caddyfile:/etc/caddy/Caddyfile - /mnt/lotus-land-story/caddy/data:/data - - /mnt/lotus-land-story/caddy/config:/config + - caddy-config:/config + restart: unless-stopped extra_hosts: - "host.docker.internal:host-gateway" + volumes: + caddy-config: handlers: - name: Restart postgres @@ -191,4 +378,9 @@ name: postgresql state: restarted + - name: Restart docker + ansible.builtin.service: + name: docker + state: restarted + # vim: ft=yaml.ansible diff --git a/lotus-land-story/templates/Caddyfile b/lotus-land-story/templates/Caddyfile index f36e473..4bd1534 100644 --- a/lotus-land-story/templates/Caddyfile +++ b/lotus-land-story/templates/Caddyfile @@ -1,2 +1,15 @@ -rss.{{ domain }} -reverse_proxy host.docker.internal:8080 +:2019 { + metrics +} + +rss.{{ domain }} { + reverse_proxy host.docker.internal:8080 +} + +prometheus.{{ domain }} { + reverse_proxy host.docker.internal:9090 +} + +grafana.{{ domain }} { + reverse_proxy host.docker.internal:3000 +} diff --git a/lotus-land-story/templates/daemon.json b/lotus-land-story/templates/daemon.json new file mode 100644 index 0000000..ba28dd9 --- /dev/null +++ b/lotus-land-story/templates/daemon.json @@ -0,0 +1,4 @@ +{ + "data-root": "/mnt/lotus-land-story/docker", + "metrics-addr": "{{ docker_ip.address }}:9323" +}