You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
132 lines
4.2 KiB
132 lines
4.2 KiB
# https://www.parseable.com/docs/log-ingestion/agents/vector
|
|
# https://vector.dev/docs/setup/installation/platforms/docker/
|
|
|
|
- name: Set up Parseable
|
|
hosts: lotus-land-story
|
|
vars_files:
|
|
- vars.yml
|
|
tasks:
|
|
|
|
- name: Create directories for volume mounting
|
|
ansible.builtin.file:
|
|
path: /mnt/lotus-land-story/parseable/{{ item }}
|
|
state: directory
|
|
mode: "0755"
|
|
loop:
|
|
- data
|
|
- staging
|
|
|
|
- name: Configure Vector
|
|
ansible.builtin.copy:
|
|
dest: /mnt/lotus-land-story/parseable/vector.yml
|
|
content: |
|
|
sources:
|
|
# vector_metrics:
|
|
# type: internal_metrics
|
|
raw_docker_logs:
|
|
type: docker_logs
|
|
transforms:
|
|
docker_logs:
|
|
type: remap
|
|
inputs:
|
|
- raw_docker_logs
|
|
source: |
|
|
if includes(["authelia", "caddy", "miniflux"], .container_name) {
|
|
. |= object!(parse_json!(.message))
|
|
} else if .container_name == "paperless" {
|
|
# asctime has trailing milliseconds, which I can't figure out
|
|
# how to parse, but I also don't care about it, so drop it
|
|
parsed = parse_regex!(
|
|
.message,
|
|
r'\[(?P<asctime>.*?),\d*\] \[(?P<level>.*?)\] \[(?P<name>.*?)\] (?P<message>.*)',
|
|
)
|
|
.paperless_time = parse_timestamp!(del(parsed.asctime), format: "%F %T")
|
|
. |= parsed
|
|
} else if includes(["grafana", "loki"], .container_name) {
|
|
. |= parse_key_value!(.message)
|
|
}
|
|
sinks:
|
|
# console:
|
|
# type: console
|
|
# inputs:
|
|
# - demo_logs
|
|
# encoding:
|
|
# codec: json
|
|
parseable:
|
|
type: http
|
|
method: post
|
|
batch:
|
|
max_bytes: 10485760
|
|
max_events: 1000
|
|
timeout_secs: 10
|
|
compression: gzip
|
|
inputs:
|
|
- docker_logs
|
|
encoding:
|
|
codec: json
|
|
uri: http://parseable:8000/api/v1/ingest
|
|
auth:
|
|
strategy: basic
|
|
user: alpha
|
|
password: {{ parseable.password }}
|
|
request:
|
|
headers:
|
|
X-P-Stream: vector
|
|
healthcheck:
|
|
enabled: true
|
|
path: http://parseable/api/v1/liveness
|
|
port: 8000
|
|
# prometheus:
|
|
# type: prometheus_remote_write
|
|
# endpoint: http://prometheus:9090
|
|
# inputs:
|
|
# - vector_metrics
|
|
mode: "0600"
|
|
|
|
- name: Get docker network
|
|
community.docker.docker_network:
|
|
name: lotus_land_story
|
|
register: docker_network
|
|
|
|
# https://www.parseable.com/logstash/docker-compose.yaml
|
|
- name: Run Parseable
|
|
community.docker.docker_container:
|
|
restart: true
|
|
name: parseable
|
|
image: parseable/parseable:v0.7.3
|
|
command:
|
|
- parseable
|
|
- local-store
|
|
env:
|
|
P_FS_DIR: /parseable/data
|
|
P_STAGING_DIR: /parseable/staging
|
|
P_USERNAME: alpha
|
|
P_PASSWORD: "{{ parseable.password }}"
|
|
P_OIDC_CLIENT_ID: parseable
|
|
P_OIDC_CLIENT_SECRET: "{{ parseable.oidc_secret }}"
|
|
P_OIDC_ISSUER: https://auth.{{ domain }}
|
|
P_ORIGIN_URI: https://logs.{{ domain }}
|
|
# RUST_LOG: warning
|
|
volumes:
|
|
- ./data:/parseable/data
|
|
- ./staging:/parseable/staging
|
|
restart_policy: unless-stopped
|
|
networks:
|
|
- name: lotus_land_story
|
|
|
|
- name: Run Vector
|
|
community.docker.docker_container:
|
|
restart: true
|
|
name: vector
|
|
image: timberio/vector:0.35.0-alpine
|
|
env:
|
|
# VECTOR_LOG: debug
|
|
volumes:
|
|
- /mnt/lotus-land-story/parseable/vector.yml:/etc/vector/vector.yaml
|
|
- /var/run/docker.sock:/var/run/docker.sock # for docker_logs
|
|
restart_policy: unless-stopped
|
|
networks:
|
|
- name: lotus_land_story
|
|
|
|
# vim: ft=yaml.ansible
|