boxen/lotus-land-story/logs.yml

# https://www.parseable.com/docs/log-ingestion/agents/vector
# https://vector.dev/docs/setup/installation/platforms/docker/

- name: Set up Parseable
  hosts: lotus-land-story
  vars_files:
    - vars.yml
  tasks:

    - name: Create directories for volume mounting
      ansible.builtin.file:
        path: /mnt/lotus-land-story/parseable/{{ item }}
        state: directory
        mode: "0755"
      loop:
        - data
        - staging

    - name: Configure Vector
      ansible.builtin.copy:
        dest: /mnt/lotus-land-story/parseable/vector.yml
        content: |
          sources:
            # vector_metrics:
            #   type: internal_metrics
            raw_docker_logs:
              type: docker_logs
          transforms:
            docker_logs:
              type: remap
              inputs:
                - raw_docker_logs
              source: |
                if includes(["authelia", "caddy", "miniflux"], .container_name) {
                  . |= object!(parse_json!(.message))
                } else if .container_name == "paperless" {
                  # asctime has trailing milliseconds, which I can't figure out
                  # how to parse, but I also don't care about it, so drop it
                  parsed = parse_regex!(
                    .message,
                    r'\[(?P<asctime>.*?),\d*\] \[(?P<level>.*?)\] \[(?P<name>.*?)\] (?P<message>.*)',
                  )
                  .paperless_time = parse_timestamp!(del(parsed.asctime), format: "%F %T")
                  . |= parsed
                } else if includes(["grafana", "loki"], .container_name) {
                  . |= parse_key_value!(.message)
                }
          sinks:
            # console:
            #   type: console
            #   inputs:
            #     - demo_logs
            #   encoding:
            #     codec: json
            parseable:
              type: http
              method: post
              batch:
                max_bytes: 10485760
                max_events: 1000
                timeout_secs: 10
              compression: gzip
              inputs:
                - docker_logs
              encoding:
                codec: json
              uri: http://parseable:8000/api/v1/ingest
              auth:
                strategy: basic
                user: alpha
                password: {{ parseable.password }}
              request:
                headers:
                  X-P-Stream: vector
              healthcheck:
                enabled: true
                path: http://parseable/api/v1/liveness
                port: 8000
            # prometheus:
            #   type: prometheus_remote_write
            #   endpoint: http://prometheus:9090
            #   inputs:
            #     - vector_metrics
        mode: "0600"

    - name: Get docker network
      community.docker.docker_network:
        name: lotus_land_story
      register: docker_network

    # https://www.parseable.com/logstash/docker-compose.yaml
    - name: Run Parseable
      community.docker.docker_container:
        restart: true
        name: parseable
        image: parseable/parseable:v0.7.3
        command:
          - parseable
          - local-store
        env:
          P_FS_DIR: /parseable/data
          P_STAGING_DIR: /parseable/staging
          P_USERNAME: alpha
          P_PASSWORD: "{{ parseable.password }}"
          P_OIDC_CLIENT_ID: parseable
          P_OIDC_CLIENT_SECRET: "{{ parseable.oidc_secret }}"
          P_OIDC_ISSUER: https://auth.{{ domain }}
          P_ORIGIN_URI: https://logs.{{ domain }}
          # RUST_LOG: warning
        volumes:
          - ./data:/parseable/data
          - ./staging:/parseable/staging
        restart_policy: unless-stopped
        networks:
          - name: lotus_land_story

    - name: Run Vector
      community.docker.docker_container:
        restart: true
        name: vector
        image: timberio/vector:0.35.0-alpine
        env:
          # VECTOR_LOG: debug
        volumes:
          - /mnt/lotus-land-story/parseable/vector.yml:/etc/vector/vector.yaml
          - /var/run/docker.sock:/var/run/docker.sock # for docker_logs
        restart_policy: unless-stopped
        networks:
          - name: lotus_land_story

# vim: ft=yaml.ansible