microdao-daarion/infrastructure/ansible/playbooks/postgres-ha.yml

# DAARION Network - PostgreSQL HA Setup
# Patroni + PgBouncer + pgBackRest + Monitoring
---
# =============================================================================
# POSTGRESQL HA WITH PATRONI
# =============================================================================
- name: Setup PostgreSQL HA Cluster
  hosts: database_nodes
  become: yes
  
  vars:
    # Patroni
    patroni_version: "3.2.0"
    patroni_scope: "daarion-cluster"
    patroni_namespace: "/daarion"
    
    # PostgreSQL
    postgres_version: "16"
    postgres_data_dir: "/var/lib/postgresql/{{ postgres_version }}/main"
    postgres_config_dir: "/etc/postgresql/{{ postgres_version }}/main"
    
    # PgBouncer
    pgbouncer_port: 6432
    pgbouncer_max_client_conn: 1000
    pgbouncer_default_pool_size: 50
    
    # Backup
    pgbackrest_repo_path: "/var/lib/pgbackrest"
    pgbackrest_s3_bucket: "daarion-backups"
    
    # Consul for DCS
    consul_host: "{{ hostvars[groups['masters'][0]].ansible_host }}"
    
  tasks:
    # =========================================================================
    # PREREQUISITES
    # =========================================================================
    - name: Add PostgreSQL APT repository
      shell: |
        curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor -o /usr/share/keyrings/postgresql-keyring.gpg
        echo "deb [signed-by=/usr/share/keyrings/postgresql-keyring.gpg] http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list
      args:
        creates: /etc/apt/sources.list.d/pgdg.list
        
    - name: Update apt cache
      apt:
        update_cache: yes
        
    - name: Install PostgreSQL and dependencies
      apt:
        name:
          - postgresql-{{ postgres_version }}
          - postgresql-contrib-{{ postgres_version }}
          - python3-pip
          - python3-psycopg2
          - python3-consul
          - pgbouncer
          - pgbackrest
        state: present
        
    # =========================================================================
    # PATRONI INSTALLATION
    # =========================================================================
    - name: Install Patroni
      pip:
        name:
          - patroni[consul]=={{ patroni_version }}
          - python-consul
        state: present
        
    - name: Create Patroni directories
      file:
        path: "{{ item }}"
        state: directory
        owner: postgres
        group: postgres
        mode: '0750'
      loop:
        - /etc/patroni
        - /var/log/patroni
        
    - name: Configure Patroni
      template:
        src: templates/patroni.yml.j2
        dest: /etc/patroni/patroni.yml
        owner: postgres
        group: postgres
        mode: '0640'
      notify: restart patroni
      
    - name: Create Patroni systemd service
      copy:
        dest: /etc/systemd/system/patroni.service
        content: |
          [Unit]
          Description=Patroni PostgreSQL Cluster Manager
          After=network.target consul.service
          
          [Service]
          Type=simple
          User=postgres
          Group=postgres
          ExecStart=/usr/local/bin/patroni /etc/patroni/patroni.yml
          ExecReload=/bin/kill -HUP $MAINPID
          KillMode=process
          TimeoutSec=30
          Restart=on-failure
          
          [Install]
          WantedBy=multi-user.target
      notify:
        - reload systemd
        - restart patroni
        
    # =========================================================================
    # PGBOUNCER
    # =========================================================================
    - name: Configure PgBouncer
      template:
        src: templates/pgbouncer.ini.j2
        dest: /etc/pgbouncer/pgbouncer.ini
        owner: postgres
        group: postgres
        mode: '0640'
      notify: restart pgbouncer
      
    - name: Configure PgBouncer userlist
      copy:
        dest: /etc/pgbouncer/userlist.txt
        content: |
          "{{ postgres_user }}" "{{ postgres_password }}"
          "pgbouncer" "{{ pgbouncer_password | default('pgbouncer_secret') }}"
        owner: postgres
        group: postgres
        mode: '0600'
      notify: restart pgbouncer
      
    - name: Enable PgBouncer
      service:
        name: pgbouncer
        enabled: yes
        state: started
        
    # =========================================================================
    # PGBACKREST
    # =========================================================================
    - name: Create pgBackRest directories
      file:
        path: "{{ item }}"
        state: directory
        owner: postgres
        group: postgres
        mode: '0750'
      loop:
        - "{{ pgbackrest_repo_path }}"
        - /var/log/pgbackrest
        
    - name: Configure pgBackRest
      template:
        src: templates/pgbackrest.conf.j2
        dest: /etc/pgbackrest.conf
        owner: postgres
        group: postgres
        mode: '0640'
        
    - name: Setup backup cron
      cron:
        name: "Daily PostgreSQL backup"
        hour: "2"
        minute: "0"
        user: postgres
        job: "pgbackrest --stanza={{ patroni_scope }} --type=diff backup >> /var/log/pgbackrest/backup.log 2>&1"
        
    - name: Setup weekly full backup
      cron:
        name: "Weekly full PostgreSQL backup"
        weekday: "0"
        hour: "3"
        minute: "0"
        user: postgres
        job: "pgbackrest --stanza={{ patroni_scope }} --type=full backup >> /var/log/pgbackrest/backup.log 2>&1"
        
    # =========================================================================
    # MONITORING
    # =========================================================================
    - name: Install postgres_exporter
      shell: |
        curl -sL https://github.com/prometheus-community/postgres_exporter/releases/download/v0.15.0/postgres_exporter-0.15.0.linux-amd64.tar.gz | tar xz
        mv postgres_exporter-0.15.0.linux-amd64/postgres_exporter /usr/local/bin/
        rm -rf postgres_exporter-0.15.0.linux-amd64
      args:
        creates: /usr/local/bin/postgres_exporter
        
    - name: Create postgres_exporter systemd service
      copy:
        dest: /etc/systemd/system/postgres_exporter.service
        content: |
          [Unit]
          Description=Prometheus PostgreSQL Exporter
          After=network.target postgresql.service
          
          [Service]
          Type=simple
          User=postgres
          Environment="DATA_SOURCE_NAME=postgresql://{{ postgres_user }}:{{ postgres_password }}@localhost:5432/{{ postgres_db }}?sslmode=disable"
          ExecStart=/usr/local/bin/postgres_exporter --web.listen-address=:9187
          Restart=on-failure
          
          [Install]
          WantedBy=multi-user.target
      notify:
        - reload systemd
        - restart postgres_exporter
        
    - name: Enable postgres_exporter
      service:
        name: postgres_exporter
        enabled: yes
        state: started
        
    # =========================================================================
    # VERIFICATION
    # =========================================================================
    - name: Show PostgreSQL HA status
      debug:
        msg: |
          PostgreSQL HA Setup Complete!
          
          Components:
          - Patroni: Cluster management
          - PgBouncer: Connection pooling (port {{ pgbouncer_port }})
          - pgBackRest: Backups
          - postgres_exporter: Metrics (port 9187)
          
          Connection strings:
          - Direct: postgresql://{{ postgres_user }}@{{ ansible_host }}:5432/{{ postgres_db }}
          - Pooled: postgresql://{{ postgres_user }}@{{ ansible_host }}:{{ pgbouncer_port }}/{{ postgres_db }}
          
  handlers:
    - name: reload systemd
      systemd:
        daemon_reload: yes
        
    - name: restart patroni
      service:
        name: patroni
        state: restarted
        
    - name: restart pgbouncer
      service:
        name: pgbouncer
        state: restarted
        
    - name: restart postgres_exporter
      service:
        name: postgres_exporter
        state: restarted