🧠 Add Agent Memory System with PostgreSQL + Qdrant + Cohere
Features: - Three-tier memory architecture (short/mid/long-term) - PostgreSQL schema for conversations, events, memories - Qdrant vector database for semantic search - Cohere embeddings (embed-multilingual-v3.0, 1024 dims) - FastAPI Memory Service with full CRUD - External Secrets integration with Vault - Kubernetes deployment manifests Components: - infrastructure/database/agent-memory-schema.sql - infrastructure/kubernetes/apps/qdrant/ - infrastructure/kubernetes/apps/memory-service/ - services/memory-service/ (FastAPI app) Also includes: - External Secrets Operator - Traefik Ingress Controller - Cert-Manager with Let's Encrypt - ArgoCD for GitOps
This commit is contained in:
Apple
257
infrastructure/ansible/playbooks/postgres-ha.yml
Normal file
257
infrastructure/ansible/playbooks/postgres-ha.yml
Normal file
@@ -0,0 +1,257 @@
|
|||||||
|
# DAARION Network - PostgreSQL HA Setup
|
||||||
|
# Patroni + PgBouncer + pgBackRest + Monitoring
|
||||||
|
---
|
||||||
|
# =============================================================================
|
||||||
|
# POSTGRESQL HA WITH PATRONI
|
||||||
|
# =============================================================================
|
||||||
|
- name: Setup PostgreSQL HA Cluster
|
||||||
|
hosts: database_nodes
|
||||||
|
become: yes
|
||||||
|
|
||||||
|
vars:
|
||||||
|
# Patroni
|
||||||
|
patroni_version: "3.2.0"
|
||||||
|
patroni_scope: "daarion-cluster"
|
||||||
|
patroni_namespace: "/daarion"
|
||||||
|
|
||||||
|
# PostgreSQL
|
||||||
|
postgres_version: "16"
|
||||||
|
postgres_data_dir: "/var/lib/postgresql/{{ postgres_version }}/main"
|
||||||
|
postgres_config_dir: "/etc/postgresql/{{ postgres_version }}/main"
|
||||||
|
|
||||||
|
# PgBouncer
|
||||||
|
pgbouncer_port: 6432
|
||||||
|
pgbouncer_max_client_conn: 1000
|
||||||
|
pgbouncer_default_pool_size: 50
|
||||||
|
|
||||||
|
# Backup
|
||||||
|
pgbackrest_repo_path: "/var/lib/pgbackrest"
|
||||||
|
pgbackrest_s3_bucket: "daarion-backups"
|
||||||
|
|
||||||
|
# Consul for DCS
|
||||||
|
consul_host: "{{ hostvars[groups['masters'][0]].ansible_host }}"
|
||||||
|
|
||||||
|
tasks:
|
||||||
|
# =========================================================================
|
||||||
|
# PREREQUISITES
|
||||||
|
# =========================================================================
|
||||||
|
- name: Add PostgreSQL APT repository
|
||||||
|
shell: |
|
||||||
|
curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor -o /usr/share/keyrings/postgresql-keyring.gpg
|
||||||
|
echo "deb [signed-by=/usr/share/keyrings/postgresql-keyring.gpg] http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list
|
||||||
|
args:
|
||||||
|
creates: /etc/apt/sources.list.d/pgdg.list
|
||||||
|
|
||||||
|
- name: Update apt cache
|
||||||
|
apt:
|
||||||
|
update_cache: yes
|
||||||
|
|
||||||
|
- name: Install PostgreSQL and dependencies
|
||||||
|
apt:
|
||||||
|
name:
|
||||||
|
- postgresql-{{ postgres_version }}
|
||||||
|
- postgresql-contrib-{{ postgres_version }}
|
||||||
|
- python3-pip
|
||||||
|
- python3-psycopg2
|
||||||
|
- python3-consul
|
||||||
|
- pgbouncer
|
||||||
|
- pgbackrest
|
||||||
|
state: present
|
||||||
|
|
||||||
|
# =========================================================================
|
||||||
|
# PATRONI INSTALLATION
|
||||||
|
# =========================================================================
|
||||||
|
- name: Install Patroni
|
||||||
|
pip:
|
||||||
|
name:
|
||||||
|
- patroni[consul]=={{ patroni_version }}
|
||||||
|
- python-consul
|
||||||
|
state: present
|
||||||
|
|
||||||
|
- name: Create Patroni directories
|
||||||
|
file:
|
||||||
|
path: "{{ item }}"
|
||||||
|
state: directory
|
||||||
|
owner: postgres
|
||||||
|
group: postgres
|
||||||
|
mode: '0750'
|
||||||
|
loop:
|
||||||
|
- /etc/patroni
|
||||||
|
- /var/log/patroni
|
||||||
|
|
||||||
|
- name: Configure Patroni
|
||||||
|
template:
|
||||||
|
src: templates/patroni.yml.j2
|
||||||
|
dest: /etc/patroni/patroni.yml
|
||||||
|
owner: postgres
|
||||||
|
group: postgres
|
||||||
|
mode: '0640'
|
||||||
|
notify: restart patroni
|
||||||
|
|
||||||
|
- name: Create Patroni systemd service
|
||||||
|
copy:
|
||||||
|
dest: /etc/systemd/system/patroni.service
|
||||||
|
content: |
|
||||||
|
[Unit]
|
||||||
|
Description=Patroni PostgreSQL Cluster Manager
|
||||||
|
After=network.target consul.service
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=simple
|
||||||
|
User=postgres
|
||||||
|
Group=postgres
|
||||||
|
ExecStart=/usr/local/bin/patroni /etc/patroni/patroni.yml
|
||||||
|
ExecReload=/bin/kill -HUP $MAINPID
|
||||||
|
KillMode=process
|
||||||
|
TimeoutSec=30
|
||||||
|
Restart=on-failure
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
|
notify:
|
||||||
|
- reload systemd
|
||||||
|
- restart patroni
|
||||||
|
|
||||||
|
# =========================================================================
|
||||||
|
# PGBOUNCER
|
||||||
|
# =========================================================================
|
||||||
|
- name: Configure PgBouncer
|
||||||
|
template:
|
||||||
|
src: templates/pgbouncer.ini.j2
|
||||||
|
dest: /etc/pgbouncer/pgbouncer.ini
|
||||||
|
owner: postgres
|
||||||
|
group: postgres
|
||||||
|
mode: '0640'
|
||||||
|
notify: restart pgbouncer
|
||||||
|
|
||||||
|
- name: Configure PgBouncer userlist
|
||||||
|
copy:
|
||||||
|
dest: /etc/pgbouncer/userlist.txt
|
||||||
|
content: |
|
||||||
|
"{{ postgres_user }}" "{{ postgres_password }}"
|
||||||
|
"pgbouncer" "{{ pgbouncer_password | default('pgbouncer_secret') }}"
|
||||||
|
owner: postgres
|
||||||
|
group: postgres
|
||||||
|
mode: '0600'
|
||||||
|
notify: restart pgbouncer
|
||||||
|
|
||||||
|
- name: Enable PgBouncer
|
||||||
|
service:
|
||||||
|
name: pgbouncer
|
||||||
|
enabled: yes
|
||||||
|
state: started
|
||||||
|
|
||||||
|
# =========================================================================
|
||||||
|
# PGBACKREST
|
||||||
|
# =========================================================================
|
||||||
|
- name: Create pgBackRest directories
|
||||||
|
file:
|
||||||
|
path: "{{ item }}"
|
||||||
|
state: directory
|
||||||
|
owner: postgres
|
||||||
|
group: postgres
|
||||||
|
mode: '0750'
|
||||||
|
loop:
|
||||||
|
- "{{ pgbackrest_repo_path }}"
|
||||||
|
- /var/log/pgbackrest
|
||||||
|
|
||||||
|
- name: Configure pgBackRest
|
||||||
|
template:
|
||||||
|
src: templates/pgbackrest.conf.j2
|
||||||
|
dest: /etc/pgbackrest.conf
|
||||||
|
owner: postgres
|
||||||
|
group: postgres
|
||||||
|
mode: '0640'
|
||||||
|
|
||||||
|
- name: Setup backup cron
|
||||||
|
cron:
|
||||||
|
name: "Daily PostgreSQL backup"
|
||||||
|
hour: "2"
|
||||||
|
minute: "0"
|
||||||
|
user: postgres
|
||||||
|
job: "pgbackrest --stanza={{ patroni_scope }} --type=diff backup >> /var/log/pgbackrest/backup.log 2>&1"
|
||||||
|
|
||||||
|
- name: Setup weekly full backup
|
||||||
|
cron:
|
||||||
|
name: "Weekly full PostgreSQL backup"
|
||||||
|
weekday: "0"
|
||||||
|
hour: "3"
|
||||||
|
minute: "0"
|
||||||
|
user: postgres
|
||||||
|
job: "pgbackrest --stanza={{ patroni_scope }} --type=full backup >> /var/log/pgbackrest/backup.log 2>&1"
|
||||||
|
|
||||||
|
# =========================================================================
|
||||||
|
# MONITORING
|
||||||
|
# =========================================================================
|
||||||
|
- name: Install postgres_exporter
|
||||||
|
shell: |
|
||||||
|
curl -sL https://github.com/prometheus-community/postgres_exporter/releases/download/v0.15.0/postgres_exporter-0.15.0.linux-amd64.tar.gz | tar xz
|
||||||
|
mv postgres_exporter-0.15.0.linux-amd64/postgres_exporter /usr/local/bin/
|
||||||
|
rm -rf postgres_exporter-0.15.0.linux-amd64
|
||||||
|
args:
|
||||||
|
creates: /usr/local/bin/postgres_exporter
|
||||||
|
|
||||||
|
- name: Create postgres_exporter systemd service
|
||||||
|
copy:
|
||||||
|
dest: /etc/systemd/system/postgres_exporter.service
|
||||||
|
content: |
|
||||||
|
[Unit]
|
||||||
|
Description=Prometheus PostgreSQL Exporter
|
||||||
|
After=network.target postgresql.service
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=simple
|
||||||
|
User=postgres
|
||||||
|
Environment="DATA_SOURCE_NAME=postgresql://{{ postgres_user }}:{{ postgres_password }}@localhost:5432/{{ postgres_db }}?sslmode=disable"
|
||||||
|
ExecStart=/usr/local/bin/postgres_exporter --web.listen-address=:9187
|
||||||
|
Restart=on-failure
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
|
notify:
|
||||||
|
- reload systemd
|
||||||
|
- restart postgres_exporter
|
||||||
|
|
||||||
|
- name: Enable postgres_exporter
|
||||||
|
service:
|
||||||
|
name: postgres_exporter
|
||||||
|
enabled: yes
|
||||||
|
state: started
|
||||||
|
|
||||||
|
# =========================================================================
|
||||||
|
# VERIFICATION
|
||||||
|
# =========================================================================
|
||||||
|
- name: Show PostgreSQL HA status
|
||||||
|
debug:
|
||||||
|
msg: |
|
||||||
|
PostgreSQL HA Setup Complete!
|
||||||
|
|
||||||
|
Components:
|
||||||
|
- Patroni: Cluster management
|
||||||
|
- PgBouncer: Connection pooling (port {{ pgbouncer_port }})
|
||||||
|
- pgBackRest: Backups
|
||||||
|
- postgres_exporter: Metrics (port 9187)
|
||||||
|
|
||||||
|
Connection strings:
|
||||||
|
- Direct: postgresql://{{ postgres_user }}@{{ ansible_host }}:5432/{{ postgres_db }}
|
||||||
|
- Pooled: postgresql://{{ postgres_user }}@{{ ansible_host }}:{{ pgbouncer_port }}/{{ postgres_db }}
|
||||||
|
|
||||||
|
handlers:
|
||||||
|
- name: reload systemd
|
||||||
|
systemd:
|
||||||
|
daemon_reload: yes
|
||||||
|
|
||||||
|
- name: restart patroni
|
||||||
|
service:
|
||||||
|
name: patroni
|
||||||
|
state: restarted
|
||||||
|
|
||||||
|
- name: restart pgbouncer
|
||||||
|
service:
|
||||||
|
name: pgbouncer
|
||||||
|
state: restarted
|
||||||
|
|
||||||
|
- name: restart postgres_exporter
|
||||||
|
service:
|
||||||
|
name: postgres_exporter
|
||||||
|
state: restarted
|
||||||
120
infrastructure/ansible/templates/patroni.yml.j2
Normal file
120
infrastructure/ansible/templates/patroni.yml.j2
Normal file
@@ -0,0 +1,120 @@
|
|||||||
|
# Patroni Configuration for {{ inventory_hostname }}
|
||||||
|
# Generated by Ansible
|
||||||
|
|
||||||
|
scope: {{ patroni_scope }}
|
||||||
|
namespace: {{ patroni_namespace }}
|
||||||
|
name: {{ inventory_hostname }}
|
||||||
|
|
||||||
|
restapi:
|
||||||
|
listen: 0.0.0.0:8008
|
||||||
|
connect_address: {{ ansible_host }}:8008
|
||||||
|
|
||||||
|
consul:
|
||||||
|
host: {{ consul_host }}:8500
|
||||||
|
register_service: true
|
||||||
|
|
||||||
|
bootstrap:
|
||||||
|
dcs:
|
||||||
|
ttl: 30
|
||||||
|
loop_wait: 10
|
||||||
|
retry_timeout: 10
|
||||||
|
maximum_lag_on_failover: 1048576
|
||||||
|
postgresql:
|
||||||
|
use_pg_rewind: true
|
||||||
|
use_slots: true
|
||||||
|
parameters:
|
||||||
|
# Performance
|
||||||
|
max_connections: 200
|
||||||
|
shared_buffers: 256MB
|
||||||
|
effective_cache_size: 768MB
|
||||||
|
maintenance_work_mem: 64MB
|
||||||
|
checkpoint_completion_target: 0.9
|
||||||
|
wal_buffers: 16MB
|
||||||
|
default_statistics_target: 100
|
||||||
|
random_page_cost: 1.1
|
||||||
|
effective_io_concurrency: 200
|
||||||
|
work_mem: 2621kB
|
||||||
|
huge_pages: off
|
||||||
|
min_wal_size: 1GB
|
||||||
|
max_wal_size: 4GB
|
||||||
|
max_worker_processes: 4
|
||||||
|
max_parallel_workers_per_gather: 2
|
||||||
|
max_parallel_workers: 4
|
||||||
|
max_parallel_maintenance_workers: 2
|
||||||
|
|
||||||
|
# Replication
|
||||||
|
wal_level: replica
|
||||||
|
hot_standby: "on"
|
||||||
|
max_wal_senders: 10
|
||||||
|
max_replication_slots: 10
|
||||||
|
hot_standby_feedback: "on"
|
||||||
|
|
||||||
|
# Logging
|
||||||
|
log_destination: 'stderr'
|
||||||
|
logging_collector: 'on'
|
||||||
|
log_directory: 'log'
|
||||||
|
log_filename: 'postgresql-%Y-%m-%d_%H%M%S.log'
|
||||||
|
log_rotation_age: '1d'
|
||||||
|
log_rotation_size: '100MB'
|
||||||
|
log_min_duration_statement: 1000
|
||||||
|
log_checkpoints: 'on'
|
||||||
|
log_connections: 'on'
|
||||||
|
log_disconnections: 'on'
|
||||||
|
log_lock_waits: 'on'
|
||||||
|
|
||||||
|
# Archive (for pgBackRest)
|
||||||
|
archive_mode: "on"
|
||||||
|
archive_command: 'pgbackrest --stanza={{ patroni_scope }} archive-push %p'
|
||||||
|
|
||||||
|
initdb:
|
||||||
|
- encoding: UTF8
|
||||||
|
- data-checksums
|
||||||
|
|
||||||
|
pg_hba:
|
||||||
|
- host replication replicator 0.0.0.0/0 scram-sha-256
|
||||||
|
- host all all 0.0.0.0/0 scram-sha-256
|
||||||
|
|
||||||
|
users:
|
||||||
|
{{ postgres_user }}:
|
||||||
|
password: {{ postgres_password }}
|
||||||
|
options:
|
||||||
|
- createrole
|
||||||
|
- createdb
|
||||||
|
replicator:
|
||||||
|
password: {{ replicator_password | default('replicator_secret') }}
|
||||||
|
options:
|
||||||
|
- replication
|
||||||
|
|
||||||
|
postgresql:
|
||||||
|
listen: 0.0.0.0:5432
|
||||||
|
connect_address: {{ ansible_host }}:5432
|
||||||
|
data_dir: {{ postgres_data_dir }}
|
||||||
|
bin_dir: /usr/lib/postgresql/{{ postgres_version }}/bin
|
||||||
|
config_dir: {{ postgres_config_dir }}
|
||||||
|
pgpass: /var/lib/postgresql/.pgpass
|
||||||
|
|
||||||
|
authentication:
|
||||||
|
replication:
|
||||||
|
username: replicator
|
||||||
|
password: {{ replicator_password | default('replicator_secret') }}
|
||||||
|
superuser:
|
||||||
|
username: postgres
|
||||||
|
password: {{ postgres_superuser_password | default('postgres_secret') }}
|
||||||
|
rewind:
|
||||||
|
username: rewind
|
||||||
|
password: {{ rewind_password | default('rewind_secret') }}
|
||||||
|
|
||||||
|
parameters:
|
||||||
|
unix_socket_directories: '/var/run/postgresql'
|
||||||
|
|
||||||
|
pg_hba:
|
||||||
|
- local all all peer
|
||||||
|
- host all all 127.0.0.1/32 scram-sha-256
|
||||||
|
- host all all 0.0.0.0/0 scram-sha-256
|
||||||
|
- host replication replicator 0.0.0.0/0 scram-sha-256
|
||||||
|
|
||||||
|
tags:
|
||||||
|
nofailover: false
|
||||||
|
noloadbalance: false
|
||||||
|
clonefrom: false
|
||||||
|
nosync: false
|
||||||
40
infrastructure/ansible/templates/pgbackrest.conf.j2
Normal file
40
infrastructure/ansible/templates/pgbackrest.conf.j2
Normal file
@@ -0,0 +1,40 @@
|
|||||||
|
# pgBackRest Configuration for {{ inventory_hostname }}
|
||||||
|
# Generated by Ansible
|
||||||
|
|
||||||
|
[global]
|
||||||
|
# Repository
|
||||||
|
repo1-path={{ pgbackrest_repo_path }}
|
||||||
|
repo1-retention-full=2
|
||||||
|
repo1-retention-diff=7
|
||||||
|
|
||||||
|
# S3 (optional - uncomment for cloud backups)
|
||||||
|
# repo2-type=s3
|
||||||
|
# repo2-path=/backup
|
||||||
|
# repo2-s3-bucket={{ pgbackrest_s3_bucket }}
|
||||||
|
# repo2-s3-endpoint=s3.eu-central-1.amazonaws.com
|
||||||
|
# repo2-s3-region=eu-central-1
|
||||||
|
# repo2-s3-key={{ pgbackrest_s3_key | default('') }}
|
||||||
|
# repo2-s3-key-secret={{ pgbackrest_s3_secret | default('') }}
|
||||||
|
# repo2-retention-full=4
|
||||||
|
# repo2-retention-diff=14
|
||||||
|
|
||||||
|
# Compression
|
||||||
|
compress-type=zst
|
||||||
|
compress-level=3
|
||||||
|
|
||||||
|
# Parallel
|
||||||
|
process-max=4
|
||||||
|
|
||||||
|
# Logging
|
||||||
|
log-level-console=info
|
||||||
|
log-level-file=detail
|
||||||
|
log-path=/var/log/pgbackrest
|
||||||
|
|
||||||
|
# Archive
|
||||||
|
archive-async=y
|
||||||
|
archive-push-queue-max=4GB
|
||||||
|
|
||||||
|
[{{ patroni_scope }}]
|
||||||
|
pg1-path={{ postgres_data_dir }}
|
||||||
|
pg1-port=5432
|
||||||
|
pg1-user=postgres
|
||||||
44
infrastructure/ansible/templates/pgbouncer.ini.j2
Normal file
44
infrastructure/ansible/templates/pgbouncer.ini.j2
Normal file
@@ -0,0 +1,44 @@
|
|||||||
|
# PgBouncer Configuration for {{ inventory_hostname }}
|
||||||
|
# Generated by Ansible
|
||||||
|
|
||||||
|
[databases]
|
||||||
|
{{ postgres_db }} = host=127.0.0.1 port=5432 dbname={{ postgres_db }}
|
||||||
|
* = host=127.0.0.1 port=5432
|
||||||
|
|
||||||
|
[pgbouncer]
|
||||||
|
listen_addr = 0.0.0.0
|
||||||
|
listen_port = {{ pgbouncer_port }}
|
||||||
|
unix_socket_dir = /var/run/postgresql
|
||||||
|
|
||||||
|
auth_type = scram-sha-256
|
||||||
|
auth_file = /etc/pgbouncer/userlist.txt
|
||||||
|
|
||||||
|
# Pool settings
|
||||||
|
pool_mode = transaction
|
||||||
|
max_client_conn = {{ pgbouncer_max_client_conn }}
|
||||||
|
default_pool_size = {{ pgbouncer_default_pool_size }}
|
||||||
|
min_pool_size = 10
|
||||||
|
reserve_pool_size = 5
|
||||||
|
reserve_pool_timeout = 3
|
||||||
|
|
||||||
|
# Timeouts
|
||||||
|
server_connect_timeout = 15
|
||||||
|
server_idle_timeout = 600
|
||||||
|
server_lifetime = 3600
|
||||||
|
client_idle_timeout = 0
|
||||||
|
client_login_timeout = 60
|
||||||
|
query_timeout = 0
|
||||||
|
query_wait_timeout = 120
|
||||||
|
|
||||||
|
# Logging
|
||||||
|
log_connections = 1
|
||||||
|
log_disconnections = 1
|
||||||
|
log_pooler_errors = 1
|
||||||
|
stats_period = 60
|
||||||
|
|
||||||
|
# Admin
|
||||||
|
admin_users = pgbouncer,{{ postgres_user }}
|
||||||
|
stats_users = pgbouncer,{{ postgres_user }}
|
||||||
|
|
||||||
|
# Security
|
||||||
|
ignore_startup_parameters = extra_float_digits
|
||||||
458
infrastructure/database/agent-memory-schema.sql
Normal file
458
infrastructure/database/agent-memory-schema.sql
Normal file
@@ -0,0 +1,458 @@
|
|||||||
|
-- ============================================================================
|
||||||
|
-- DAARION Agent Memory System - PostgreSQL Schema
|
||||||
|
-- Version: 1.0.0
|
||||||
|
-- Date: 2026-01-10
|
||||||
|
--
|
||||||
|
-- Трирівнева пам'ять агентів:
|
||||||
|
-- 1. Short-term: conversation_events (робочий буфер)
|
||||||
|
-- 2. Mid-term: thread_summaries (сесійна/тематична)
|
||||||
|
-- 3. Long-term: long_term_memory_items (персональна/проектна)
|
||||||
|
-- ============================================================================
|
||||||
|
|
||||||
|
-- Extensions
|
||||||
|
CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
|
||||||
|
CREATE EXTENSION IF NOT EXISTS "pgcrypto";
|
||||||
|
|
||||||
|
-- ============================================================================
|
||||||
|
-- CORE ENTITIES
|
||||||
|
-- ============================================================================
|
||||||
|
|
||||||
|
-- Organizations (top-level tenant)
|
||||||
|
CREATE TABLE IF NOT EXISTS organizations (
|
||||||
|
org_id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||||
|
name VARCHAR(255) NOT NULL,
|
||||||
|
settings JSONB DEFAULT '{}',
|
||||||
|
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||||
|
updated_at TIMESTAMPTZ DEFAULT NOW()
|
||||||
|
);
|
||||||
|
|
||||||
|
-- Workspaces (projects within org)
|
||||||
|
CREATE TABLE IF NOT EXISTS workspaces (
|
||||||
|
workspace_id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||||
|
org_id UUID NOT NULL REFERENCES organizations(org_id) ON DELETE CASCADE,
|
||||||
|
name VARCHAR(255) NOT NULL,
|
||||||
|
description TEXT,
|
||||||
|
settings JSONB DEFAULT '{}',
|
||||||
|
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||||
|
updated_at TIMESTAMPTZ DEFAULT NOW()
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX idx_workspaces_org ON workspaces(org_id);
|
||||||
|
|
||||||
|
-- Users
|
||||||
|
CREATE TABLE IF NOT EXISTS users (
|
||||||
|
user_id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||||
|
org_id UUID NOT NULL REFERENCES organizations(org_id) ON DELETE CASCADE,
|
||||||
|
external_id VARCHAR(255), -- for SSO/OAuth mapping
|
||||||
|
email VARCHAR(255),
|
||||||
|
display_name VARCHAR(255),
|
||||||
|
preferences JSONB DEFAULT '{}',
|
||||||
|
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||||
|
updated_at TIMESTAMPTZ DEFAULT NOW(),
|
||||||
|
UNIQUE(org_id, external_id)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX idx_users_org ON users(org_id);
|
||||||
|
CREATE INDEX idx_users_email ON users(email);
|
||||||
|
|
||||||
|
-- Agents
|
||||||
|
CREATE TABLE IF NOT EXISTS agents (
|
||||||
|
agent_id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||||
|
org_id UUID NOT NULL REFERENCES organizations(org_id) ON DELETE CASCADE,
|
||||||
|
name VARCHAR(255) NOT NULL,
|
||||||
|
type VARCHAR(50) NOT NULL, -- 'assistant', 'specialist', 'coordinator'
|
||||||
|
model VARCHAR(100), -- 'claude-3-opus', 'gpt-4', etc.
|
||||||
|
system_prompt TEXT,
|
||||||
|
capabilities JSONB DEFAULT '[]', -- ['code', 'search', 'memory', 'tools']
|
||||||
|
settings JSONB DEFAULT '{}',
|
||||||
|
is_active BOOLEAN DEFAULT true,
|
||||||
|
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||||
|
updated_at TIMESTAMPTZ DEFAULT NOW()
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX idx_agents_org ON agents(org_id);
|
||||||
|
CREATE INDEX idx_agents_type ON agents(type);
|
||||||
|
|
||||||
|
-- ============================================================================
|
||||||
|
-- CONVERSATION LAYER (Short-term Memory)
|
||||||
|
-- ============================================================================
|
||||||
|
|
||||||
|
-- Conversation threads
|
||||||
|
CREATE TABLE IF NOT EXISTS conversation_threads (
|
||||||
|
thread_id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||||
|
org_id UUID NOT NULL REFERENCES organizations(org_id) ON DELETE CASCADE,
|
||||||
|
workspace_id UUID REFERENCES workspaces(workspace_id) ON DELETE SET NULL,
|
||||||
|
user_id UUID NOT NULL REFERENCES users(user_id) ON DELETE CASCADE,
|
||||||
|
agent_id UUID REFERENCES agents(agent_id) ON DELETE SET NULL,
|
||||||
|
|
||||||
|
title VARCHAR(500),
|
||||||
|
status VARCHAR(50) DEFAULT 'active', -- 'active', 'archived', 'completed'
|
||||||
|
|
||||||
|
-- Metadata
|
||||||
|
tags JSONB DEFAULT '[]',
|
||||||
|
metadata JSONB DEFAULT '{}',
|
||||||
|
|
||||||
|
-- Stats
|
||||||
|
message_count INTEGER DEFAULT 0,
|
||||||
|
total_tokens INTEGER DEFAULT 0,
|
||||||
|
|
||||||
|
-- Timestamps
|
||||||
|
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||||
|
last_activity_at TIMESTAMPTZ DEFAULT NOW(),
|
||||||
|
archived_at TIMESTAMPTZ
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX idx_threads_org ON conversation_threads(org_id);
|
||||||
|
CREATE INDEX idx_threads_workspace ON conversation_threads(workspace_id);
|
||||||
|
CREATE INDEX idx_threads_user ON conversation_threads(user_id);
|
||||||
|
CREATE INDEX idx_threads_agent ON conversation_threads(agent_id);
|
||||||
|
CREATE INDEX idx_threads_status ON conversation_threads(status);
|
||||||
|
CREATE INDEX idx_threads_last_activity ON conversation_threads(last_activity_at DESC);
|
||||||
|
|
||||||
|
-- Conversation events (Event Log - source of truth)
|
||||||
|
CREATE TABLE IF NOT EXISTS conversation_events (
|
||||||
|
event_id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||||
|
thread_id UUID NOT NULL REFERENCES conversation_threads(thread_id) ON DELETE CASCADE,
|
||||||
|
|
||||||
|
-- Event type
|
||||||
|
event_type VARCHAR(50) NOT NULL, -- 'message', 'tool_call', 'tool_result', 'decision', 'summary', 'memory_write', 'memory_retract', 'error'
|
||||||
|
|
||||||
|
-- For messages
|
||||||
|
role VARCHAR(20), -- 'user', 'assistant', 'system', 'tool'
|
||||||
|
content TEXT,
|
||||||
|
|
||||||
|
-- For tool calls
|
||||||
|
tool_name VARCHAR(100),
|
||||||
|
tool_input JSONB,
|
||||||
|
tool_output JSONB,
|
||||||
|
|
||||||
|
-- Structured payload (flexible)
|
||||||
|
payload JSONB DEFAULT '{}',
|
||||||
|
|
||||||
|
-- Token tracking
|
||||||
|
token_count INTEGER,
|
||||||
|
|
||||||
|
-- Metadata
|
||||||
|
model_used VARCHAR(100),
|
||||||
|
latency_ms INTEGER,
|
||||||
|
metadata JSONB DEFAULT '{}',
|
||||||
|
|
||||||
|
-- Ordering
|
||||||
|
sequence_num SERIAL,
|
||||||
|
created_at TIMESTAMPTZ DEFAULT NOW()
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX idx_events_thread ON conversation_events(thread_id);
|
||||||
|
CREATE INDEX idx_events_type ON conversation_events(event_type);
|
||||||
|
CREATE INDEX idx_events_created ON conversation_events(created_at DESC);
|
||||||
|
CREATE INDEX idx_events_thread_seq ON conversation_events(thread_id, sequence_num);
|
||||||
|
|
||||||
|
-- ============================================================================
|
||||||
|
-- SUMMARY LAYER (Mid-term Memory)
|
||||||
|
-- ============================================================================
|
||||||
|
|
||||||
|
-- Thread summaries (rolling compression)
|
||||||
|
CREATE TABLE IF NOT EXISTS thread_summaries (
|
||||||
|
summary_id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||||
|
thread_id UUID NOT NULL REFERENCES conversation_threads(thread_id) ON DELETE CASCADE,
|
||||||
|
|
||||||
|
-- Version tracking
|
||||||
|
version INTEGER NOT NULL DEFAULT 1,
|
||||||
|
|
||||||
|
-- Summary content
|
||||||
|
summary_text TEXT NOT NULL,
|
||||||
|
|
||||||
|
-- Structured state
|
||||||
|
state JSONB DEFAULT '{}', -- {goals: [], decisions: [], open_questions: [], next_steps: [], key_facts: []}
|
||||||
|
|
||||||
|
-- Coverage
|
||||||
|
events_from_seq INTEGER, -- first event sequence included
|
||||||
|
events_to_seq INTEGER, -- last event sequence included
|
||||||
|
events_count INTEGER,
|
||||||
|
|
||||||
|
-- Token info
|
||||||
|
original_tokens INTEGER, -- tokens before compression
|
||||||
|
summary_tokens INTEGER, -- tokens after compression
|
||||||
|
compression_ratio FLOAT,
|
||||||
|
|
||||||
|
-- Timestamps
|
||||||
|
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||||
|
|
||||||
|
UNIQUE(thread_id, version)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX idx_summaries_thread ON thread_summaries(thread_id);
|
||||||
|
CREATE INDEX idx_summaries_thread_version ON thread_summaries(thread_id, version DESC);
|
||||||
|
|
||||||
|
-- ============================================================================
|
||||||
|
-- LONG-TERM MEMORY LAYER
|
||||||
|
-- ============================================================================
|
||||||
|
|
||||||
|
-- Memory categories enum
|
||||||
|
CREATE TYPE memory_category AS ENUM (
|
||||||
|
'preference', -- user likes/dislikes
|
||||||
|
'identity', -- who the user is
|
||||||
|
'constraint', -- limitations, rules
|
||||||
|
'project_fact', -- project-specific knowledge
|
||||||
|
'relationship', -- connections between entities
|
||||||
|
'skill', -- user capabilities
|
||||||
|
'goal', -- user objectives
|
||||||
|
'context', -- situational info
|
||||||
|
'feedback' -- user corrections/confirmations
|
||||||
|
);
|
||||||
|
|
||||||
|
-- Retention policy enum
|
||||||
|
CREATE TYPE retention_policy AS ENUM (
|
||||||
|
'permanent', -- keep until explicitly deleted
|
||||||
|
'session', -- delete after session
|
||||||
|
'ttl_days', -- delete after N days
|
||||||
|
'until_revoked' -- keep until user revokes
|
||||||
|
);
|
||||||
|
|
||||||
|
-- Long-term memory items
|
||||||
|
CREATE TABLE IF NOT EXISTS long_term_memory_items (
|
||||||
|
memory_id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||||
|
|
||||||
|
-- Scope (all nullable for flexibility)
|
||||||
|
org_id UUID REFERENCES organizations(org_id) ON DELETE CASCADE,
|
||||||
|
workspace_id UUID REFERENCES workspaces(workspace_id) ON DELETE SET NULL,
|
||||||
|
user_id UUID REFERENCES users(user_id) ON DELETE CASCADE,
|
||||||
|
agent_id UUID REFERENCES agents(agent_id) ON DELETE SET NULL, -- null = global for user
|
||||||
|
|
||||||
|
-- Content
|
||||||
|
category memory_category NOT NULL,
|
||||||
|
fact_text TEXT NOT NULL, -- atomic statement
|
||||||
|
fact_embedding_id VARCHAR(100), -- reference to Qdrant point ID
|
||||||
|
|
||||||
|
-- Confidence & validation
|
||||||
|
confidence FLOAT DEFAULT 0.8 CHECK (confidence >= 0 AND confidence <= 1),
|
||||||
|
is_verified BOOLEAN DEFAULT false,
|
||||||
|
verification_count INTEGER DEFAULT 0,
|
||||||
|
|
||||||
|
-- Source tracking
|
||||||
|
source_event_id UUID REFERENCES conversation_events(event_id) ON DELETE SET NULL,
|
||||||
|
source_thread_id UUID REFERENCES conversation_threads(thread_id) ON DELETE SET NULL,
|
||||||
|
extraction_method VARCHAR(50), -- 'explicit', 'inferred', 'confirmed', 'imported'
|
||||||
|
|
||||||
|
-- Lifecycle
|
||||||
|
valid_from TIMESTAMPTZ DEFAULT NOW(),
|
||||||
|
valid_to TIMESTAMPTZ, -- null = currently valid
|
||||||
|
last_confirmed_at TIMESTAMPTZ,
|
||||||
|
last_used_at TIMESTAMPTZ,
|
||||||
|
use_count INTEGER DEFAULT 0,
|
||||||
|
|
||||||
|
-- Privacy & retention
|
||||||
|
is_sensitive BOOLEAN DEFAULT false,
|
||||||
|
retention retention_policy DEFAULT 'until_revoked',
|
||||||
|
ttl_days INTEGER, -- if retention = 'ttl_days'
|
||||||
|
|
||||||
|
-- Metadata
|
||||||
|
tags JSONB DEFAULT '[]',
|
||||||
|
metadata JSONB DEFAULT '{}',
|
||||||
|
|
||||||
|
-- Timestamps
|
||||||
|
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||||
|
updated_at TIMESTAMPTZ DEFAULT NOW()
|
||||||
|
);
|
||||||
|
|
||||||
|
-- Indexes for memory retrieval
|
||||||
|
CREATE INDEX idx_memory_org ON long_term_memory_items(org_id);
|
||||||
|
CREATE INDEX idx_memory_workspace ON long_term_memory_items(workspace_id);
|
||||||
|
CREATE INDEX idx_memory_user ON long_term_memory_items(user_id);
|
||||||
|
CREATE INDEX idx_memory_agent ON long_term_memory_items(agent_id);
|
||||||
|
CREATE INDEX idx_memory_category ON long_term_memory_items(category);
|
||||||
|
CREATE INDEX idx_memory_user_agent ON long_term_memory_items(user_id, agent_id);
|
||||||
|
CREATE INDEX idx_memory_valid ON long_term_memory_items(valid_from, valid_to);
|
||||||
|
CREATE INDEX idx_memory_confidence ON long_term_memory_items(confidence DESC);
|
||||||
|
CREATE INDEX idx_memory_created ON long_term_memory_items(created_at DESC);
|
||||||
|
|
||||||
|
-- GIN index for tags search
|
||||||
|
CREATE INDEX idx_memory_tags ON long_term_memory_items USING GIN (tags);
|
||||||
|
|
||||||
|
-- Memory feedback (user corrections)
|
||||||
|
CREATE TABLE IF NOT EXISTS memory_feedback (
|
||||||
|
feedback_id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||||
|
memory_id UUID NOT NULL REFERENCES long_term_memory_items(memory_id) ON DELETE CASCADE,
|
||||||
|
user_id UUID NOT NULL REFERENCES users(user_id) ON DELETE CASCADE,
|
||||||
|
|
||||||
|
action VARCHAR(20) NOT NULL, -- 'confirm', 'reject', 'edit', 'delete'
|
||||||
|
old_value TEXT,
|
||||||
|
new_value TEXT,
|
||||||
|
reason TEXT,
|
||||||
|
|
||||||
|
created_at TIMESTAMPTZ DEFAULT NOW()
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX idx_feedback_memory ON memory_feedback(memory_id);
|
||||||
|
CREATE INDEX idx_feedback_user ON memory_feedback(user_id);
|
||||||
|
|
||||||
|
-- ============================================================================
|
||||||
|
-- HELPER VIEWS
|
||||||
|
-- ============================================================================
|
||||||
|
|
||||||
|
-- Active memories for a user (across all agents)
|
||||||
|
CREATE OR REPLACE VIEW v_active_user_memories AS
|
||||||
|
SELECT
|
||||||
|
m.*,
|
||||||
|
a.name as agent_name
|
||||||
|
FROM long_term_memory_items m
|
||||||
|
LEFT JOIN agents a ON m.agent_id = a.agent_id
|
||||||
|
WHERE m.valid_to IS NULL
|
||||||
|
AND m.confidence >= 0.5
|
||||||
|
ORDER BY m.confidence DESC, m.last_used_at DESC NULLS LAST;
|
||||||
|
|
||||||
|
-- Recent conversations with summaries
|
||||||
|
CREATE OR REPLACE VIEW v_recent_conversations AS
|
||||||
|
SELECT
|
||||||
|
t.thread_id,
|
||||||
|
t.title,
|
||||||
|
t.user_id,
|
||||||
|
t.agent_id,
|
||||||
|
t.message_count,
|
||||||
|
t.last_activity_at,
|
||||||
|
s.summary_text,
|
||||||
|
s.state
|
||||||
|
FROM conversation_threads t
|
||||||
|
LEFT JOIN LATERAL (
|
||||||
|
SELECT summary_text, state
|
||||||
|
FROM thread_summaries
|
||||||
|
WHERE thread_id = t.thread_id
|
||||||
|
ORDER BY version DESC
|
||||||
|
LIMIT 1
|
||||||
|
) s ON true
|
||||||
|
WHERE t.status = 'active'
|
||||||
|
ORDER BY t.last_activity_at DESC;
|
||||||
|
|
||||||
|
-- ============================================================================
|
||||||
|
-- FUNCTIONS
|
||||||
|
-- ============================================================================
|
||||||
|
|
||||||
|
-- Update thread stats after new event
|
||||||
|
CREATE OR REPLACE FUNCTION update_thread_stats()
|
||||||
|
RETURNS TRIGGER AS $$
|
||||||
|
BEGIN
|
||||||
|
UPDATE conversation_threads
|
||||||
|
SET
|
||||||
|
message_count = message_count + CASE WHEN NEW.event_type = 'message' THEN 1 ELSE 0 END,
|
||||||
|
total_tokens = total_tokens + COALESCE(NEW.token_count, 0),
|
||||||
|
last_activity_at = NOW()
|
||||||
|
WHERE thread_id = NEW.thread_id;
|
||||||
|
RETURN NEW;
|
||||||
|
END;
|
||||||
|
$$ LANGUAGE plpgsql;
|
||||||
|
|
||||||
|
CREATE TRIGGER trg_update_thread_stats
|
||||||
|
AFTER INSERT ON conversation_events
|
||||||
|
FOR EACH ROW EXECUTE FUNCTION update_thread_stats();
|
||||||
|
|
||||||
|
-- Update memory usage stats
|
||||||
|
CREATE OR REPLACE FUNCTION update_memory_usage()
|
||||||
|
RETURNS TRIGGER AS $$
|
||||||
|
BEGIN
|
||||||
|
UPDATE long_term_memory_items
|
||||||
|
SET
|
||||||
|
use_count = use_count + 1,
|
||||||
|
last_used_at = NOW()
|
||||||
|
WHERE memory_id = NEW.memory_id;
|
||||||
|
RETURN NEW;
|
||||||
|
END;
|
||||||
|
$$ LANGUAGE plpgsql;
|
||||||
|
|
||||||
|
-- Auto-update updated_at
|
||||||
|
CREATE OR REPLACE FUNCTION update_updated_at()
|
||||||
|
RETURNS TRIGGER AS $$
|
||||||
|
BEGIN
|
||||||
|
NEW.updated_at = NOW();
|
||||||
|
RETURN NEW;
|
||||||
|
END;
|
||||||
|
$$ LANGUAGE plpgsql;
|
||||||
|
|
||||||
|
CREATE TRIGGER trg_organizations_updated
|
||||||
|
BEFORE UPDATE ON organizations
|
||||||
|
FOR EACH ROW EXECUTE FUNCTION update_updated_at();
|
||||||
|
|
||||||
|
CREATE TRIGGER trg_workspaces_updated
|
||||||
|
BEFORE UPDATE ON workspaces
|
||||||
|
FOR EACH ROW EXECUTE FUNCTION update_updated_at();
|
||||||
|
|
||||||
|
CREATE TRIGGER trg_users_updated
|
||||||
|
BEFORE UPDATE ON users
|
||||||
|
FOR EACH ROW EXECUTE FUNCTION update_updated_at();
|
||||||
|
|
||||||
|
CREATE TRIGGER trg_agents_updated
|
||||||
|
BEFORE UPDATE ON agents
|
||||||
|
FOR EACH ROW EXECUTE FUNCTION update_updated_at();
|
||||||
|
|
||||||
|
CREATE TRIGGER trg_memory_updated
|
||||||
|
BEFORE UPDATE ON long_term_memory_items
|
||||||
|
FOR EACH ROW EXECUTE FUNCTION update_updated_at();
|
||||||
|
|
||||||
|
-- ============================================================================
|
||||||
|
-- INITIAL DATA
|
||||||
|
-- ============================================================================
|
||||||
|
|
||||||
|
-- Default organization
|
||||||
|
INSERT INTO organizations (org_id, name, settings)
|
||||||
|
VALUES (
|
||||||
|
'a0000000-0000-0000-0000-000000000001',
|
||||||
|
'DAARION',
|
||||||
|
'{"tier": "enterprise", "features": ["memory", "multi-agent", "knowledge-base"]}'
|
||||||
|
) ON CONFLICT DO NOTHING;
|
||||||
|
|
||||||
|
-- Default workspace
|
||||||
|
INSERT INTO workspaces (workspace_id, org_id, name, description)
|
||||||
|
VALUES (
|
||||||
|
'b0000000-0000-0000-0000-000000000001',
|
||||||
|
'a0000000-0000-0000-0000-000000000001',
|
||||||
|
'MicroDAO',
|
||||||
|
'Main development workspace for DAARION project'
|
||||||
|
) ON CONFLICT DO NOTHING;
|
||||||
|
|
||||||
|
-- Default user (Ivan)
|
||||||
|
INSERT INTO users (user_id, org_id, external_id, display_name, preferences)
|
||||||
|
VALUES (
|
||||||
|
'c0000000-0000-0000-0000-000000000001',
|
||||||
|
'a0000000-0000-0000-0000-000000000001',
|
||||||
|
'ivan',
|
||||||
|
'Ivan Tytar',
|
||||||
|
'{"language": "uk", "timezone": "Europe/Kyiv"}'
|
||||||
|
) ON CONFLICT DO NOTHING;
|
||||||
|
|
||||||
|
-- Default agents
|
||||||
|
INSERT INTO agents (agent_id, org_id, name, type, model, capabilities) VALUES
|
||||||
|
(
|
||||||
|
'd0000000-0000-0000-0000-000000000001',
|
||||||
|
'a0000000-0000-0000-0000-000000000001',
|
||||||
|
'Claude Assistant',
|
||||||
|
'assistant',
|
||||||
|
'claude-3-opus',
|
||||||
|
'["code", "memory", "tools", "search"]'
|
||||||
|
),
|
||||||
|
(
|
||||||
|
'd0000000-0000-0000-0000-000000000002',
|
||||||
|
'a0000000-0000-0000-0000-000000000001',
|
||||||
|
'Code Specialist',
|
||||||
|
'specialist',
|
||||||
|
'claude-3-opus',
|
||||||
|
'["code", "review", "refactor"]'
|
||||||
|
),
|
||||||
|
(
|
||||||
|
'd0000000-0000-0000-0000-000000000003',
|
||||||
|
'a0000000-0000-0000-0000-000000000001',
|
||||||
|
'DevOps Agent',
|
||||||
|
'specialist',
|
||||||
|
'claude-3-opus',
|
||||||
|
'["infrastructure", "deployment", "monitoring"]'
|
||||||
|
)
|
||||||
|
ON CONFLICT DO NOTHING;
|
||||||
|
|
||||||
|
-- ============================================================================
|
||||||
|
-- COMMENTS
|
||||||
|
-- ============================================================================
|
||||||
|
|
||||||
|
COMMENT ON TABLE conversation_events IS 'Event log for all conversation activities - source of truth';
|
||||||
|
COMMENT ON TABLE thread_summaries IS 'Rolling summaries for context compression (mid-term memory)';
|
||||||
|
COMMENT ON TABLE long_term_memory_items IS 'Persistent facts about users/projects (long-term memory)';
|
||||||
|
COMMENT ON COLUMN long_term_memory_items.fact_text IS 'Atomic statement - one fact per row';
|
||||||
|
COMMENT ON COLUMN long_term_memory_items.confidence IS 'Confidence score 0-1, increases with confirmations';
|
||||||
|
COMMENT ON COLUMN long_term_memory_items.fact_embedding_id IS 'Reference to Qdrant vector point ID';
|
||||||
105
infrastructure/kubernetes/apps/memory-service/deployment.yaml
Normal file
105
infrastructure/kubernetes/apps/memory-service/deployment.yaml
Normal file
@@ -0,0 +1,105 @@
|
|||||||
|
---
|
||||||
|
# DAARION Memory Service
|
||||||
|
# Agent memory management with PostgreSQL + Qdrant + Cohere
|
||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: memory-service
|
||||||
|
namespace: daarion
|
||||||
|
labels:
|
||||||
|
app: memory-service
|
||||||
|
component: memory
|
||||||
|
spec:
|
||||||
|
replicas: 1
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app: memory-service
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app: memory-service
|
||||||
|
component: memory
|
||||||
|
spec:
|
||||||
|
nodeSelector:
|
||||||
|
node-role.kubernetes.io/control-plane: "true"
|
||||||
|
containers:
|
||||||
|
- name: memory-service
|
||||||
|
image: ghcr.io/ivantytar/memory-service:latest
|
||||||
|
imagePullPolicy: Always
|
||||||
|
ports:
|
||||||
|
- containerPort: 8000
|
||||||
|
name: http
|
||||||
|
envFrom:
|
||||||
|
- secretRef:
|
||||||
|
name: memory-service-secrets
|
||||||
|
env:
|
||||||
|
- name: MEMORY_DEBUG
|
||||||
|
value: "false"
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
memory: "256Mi"
|
||||||
|
cpu: "100m"
|
||||||
|
limits:
|
||||||
|
memory: "512Mi"
|
||||||
|
cpu: "500m"
|
||||||
|
livenessProbe:
|
||||||
|
httpGet:
|
||||||
|
path: /health
|
||||||
|
port: 8000
|
||||||
|
initialDelaySeconds: 30
|
||||||
|
periodSeconds: 10
|
||||||
|
readinessProbe:
|
||||||
|
httpGet:
|
||||||
|
path: /health
|
||||||
|
port: 8000
|
||||||
|
initialDelaySeconds: 5
|
||||||
|
periodSeconds: 5
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Service
|
||||||
|
metadata:
|
||||||
|
name: memory-service
|
||||||
|
namespace: daarion
|
||||||
|
spec:
|
||||||
|
selector:
|
||||||
|
app: memory-service
|
||||||
|
ports:
|
||||||
|
- name: http
|
||||||
|
port: 8000
|
||||||
|
targetPort: 8000
|
||||||
|
type: ClusterIP
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Service
|
||||||
|
metadata:
|
||||||
|
name: memory-service-external
|
||||||
|
namespace: daarion
|
||||||
|
spec:
|
||||||
|
selector:
|
||||||
|
app: memory-service
|
||||||
|
ports:
|
||||||
|
- name: http
|
||||||
|
port: 8000
|
||||||
|
targetPort: 8000
|
||||||
|
nodePort: 30800
|
||||||
|
type: NodePort
|
||||||
|
---
|
||||||
|
apiVersion: networking.k8s.io/v1
|
||||||
|
kind: Ingress
|
||||||
|
metadata:
|
||||||
|
name: memory-service-ingress
|
||||||
|
namespace: daarion
|
||||||
|
annotations:
|
||||||
|
traefik.ingress.kubernetes.io/router.entrypoints: web
|
||||||
|
spec:
|
||||||
|
rules:
|
||||||
|
- host: memory.daarion.local
|
||||||
|
http:
|
||||||
|
paths:
|
||||||
|
- path: /
|
||||||
|
pathType: Prefix
|
||||||
|
backend:
|
||||||
|
service:
|
||||||
|
name: memory-service
|
||||||
|
port:
|
||||||
|
number: 8000
|
||||||
133
infrastructure/kubernetes/apps/qdrant/deployment.yaml
Normal file
133
infrastructure/kubernetes/apps/qdrant/deployment.yaml
Normal file
@@ -0,0 +1,133 @@
|
|||||||
|
---
|
||||||
|
# DAARION Qdrant Vector Database
|
||||||
|
# For semantic search in agent memory system
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Namespace
|
||||||
|
metadata:
|
||||||
|
name: qdrant
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: PersistentVolumeClaim
|
||||||
|
metadata:
|
||||||
|
name: qdrant-storage
|
||||||
|
namespace: qdrant
|
||||||
|
spec:
|
||||||
|
accessModes:
|
||||||
|
- ReadWriteOnce
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
storage: 20Gi
|
||||||
|
storageClassName: local-path
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: ConfigMap
|
||||||
|
metadata:
|
||||||
|
name: qdrant-config
|
||||||
|
namespace: qdrant
|
||||||
|
data:
|
||||||
|
config.yaml: |
|
||||||
|
log_level: INFO
|
||||||
|
|
||||||
|
storage:
|
||||||
|
storage_path: /qdrant/storage
|
||||||
|
snapshots_path: /qdrant/snapshots
|
||||||
|
|
||||||
|
service:
|
||||||
|
http_port: 6333
|
||||||
|
grpc_port: 6334
|
||||||
|
|
||||||
|
cluster:
|
||||||
|
enabled: false
|
||||||
|
|
||||||
|
telemetry_disabled: true
|
||||||
|
---
|
||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: qdrant
|
||||||
|
namespace: qdrant
|
||||||
|
labels:
|
||||||
|
app: qdrant
|
||||||
|
spec:
|
||||||
|
replicas: 1
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app: qdrant
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app: qdrant
|
||||||
|
spec:
|
||||||
|
nodeSelector:
|
||||||
|
node-role.kubernetes.io/control-plane: "true"
|
||||||
|
containers:
|
||||||
|
- name: qdrant
|
||||||
|
image: qdrant/qdrant:v1.7.4
|
||||||
|
ports:
|
||||||
|
- containerPort: 6333
|
||||||
|
name: http
|
||||||
|
- containerPort: 6334
|
||||||
|
name: grpc
|
||||||
|
volumeMounts:
|
||||||
|
- name: storage
|
||||||
|
mountPath: /qdrant/storage
|
||||||
|
- name: config
|
||||||
|
mountPath: /qdrant/config
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
memory: "512Mi"
|
||||||
|
cpu: "250m"
|
||||||
|
limits:
|
||||||
|
memory: "2Gi"
|
||||||
|
cpu: "1"
|
||||||
|
livenessProbe:
|
||||||
|
httpGet:
|
||||||
|
path: /
|
||||||
|
port: 6333
|
||||||
|
initialDelaySeconds: 30
|
||||||
|
periodSeconds: 10
|
||||||
|
readinessProbe:
|
||||||
|
httpGet:
|
||||||
|
path: /readyz
|
||||||
|
port: 6333
|
||||||
|
initialDelaySeconds: 5
|
||||||
|
periodSeconds: 5
|
||||||
|
volumes:
|
||||||
|
- name: storage
|
||||||
|
persistentVolumeClaim:
|
||||||
|
claimName: qdrant-storage
|
||||||
|
- name: config
|
||||||
|
configMap:
|
||||||
|
name: qdrant-config
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Service
|
||||||
|
metadata:
|
||||||
|
name: qdrant
|
||||||
|
namespace: qdrant
|
||||||
|
spec:
|
||||||
|
selector:
|
||||||
|
app: qdrant
|
||||||
|
ports:
|
||||||
|
- name: http
|
||||||
|
port: 6333
|
||||||
|
targetPort: 6333
|
||||||
|
- name: grpc
|
||||||
|
port: 6334
|
||||||
|
targetPort: 6334
|
||||||
|
type: ClusterIP
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Service
|
||||||
|
metadata:
|
||||||
|
name: qdrant-external
|
||||||
|
namespace: qdrant
|
||||||
|
spec:
|
||||||
|
selector:
|
||||||
|
app: qdrant
|
||||||
|
ports:
|
||||||
|
- name: http
|
||||||
|
port: 6333
|
||||||
|
targetPort: 6333
|
||||||
|
nodePort: 30633
|
||||||
|
type: NodePort
|
||||||
@@ -1,24 +1,23 @@
|
|||||||
|
# DAARION Memory Service
|
||||||
FROM python:3.11-slim
|
FROM python:3.11-slim
|
||||||
|
|
||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
|
|
||||||
# Встановлюємо системні залежності
|
# Install dependencies
|
||||||
RUN apt-get update && apt-get install -y \
|
|
||||||
gcc \
|
|
||||||
postgresql-client \
|
|
||||||
curl \
|
|
||||||
&& rm -rf /var/lib/apt/lists/*
|
|
||||||
|
|
||||||
# Копіюємо requirements та встановлюємо залежності
|
|
||||||
COPY requirements.txt .
|
COPY requirements.txt .
|
||||||
RUN pip install --no-cache-dir -r requirements.txt
|
RUN pip install --no-cache-dir -r requirements.txt
|
||||||
|
|
||||||
# Копіюємо код
|
# Copy application
|
||||||
COPY . .
|
COPY app/ ./app/
|
||||||
|
|
||||||
# Відкриваємо порт
|
# Environment
|
||||||
|
ENV PYTHONPATH=/app
|
||||||
|
ENV PYTHONUNBUFFERED=1
|
||||||
|
|
||||||
|
# Health check
|
||||||
|
HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
|
||||||
|
CMD curl -f http://localhost:8000/health || exit 1
|
||||||
|
|
||||||
|
# Run
|
||||||
EXPOSE 8000
|
EXPOSE 8000
|
||||||
|
|
||||||
# Запускаємо додаток
|
|
||||||
CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"]
|
CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"]
|
||||||
|
|
||||||
|
|||||||
@@ -1 +1 @@
|
|||||||
# Memory Service for DAARION.city
|
"""DAARION Memory Service"""
|
||||||
|
|||||||
56
services/memory-service/app/config.py
Normal file
56
services/memory-service/app/config.py
Normal file
@@ -0,0 +1,56 @@
|
|||||||
|
"""
|
||||||
|
DAARION Memory Service Configuration
|
||||||
|
"""
|
||||||
|
from pydantic_settings import BaseSettings
|
||||||
|
from functools import lru_cache
|
||||||
|
|
||||||
|
|
||||||
|
class Settings(BaseSettings):
|
||||||
|
"""Application settings"""
|
||||||
|
|
||||||
|
# Service
|
||||||
|
service_name: str = "memory-service"
|
||||||
|
debug: bool = False
|
||||||
|
|
||||||
|
# PostgreSQL
|
||||||
|
postgres_host: str = "daarion-pooler.daarion"
|
||||||
|
postgres_port: int = 5432
|
||||||
|
postgres_user: str = "daarion"
|
||||||
|
postgres_password: str = "DaarionDB2026!"
|
||||||
|
postgres_db: str = "daarion_main"
|
||||||
|
|
||||||
|
@property
|
||||||
|
def postgres_url(self) -> str:
|
||||||
|
return f"postgresql+asyncpg://{self.postgres_user}:{self.postgres_password}@{self.postgres_host}:{self.postgres_port}/{self.postgres_db}"
|
||||||
|
|
||||||
|
# Qdrant
|
||||||
|
qdrant_host: str = "qdrant.qdrant"
|
||||||
|
qdrant_port: int = 6333
|
||||||
|
qdrant_collection_memories: str = "memories"
|
||||||
|
qdrant_collection_messages: str = "messages"
|
||||||
|
|
||||||
|
# Cohere (embeddings)
|
||||||
|
cohere_api_key: str = "nOdOXnuepLku2ipJWpe6acWgAsJCsDhMO0RnaEJB"
|
||||||
|
cohere_model: str = "embed-multilingual-v3.0" # 1024 dimensions
|
||||||
|
embedding_dimensions: int = 1024
|
||||||
|
|
||||||
|
# Memory settings
|
||||||
|
short_term_window_messages: int = 20
|
||||||
|
short_term_window_minutes: int = 60
|
||||||
|
summary_trigger_tokens: int = 4000
|
||||||
|
summary_target_tokens: int = 500
|
||||||
|
retrieval_top_k: int = 10
|
||||||
|
|
||||||
|
# Confidence thresholds
|
||||||
|
memory_min_confidence: float = 0.5
|
||||||
|
memory_confirm_boost: float = 0.1
|
||||||
|
memory_reject_penalty: float = 0.3
|
||||||
|
|
||||||
|
class Config:
|
||||||
|
env_prefix = "MEMORY_"
|
||||||
|
env_file = ".env"
|
||||||
|
|
||||||
|
|
||||||
|
@lru_cache()
|
||||||
|
def get_settings() -> Settings:
|
||||||
|
return Settings()
|
||||||
430
services/memory-service/app/database.py
Normal file
430
services/memory-service/app/database.py
Normal file
@@ -0,0 +1,430 @@
|
|||||||
|
"""
|
||||||
|
DAARION Memory Service - PostgreSQL Database Layer
|
||||||
|
"""
|
||||||
|
from typing import List, Optional, Dict, Any
|
||||||
|
from uuid import UUID, uuid4
|
||||||
|
from datetime import datetime
|
||||||
|
import structlog
|
||||||
|
import asyncpg
|
||||||
|
|
||||||
|
from .config import get_settings
|
||||||
|
from .models import EventType, MessageRole, MemoryCategory, RetentionPolicy, FeedbackAction
|
||||||
|
|
||||||
|
logger = structlog.get_logger()
|
||||||
|
settings = get_settings()
|
||||||
|
|
||||||
|
|
||||||
|
class Database:
|
||||||
|
"""PostgreSQL database operations"""
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
self.pool: Optional[asyncpg.Pool] = None
|
||||||
|
|
||||||
|
async def connect(self):
|
||||||
|
"""Connect to database"""
|
||||||
|
self.pool = await asyncpg.create_pool(
|
||||||
|
host=settings.postgres_host,
|
||||||
|
port=settings.postgres_port,
|
||||||
|
user=settings.postgres_user,
|
||||||
|
password=settings.postgres_password,
|
||||||
|
database=settings.postgres_db,
|
||||||
|
min_size=5,
|
||||||
|
max_size=20
|
||||||
|
)
|
||||||
|
logger.info("database_connected")
|
||||||
|
|
||||||
|
async def disconnect(self):
|
||||||
|
"""Disconnect from database"""
|
||||||
|
if self.pool:
|
||||||
|
await self.pool.close()
|
||||||
|
logger.info("database_disconnected")
|
||||||
|
|
||||||
|
# ========================================================================
|
||||||
|
# THREADS
|
||||||
|
# ========================================================================
|
||||||
|
|
||||||
|
async def create_thread(
|
||||||
|
self,
|
||||||
|
org_id: UUID,
|
||||||
|
user_id: UUID,
|
||||||
|
workspace_id: Optional[UUID] = None,
|
||||||
|
agent_id: Optional[UUID] = None,
|
||||||
|
title: Optional[str] = None,
|
||||||
|
tags: List[str] = [],
|
||||||
|
metadata: dict = {}
|
||||||
|
) -> Dict[str, Any]:
|
||||||
|
"""Create new conversation thread"""
|
||||||
|
thread_id = uuid4()
|
||||||
|
|
||||||
|
async with self.pool.acquire() as conn:
|
||||||
|
row = await conn.fetchrow("""
|
||||||
|
INSERT INTO conversation_threads
|
||||||
|
(thread_id, org_id, workspace_id, user_id, agent_id, title, tags, metadata)
|
||||||
|
VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
|
||||||
|
RETURNING *
|
||||||
|
""", thread_id, org_id, workspace_id, user_id, agent_id, title, tags, metadata)
|
||||||
|
|
||||||
|
logger.info("thread_created", thread_id=str(thread_id))
|
||||||
|
return dict(row)
|
||||||
|
|
||||||
|
async def get_thread(self, thread_id: UUID) -> Optional[Dict[str, Any]]:
|
||||||
|
"""Get thread by ID"""
|
||||||
|
async with self.pool.acquire() as conn:
|
||||||
|
row = await conn.fetchrow("""
|
||||||
|
SELECT * FROM conversation_threads WHERE thread_id = $1
|
||||||
|
""", thread_id)
|
||||||
|
return dict(row) if row else None
|
||||||
|
|
||||||
|
async def list_threads(
|
||||||
|
self,
|
||||||
|
org_id: UUID,
|
||||||
|
user_id: UUID,
|
||||||
|
workspace_id: Optional[UUID] = None,
|
||||||
|
agent_id: Optional[UUID] = None,
|
||||||
|
limit: int = 20
|
||||||
|
) -> List[Dict[str, Any]]:
|
||||||
|
"""List threads for user"""
|
||||||
|
async with self.pool.acquire() as conn:
|
||||||
|
query = """
|
||||||
|
SELECT * FROM conversation_threads
|
||||||
|
WHERE org_id = $1 AND user_id = $2 AND status = 'active'
|
||||||
|
"""
|
||||||
|
params = [org_id, user_id]
|
||||||
|
|
||||||
|
if workspace_id:
|
||||||
|
query += f" AND workspace_id = ${len(params) + 1}"
|
||||||
|
params.append(workspace_id)
|
||||||
|
if agent_id:
|
||||||
|
query += f" AND agent_id = ${len(params) + 1}"
|
||||||
|
params.append(agent_id)
|
||||||
|
|
||||||
|
query += f" ORDER BY last_activity_at DESC LIMIT ${len(params) + 1}"
|
||||||
|
params.append(limit)
|
||||||
|
|
||||||
|
rows = await conn.fetch(query, *params)
|
||||||
|
|
||||||
|
return [dict(row) for row in rows]
|
||||||
|
|
||||||
|
# ========================================================================
|
||||||
|
# EVENTS
|
||||||
|
# ========================================================================
|
||||||
|
|
||||||
|
async def add_event(
|
||||||
|
self,
|
||||||
|
thread_id: UUID,
|
||||||
|
event_type: EventType,
|
||||||
|
role: Optional[MessageRole] = None,
|
||||||
|
content: Optional[str] = None,
|
||||||
|
tool_name: Optional[str] = None,
|
||||||
|
tool_input: Optional[dict] = None,
|
||||||
|
tool_output: Optional[dict] = None,
|
||||||
|
payload: dict = {},
|
||||||
|
token_count: Optional[int] = None,
|
||||||
|
model_used: Optional[str] = None,
|
||||||
|
latency_ms: Optional[int] = None,
|
||||||
|
metadata: dict = {}
|
||||||
|
) -> Dict[str, Any]:
|
||||||
|
"""Add event to conversation"""
|
||||||
|
event_id = uuid4()
|
||||||
|
|
||||||
|
async with self.pool.acquire() as conn:
|
||||||
|
row = await conn.fetchrow("""
|
||||||
|
INSERT INTO conversation_events
|
||||||
|
(event_id, thread_id, event_type, role, content, tool_name,
|
||||||
|
tool_input, tool_output, payload, token_count, model_used,
|
||||||
|
latency_ms, metadata)
|
||||||
|
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)
|
||||||
|
RETURNING *
|
||||||
|
""", event_id, thread_id, event_type.value,
|
||||||
|
role.value if role else None, content, tool_name,
|
||||||
|
tool_input, tool_output, payload, token_count, model_used,
|
||||||
|
latency_ms, metadata)
|
||||||
|
|
||||||
|
logger.info("event_added", event_id=str(event_id), type=event_type.value)
|
||||||
|
return dict(row)
|
||||||
|
|
||||||
|
async def get_events(
|
||||||
|
self,
|
||||||
|
thread_id: UUID,
|
||||||
|
limit: int = 50,
|
||||||
|
offset: int = 0
|
||||||
|
) -> List[Dict[str, Any]]:
|
||||||
|
"""Get events for thread"""
|
||||||
|
async with self.pool.acquire() as conn:
|
||||||
|
rows = await conn.fetch("""
|
||||||
|
SELECT * FROM conversation_events
|
||||||
|
WHERE thread_id = $1
|
||||||
|
ORDER BY sequence_num DESC
|
||||||
|
LIMIT $2 OFFSET $3
|
||||||
|
""", thread_id, limit, offset)
|
||||||
|
|
||||||
|
return [dict(row) for row in rows]
|
||||||
|
|
||||||
|
async def get_events_for_summary(
|
||||||
|
self,
|
||||||
|
thread_id: UUID,
|
||||||
|
after_seq: Optional[int] = None
|
||||||
|
) -> List[Dict[str, Any]]:
|
||||||
|
"""Get events for summarization"""
|
||||||
|
async with self.pool.acquire() as conn:
|
||||||
|
if after_seq:
|
||||||
|
rows = await conn.fetch("""
|
||||||
|
SELECT * FROM conversation_events
|
||||||
|
WHERE thread_id = $1 AND sequence_num > $2
|
||||||
|
ORDER BY sequence_num ASC
|
||||||
|
""", thread_id, after_seq)
|
||||||
|
else:
|
||||||
|
rows = await conn.fetch("""
|
||||||
|
SELECT * FROM conversation_events
|
||||||
|
WHERE thread_id = $1
|
||||||
|
ORDER BY sequence_num ASC
|
||||||
|
""", thread_id)
|
||||||
|
|
||||||
|
return [dict(row) for row in rows]
|
||||||
|
|
||||||
|
# ========================================================================
|
||||||
|
# MEMORIES
|
||||||
|
# ========================================================================
|
||||||
|
|
||||||
|
async def create_memory(
|
||||||
|
self,
|
||||||
|
org_id: UUID,
|
||||||
|
user_id: UUID,
|
||||||
|
category: MemoryCategory,
|
||||||
|
fact_text: str,
|
||||||
|
workspace_id: Optional[UUID] = None,
|
||||||
|
agent_id: Optional[UUID] = None,
|
||||||
|
confidence: float = 0.8,
|
||||||
|
source_event_id: Optional[UUID] = None,
|
||||||
|
source_thread_id: Optional[UUID] = None,
|
||||||
|
extraction_method: str = "explicit",
|
||||||
|
is_sensitive: bool = False,
|
||||||
|
retention: RetentionPolicy = RetentionPolicy.UNTIL_REVOKED,
|
||||||
|
ttl_days: Optional[int] = None,
|
||||||
|
tags: List[str] = [],
|
||||||
|
metadata: dict = {}
|
||||||
|
) -> Dict[str, Any]:
|
||||||
|
"""Create long-term memory item"""
|
||||||
|
memory_id = uuid4()
|
||||||
|
|
||||||
|
async with self.pool.acquire() as conn:
|
||||||
|
row = await conn.fetchrow("""
|
||||||
|
INSERT INTO long_term_memory_items
|
||||||
|
(memory_id, org_id, workspace_id, user_id, agent_id, category,
|
||||||
|
fact_text, confidence, source_event_id, source_thread_id,
|
||||||
|
extraction_method, is_sensitive, retention, ttl_days, tags, metadata)
|
||||||
|
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16)
|
||||||
|
RETURNING *
|
||||||
|
""", memory_id, org_id, workspace_id, user_id, agent_id, category.value,
|
||||||
|
fact_text, confidence, source_event_id, source_thread_id,
|
||||||
|
extraction_method, is_sensitive, retention.value, ttl_days, tags, metadata)
|
||||||
|
|
||||||
|
logger.info("memory_created", memory_id=str(memory_id), category=category.value)
|
||||||
|
return dict(row)
|
||||||
|
|
||||||
|
async def get_memory(self, memory_id: UUID) -> Optional[Dict[str, Any]]:
|
||||||
|
"""Get memory by ID"""
|
||||||
|
async with self.pool.acquire() as conn:
|
||||||
|
row = await conn.fetchrow("""
|
||||||
|
SELECT * FROM long_term_memory_items
|
||||||
|
WHERE memory_id = $1 AND valid_to IS NULL
|
||||||
|
""", memory_id)
|
||||||
|
return dict(row) if row else None
|
||||||
|
|
||||||
|
async def list_memories(
|
||||||
|
self,
|
||||||
|
org_id: UUID,
|
||||||
|
user_id: UUID,
|
||||||
|
agent_id: Optional[UUID] = None,
|
||||||
|
workspace_id: Optional[UUID] = None,
|
||||||
|
category: Optional[MemoryCategory] = None,
|
||||||
|
include_global: bool = True,
|
||||||
|
limit: int = 50
|
||||||
|
) -> List[Dict[str, Any]]:
|
||||||
|
"""List memories for user"""
|
||||||
|
async with self.pool.acquire() as conn:
|
||||||
|
query = """
|
||||||
|
SELECT * FROM long_term_memory_items
|
||||||
|
WHERE org_id = $1 AND user_id = $2 AND valid_to IS NULL
|
||||||
|
AND confidence >= $3
|
||||||
|
"""
|
||||||
|
params = [org_id, user_id, settings.memory_min_confidence]
|
||||||
|
|
||||||
|
if workspace_id:
|
||||||
|
query += f" AND (workspace_id = ${len(params) + 1} OR workspace_id IS NULL)"
|
||||||
|
params.append(workspace_id)
|
||||||
|
|
||||||
|
if agent_id:
|
||||||
|
if include_global:
|
||||||
|
query += f" AND (agent_id = ${len(params) + 1} OR agent_id IS NULL)"
|
||||||
|
else:
|
||||||
|
query += f" AND agent_id = ${len(params) + 1}"
|
||||||
|
params.append(agent_id)
|
||||||
|
|
||||||
|
if category:
|
||||||
|
query += f" AND category = ${len(params) + 1}"
|
||||||
|
params.append(category.value)
|
||||||
|
|
||||||
|
query += f" ORDER BY confidence DESC, last_used_at DESC NULLS LAST LIMIT ${len(params) + 1}"
|
||||||
|
params.append(limit)
|
||||||
|
|
||||||
|
rows = await conn.fetch(query, *params)
|
||||||
|
|
||||||
|
return [dict(row) for row in rows]
|
||||||
|
|
||||||
|
async def update_memory_embedding_id(self, memory_id: UUID, embedding_id: str):
|
||||||
|
"""Update memory with Qdrant point ID"""
|
||||||
|
async with self.pool.acquire() as conn:
|
||||||
|
await conn.execute("""
|
||||||
|
UPDATE long_term_memory_items
|
||||||
|
SET fact_embedding_id = $2
|
||||||
|
WHERE memory_id = $1
|
||||||
|
""", memory_id, embedding_id)
|
||||||
|
|
||||||
|
async def update_memory_confidence(
|
||||||
|
self,
|
||||||
|
memory_id: UUID,
|
||||||
|
confidence: float,
|
||||||
|
verified: bool = False
|
||||||
|
):
|
||||||
|
"""Update memory confidence"""
|
||||||
|
async with self.pool.acquire() as conn:
|
||||||
|
await conn.execute("""
|
||||||
|
UPDATE long_term_memory_items
|
||||||
|
SET confidence = $2,
|
||||||
|
is_verified = CASE WHEN $3 THEN true ELSE is_verified END,
|
||||||
|
verification_count = verification_count + CASE WHEN $3 THEN 1 ELSE 0 END,
|
||||||
|
last_confirmed_at = CASE WHEN $3 THEN NOW() ELSE last_confirmed_at END
|
||||||
|
WHERE memory_id = $1
|
||||||
|
""", memory_id, confidence, verified)
|
||||||
|
|
||||||
|
async def update_memory_text(self, memory_id: UUID, new_text: str):
|
||||||
|
"""Update memory text"""
|
||||||
|
async with self.pool.acquire() as conn:
|
||||||
|
await conn.execute("""
|
||||||
|
UPDATE long_term_memory_items
|
||||||
|
SET fact_text = $2
|
||||||
|
WHERE memory_id = $1
|
||||||
|
""", memory_id, new_text)
|
||||||
|
|
||||||
|
async def invalidate_memory(self, memory_id: UUID):
|
||||||
|
"""Mark memory as invalid (soft delete)"""
|
||||||
|
async with self.pool.acquire() as conn:
|
||||||
|
await conn.execute("""
|
||||||
|
UPDATE long_term_memory_items
|
||||||
|
SET valid_to = NOW()
|
||||||
|
WHERE memory_id = $1
|
||||||
|
""", memory_id)
|
||||||
|
logger.info("memory_invalidated", memory_id=str(memory_id))
|
||||||
|
|
||||||
|
async def increment_memory_usage(self, memory_id: UUID):
|
||||||
|
"""Increment memory usage counter"""
|
||||||
|
async with self.pool.acquire() as conn:
|
||||||
|
await conn.execute("""
|
||||||
|
UPDATE long_term_memory_items
|
||||||
|
SET use_count = use_count + 1, last_used_at = NOW()
|
||||||
|
WHERE memory_id = $1
|
||||||
|
""", memory_id)
|
||||||
|
|
||||||
|
# ========================================================================
|
||||||
|
# FEEDBACK
|
||||||
|
# ========================================================================
|
||||||
|
|
||||||
|
async def add_memory_feedback(
|
||||||
|
self,
|
||||||
|
memory_id: UUID,
|
||||||
|
user_id: UUID,
|
||||||
|
action: FeedbackAction,
|
||||||
|
old_value: Optional[str] = None,
|
||||||
|
new_value: Optional[str] = None,
|
||||||
|
reason: Optional[str] = None
|
||||||
|
):
|
||||||
|
"""Record user feedback on memory"""
|
||||||
|
feedback_id = uuid4()
|
||||||
|
|
||||||
|
async with self.pool.acquire() as conn:
|
||||||
|
await conn.execute("""
|
||||||
|
INSERT INTO memory_feedback
|
||||||
|
(feedback_id, memory_id, user_id, action, old_value, new_value, reason)
|
||||||
|
VALUES ($1, $2, $3, $4, $5, $6, $7)
|
||||||
|
""", feedback_id, memory_id, user_id, action.value, old_value, new_value, reason)
|
||||||
|
|
||||||
|
logger.info("feedback_recorded", memory_id=str(memory_id), action=action.value)
|
||||||
|
|
||||||
|
# ========================================================================
|
||||||
|
# SUMMARIES
|
||||||
|
# ========================================================================
|
||||||
|
|
||||||
|
async def create_summary(
|
||||||
|
self,
|
||||||
|
thread_id: UUID,
|
||||||
|
summary_text: str,
|
||||||
|
state: dict,
|
||||||
|
events_from_seq: int,
|
||||||
|
events_to_seq: int,
|
||||||
|
events_count: int,
|
||||||
|
original_tokens: Optional[int] = None,
|
||||||
|
summary_tokens: Optional[int] = None
|
||||||
|
) -> Dict[str, Any]:
|
||||||
|
"""Create thread summary"""
|
||||||
|
summary_id = uuid4()
|
||||||
|
|
||||||
|
# Get next version
|
||||||
|
async with self.pool.acquire() as conn:
|
||||||
|
version_row = await conn.fetchrow("""
|
||||||
|
SELECT COALESCE(MAX(version), 0) + 1 as next_version
|
||||||
|
FROM thread_summaries WHERE thread_id = $1
|
||||||
|
""", thread_id)
|
||||||
|
version = version_row["next_version"]
|
||||||
|
|
||||||
|
compression_ratio = None
|
||||||
|
if original_tokens and summary_tokens:
|
||||||
|
compression_ratio = summary_tokens / original_tokens
|
||||||
|
|
||||||
|
row = await conn.fetchrow("""
|
||||||
|
INSERT INTO thread_summaries
|
||||||
|
(summary_id, thread_id, version, summary_text, state,
|
||||||
|
events_from_seq, events_to_seq, events_count,
|
||||||
|
original_tokens, summary_tokens, compression_ratio)
|
||||||
|
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)
|
||||||
|
RETURNING *
|
||||||
|
""", summary_id, thread_id, version, summary_text, state,
|
||||||
|
events_from_seq, events_to_seq, events_count,
|
||||||
|
original_tokens, summary_tokens, compression_ratio)
|
||||||
|
|
||||||
|
logger.info("summary_created", summary_id=str(summary_id), version=version)
|
||||||
|
return dict(row)
|
||||||
|
|
||||||
|
async def get_latest_summary(self, thread_id: UUID) -> Optional[Dict[str, Any]]:
|
||||||
|
"""Get latest summary for thread"""
|
||||||
|
async with self.pool.acquire() as conn:
|
||||||
|
row = await conn.fetchrow("""
|
||||||
|
SELECT * FROM thread_summaries
|
||||||
|
WHERE thread_id = $1
|
||||||
|
ORDER BY version DESC
|
||||||
|
LIMIT 1
|
||||||
|
""", thread_id)
|
||||||
|
return dict(row) if row else None
|
||||||
|
|
||||||
|
# ========================================================================
|
||||||
|
# STATS
|
||||||
|
# ========================================================================
|
||||||
|
|
||||||
|
async def get_stats(self) -> Dict[str, Any]:
|
||||||
|
"""Get database statistics"""
|
||||||
|
async with self.pool.acquire() as conn:
|
||||||
|
threads = await conn.fetchval("SELECT COUNT(*) FROM conversation_threads")
|
||||||
|
events = await conn.fetchval("SELECT COUNT(*) FROM conversation_events")
|
||||||
|
memories = await conn.fetchval("SELECT COUNT(*) FROM long_term_memory_items WHERE valid_to IS NULL")
|
||||||
|
summaries = await conn.fetchval("SELECT COUNT(*) FROM thread_summaries")
|
||||||
|
|
||||||
|
return {
|
||||||
|
"threads": threads,
|
||||||
|
"events": events,
|
||||||
|
"active_memories": memories,
|
||||||
|
"summaries": summaries
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
# Global instance
|
||||||
|
db = Database()
|
||||||
86
services/memory-service/app/embedding.py
Normal file
86
services/memory-service/app/embedding.py
Normal file
@@ -0,0 +1,86 @@
|
|||||||
|
"""
|
||||||
|
DAARION Memory Service - Embedding Layer (Cohere)
|
||||||
|
"""
|
||||||
|
import cohere
|
||||||
|
from typing import List
|
||||||
|
import structlog
|
||||||
|
from tenacity import retry, stop_after_attempt, wait_exponential
|
||||||
|
|
||||||
|
from .config import get_settings
|
||||||
|
|
||||||
|
logger = structlog.get_logger()
|
||||||
|
settings = get_settings()
|
||||||
|
|
||||||
|
# Initialize Cohere client
|
||||||
|
co = cohere.Client(settings.cohere_api_key)
|
||||||
|
|
||||||
|
|
||||||
|
@retry(
|
||||||
|
stop=stop_after_attempt(3),
|
||||||
|
wait=wait_exponential(multiplier=1, min=1, max=10)
|
||||||
|
)
|
||||||
|
async def get_embeddings(
|
||||||
|
texts: List[str],
|
||||||
|
input_type: str = "search_document"
|
||||||
|
) -> List[List[float]]:
|
||||||
|
"""
|
||||||
|
Get embeddings from Cohere API.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
texts: List of texts to embed
|
||||||
|
input_type: "search_document" for indexing, "search_query" for queries
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
List of embedding vectors (1024 dimensions for embed-multilingual-v3.0)
|
||||||
|
"""
|
||||||
|
if not texts:
|
||||||
|
return []
|
||||||
|
|
||||||
|
logger.info("generating_embeddings", count=len(texts), input_type=input_type)
|
||||||
|
|
||||||
|
response = co.embed(
|
||||||
|
texts=texts,
|
||||||
|
model=settings.cohere_model,
|
||||||
|
input_type=input_type,
|
||||||
|
truncate="END"
|
||||||
|
)
|
||||||
|
|
||||||
|
embeddings = response.embeddings
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
"embeddings_generated",
|
||||||
|
count=len(embeddings),
|
||||||
|
dimensions=len(embeddings[0]) if embeddings else 0
|
||||||
|
)
|
||||||
|
|
||||||
|
return embeddings
|
||||||
|
|
||||||
|
|
||||||
|
async def get_query_embedding(query: str) -> List[float]:
|
||||||
|
"""Get embedding for a search query"""
|
||||||
|
embeddings = await get_embeddings([query], input_type="search_query")
|
||||||
|
return embeddings[0] if embeddings else []
|
||||||
|
|
||||||
|
|
||||||
|
async def get_document_embeddings(texts: List[str]) -> List[List[float]]:
|
||||||
|
"""Get embeddings for documents (memories, summaries)"""
|
||||||
|
return await get_embeddings(texts, input_type="search_document")
|
||||||
|
|
||||||
|
|
||||||
|
# Batch processing for large sets
|
||||||
|
async def batch_embed(
|
||||||
|
texts: List[str],
|
||||||
|
input_type: str = "search_document",
|
||||||
|
batch_size: int = 96 # Cohere limit
|
||||||
|
) -> List[List[float]]:
|
||||||
|
"""
|
||||||
|
Embed large number of texts in batches.
|
||||||
|
"""
|
||||||
|
all_embeddings = []
|
||||||
|
|
||||||
|
for i in range(0, len(texts), batch_size):
|
||||||
|
batch = texts[i:i + batch_size]
|
||||||
|
embeddings = await get_embeddings(batch, input_type)
|
||||||
|
all_embeddings.extend(embeddings)
|
||||||
|
|
||||||
|
return all_embeddings
|
||||||
@@ -1,443 +1,483 @@
|
|||||||
"""
|
"""
|
||||||
Memory Service - FastAPI додаток
|
DAARION Memory Service - FastAPI Application
|
||||||
Підтримує: user_facts, dialog_summaries, agent_memory_events
|
|
||||||
Інтеграція з token-gate через RBAC
|
Трирівнева пам'ять агентів:
|
||||||
|
- Short-term: conversation events (робочий буфер)
|
||||||
|
- Mid-term: thread summaries (сесійна/тематична)
|
||||||
|
- Long-term: memory items (персональна/проектна)
|
||||||
"""
|
"""
|
||||||
|
from contextlib import asynccontextmanager
|
||||||
import os
|
from typing import List, Optional
|
||||||
from typing import Optional, List, Dict, Any
|
from uuid import UUID
|
||||||
from datetime import datetime
|
import structlog
|
||||||
|
from fastapi import FastAPI, HTTPException, Query
|
||||||
from fastapi import FastAPI, Depends, HTTPException, Query, Header
|
|
||||||
from fastapi.middleware.cors import CORSMiddleware
|
from fastapi.middleware.cors import CORSMiddleware
|
||||||
from sqlalchemy.orm import Session
|
|
||||||
from sqlalchemy import create_engine
|
|
||||||
from sqlalchemy.orm import sessionmaker
|
|
||||||
|
|
||||||
from app.models import Base, UserFact, DialogSummary, AgentMemoryEvent
|
from .config import get_settings
|
||||||
from app.schemas import (
|
from .models import (
|
||||||
UserFactCreate, UserFactUpdate, UserFactResponse, UserFactUpsertRequest, UserFactUpsertResponse,
|
CreateThreadRequest, AddEventRequest, CreateMemoryRequest,
|
||||||
DialogSummaryCreate, DialogSummaryResponse, DialogSummaryListResponse,
|
MemoryFeedbackRequest, RetrievalRequest, SummaryRequest,
|
||||||
AgentMemoryEventCreate, AgentMemoryEventResponse, AgentMemoryEventListResponse,
|
ThreadResponse, EventResponse, MemoryResponse,
|
||||||
TokenGateCheck, TokenGateCheckResponse
|
SummaryResponse, RetrievalResponse, RetrievalResult,
|
||||||
)
|
ContextResponse, MemoryCategory, FeedbackAction
|
||||||
from app.crud import (
|
|
||||||
get_user_fact, get_user_facts, create_user_fact, update_user_fact,
|
|
||||||
upsert_user_fact, delete_user_fact, get_user_facts_by_token_gate,
|
|
||||||
create_dialog_summary, get_dialog_summaries, get_dialog_summary, delete_dialog_summary,
|
|
||||||
create_agent_memory_event, get_agent_memory_events, delete_agent_memory_event
|
|
||||||
)
|
)
|
||||||
|
from .vector_store import vector_store
|
||||||
|
from .database import db
|
||||||
|
|
||||||
# ========== Configuration ==========
|
logger = structlog.get_logger()
|
||||||
|
settings = get_settings()
|
||||||
|
|
||||||
DATABASE_URL = os.getenv(
|
|
||||||
"DATABASE_URL",
|
|
||||||
"sqlite:///./memory.db" # SQLite для розробки, PostgreSQL для продакшену
|
|
||||||
)
|
|
||||||
|
|
||||||
# Створюємо engine та sessionmaker
|
@asynccontextmanager
|
||||||
engine = create_engine(DATABASE_URL)
|
async def lifespan(app: FastAPI):
|
||||||
SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
|
"""Startup and shutdown events"""
|
||||||
|
# Startup
|
||||||
|
logger.info("starting_memory_service")
|
||||||
|
await db.connect()
|
||||||
|
await vector_store.initialize()
|
||||||
|
yield
|
||||||
|
# Shutdown
|
||||||
|
await db.disconnect()
|
||||||
|
logger.info("memory_service_stopped")
|
||||||
|
|
||||||
# Створюємо таблиці (для dev, в продакшені використовуйте міграції)
|
|
||||||
Base.metadata.create_all(bind=engine)
|
|
||||||
|
|
||||||
# ========== FastAPI App ==========
|
|
||||||
|
|
||||||
app = FastAPI(
|
app = FastAPI(
|
||||||
title="Memory Service",
|
title="DAARION Memory Service",
|
||||||
description="Сервіс пам'яті для MicroDAO: user_facts, dialog_summaries, agent_memory_events",
|
description="Agent memory management with PostgreSQL + Qdrant + Cohere",
|
||||||
version="1.0.0"
|
version="1.0.0",
|
||||||
|
lifespan=lifespan
|
||||||
)
|
)
|
||||||
|
|
||||||
# CORS middleware
|
|
||||||
app.add_middleware(
|
app.add_middleware(
|
||||||
CORSMiddleware,
|
CORSMiddleware,
|
||||||
allow_origins=["*"], # В продакшені обмежте це
|
allow_origins=["*"],
|
||||||
allow_credentials=True,
|
allow_credentials=True,
|
||||||
allow_methods=["*"],
|
allow_methods=["*"],
|
||||||
allow_headers=["*"],
|
allow_headers=["*"],
|
||||||
)
|
)
|
||||||
|
|
||||||
# ========== Dependencies ==========
|
|
||||||
|
|
||||||
def get_db():
|
# ============================================================================
|
||||||
"""Dependency для отримання DB сесії"""
|
# HEALTH
|
||||||
db = SessionLocal()
|
# ============================================================================
|
||||||
try:
|
|
||||||
yield db
|
|
||||||
finally:
|
|
||||||
db.close()
|
|
||||||
|
|
||||||
|
|
||||||
async def verify_token(authorization: Optional[str] = Header(None)) -> Optional[str]:
|
|
||||||
"""
|
|
||||||
Перевірка JWT токену (заглушка)
|
|
||||||
В продакшені інтегруйте з вашою системою авторизації
|
|
||||||
"""
|
|
||||||
if not authorization:
|
|
||||||
raise HTTPException(status_code=401, detail="Missing authorization header")
|
|
||||||
|
|
||||||
# Заглушка: в реальності перевіряйте JWT
|
|
||||||
# token = authorization.replace("Bearer ", "")
|
|
||||||
# user_id = verify_jwt_token(token)
|
|
||||||
# return user_id
|
|
||||||
|
|
||||||
# Для тестування повертаємо user_id з заголовка
|
|
||||||
return "u_test" # TODO: реалізувати реальну перевірку
|
|
||||||
|
|
||||||
|
|
||||||
async def check_token_gate(
|
|
||||||
user_id: str,
|
|
||||||
token_requirements: dict,
|
|
||||||
db: Session
|
|
||||||
) -> TokenGateCheckResponse:
|
|
||||||
"""
|
|
||||||
Перевірка токен-гейту (інтеграція з RBAC/Wallet Service)
|
|
||||||
Заглушка - в продакшені викликайте ваш PDP/Wallet Service
|
|
||||||
"""
|
|
||||||
# TODO: Інтегрувати з:
|
|
||||||
# - PDP Service для перевірки capabilities
|
|
||||||
# - Wallet Service для перевірки балансів
|
|
||||||
# - RBAC для перевірки ролей
|
|
||||||
|
|
||||||
# Приклад логіки:
|
|
||||||
# if "token" in token_requirements:
|
|
||||||
# token_type = token_requirements["token"]
|
|
||||||
# min_balance = token_requirements.get("min_balance", 0)
|
|
||||||
# balance = await wallet_service.get_balance(user_id, token_type)
|
|
||||||
# if balance < min_balance:
|
|
||||||
# return TokenGateCheckResponse(
|
|
||||||
# allowed=False,
|
|
||||||
# reason=f"Insufficient {token_type} balance",
|
|
||||||
# missing_requirements={"token": token_type, "required": min_balance, "current": balance}
|
|
||||||
# )
|
|
||||||
|
|
||||||
# Заглушка: завжди дозволяємо
|
|
||||||
return TokenGateCheckResponse(allowed=True)
|
|
||||||
|
|
||||||
|
|
||||||
# ========== Health Check ==========
|
|
||||||
|
|
||||||
@app.get("/health")
|
@app.get("/health")
|
||||||
async def health_check():
|
async def health():
|
||||||
"""Health check endpoint"""
|
"""Health check"""
|
||||||
return {"status": "ok", "service": "memory-service"}
|
return {
|
||||||
|
"status": "healthy",
|
||||||
|
"service": settings.service_name,
|
||||||
|
"vector_store": await vector_store.get_collection_stats()
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
# ========== User Facts Endpoints ==========
|
# ============================================================================
|
||||||
|
# THREADS (Conversations)
|
||||||
|
# ============================================================================
|
||||||
|
|
||||||
@app.post("/facts/upsert", response_model=UserFactUpsertResponse)
|
@app.post("/threads", response_model=ThreadResponse)
|
||||||
async def upsert_fact(
|
async def create_thread(request: CreateThreadRequest):
|
||||||
fact_request: UserFactUpsertRequest,
|
"""Create new conversation thread"""
|
||||||
db: Session = Depends(get_db),
|
thread = await db.create_thread(
|
||||||
user_id: str = Depends(verify_token)
|
org_id=request.org_id,
|
||||||
|
user_id=request.user_id,
|
||||||
|
workspace_id=request.workspace_id,
|
||||||
|
agent_id=request.agent_id,
|
||||||
|
title=request.title,
|
||||||
|
tags=request.tags,
|
||||||
|
metadata=request.metadata
|
||||||
|
)
|
||||||
|
return thread
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/threads/{thread_id}", response_model=ThreadResponse)
|
||||||
|
async def get_thread(thread_id: UUID):
|
||||||
|
"""Get thread by ID"""
|
||||||
|
thread = await db.get_thread(thread_id)
|
||||||
|
if not thread:
|
||||||
|
raise HTTPException(status_code=404, detail="Thread not found")
|
||||||
|
return thread
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/threads", response_model=List[ThreadResponse])
|
||||||
|
async def list_threads(
|
||||||
|
user_id: UUID = Query(...),
|
||||||
|
org_id: UUID = Query(...),
|
||||||
|
workspace_id: Optional[UUID] = None,
|
||||||
|
agent_id: Optional[UUID] = None,
|
||||||
|
limit: int = Query(default=20, le=100)
|
||||||
):
|
):
|
||||||
"""
|
"""List threads for user"""
|
||||||
Створити або оновити факт користувача (upsert)
|
threads = await db.list_threads(
|
||||||
|
org_id=org_id,
|
||||||
|
user_id=user_id,
|
||||||
|
workspace_id=workspace_id,
|
||||||
|
agent_id=agent_id,
|
||||||
|
limit=limit
|
||||||
|
)
|
||||||
|
return threads
|
||||||
|
|
||||||
Це основний ендпоінт для контрольованої довгострокової пам'яті.
|
|
||||||
Підтримує токен-гейт інтеграцію.
|
# ============================================================================
|
||||||
"""
|
# EVENTS (Short-term Memory)
|
||||||
# Перевірка токен-гейту якщо потрібно
|
# ============================================================================
|
||||||
if fact_request.token_gated and fact_request.token_requirements:
|
|
||||||
gate_check = await check_token_gate(
|
@app.post("/events", response_model=EventResponse)
|
||||||
fact_request.user_id,
|
async def add_event(request: AddEventRequest):
|
||||||
fact_request.token_requirements,
|
"""Add event to conversation (message, tool call, etc.)"""
|
||||||
db
|
event = await db.add_event(
|
||||||
)
|
thread_id=request.thread_id,
|
||||||
if not gate_check.allowed:
|
event_type=request.event_type,
|
||||||
raise HTTPException(
|
role=request.role,
|
||||||
status_code=403,
|
content=request.content,
|
||||||
detail=f"Token gate check failed: {gate_check.reason}"
|
tool_name=request.tool_name,
|
||||||
|
tool_input=request.tool_input,
|
||||||
|
tool_output=request.tool_output,
|
||||||
|
payload=request.payload,
|
||||||
|
token_count=request.token_count,
|
||||||
|
model_used=request.model_used,
|
||||||
|
latency_ms=request.latency_ms,
|
||||||
|
metadata=request.metadata
|
||||||
|
)
|
||||||
|
return event
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/threads/{thread_id}/events", response_model=List[EventResponse])
|
||||||
|
async def get_events(
|
||||||
|
thread_id: UUID,
|
||||||
|
limit: int = Query(default=50, le=200),
|
||||||
|
offset: int = Query(default=0)
|
||||||
|
):
|
||||||
|
"""Get events for thread (most recent first)"""
|
||||||
|
events = await db.get_events(thread_id, limit=limit, offset=offset)
|
||||||
|
return events
|
||||||
|
|
||||||
|
|
||||||
|
# ============================================================================
|
||||||
|
# MEMORIES (Long-term Memory)
|
||||||
|
# ============================================================================
|
||||||
|
|
||||||
|
@app.post("/memories", response_model=MemoryResponse)
|
||||||
|
async def create_memory(request: CreateMemoryRequest):
|
||||||
|
"""Create long-term memory item"""
|
||||||
|
# Create in PostgreSQL
|
||||||
|
memory = await db.create_memory(
|
||||||
|
org_id=request.org_id,
|
||||||
|
user_id=request.user_id,
|
||||||
|
workspace_id=request.workspace_id,
|
||||||
|
agent_id=request.agent_id,
|
||||||
|
category=request.category,
|
||||||
|
fact_text=request.fact_text,
|
||||||
|
confidence=request.confidence,
|
||||||
|
source_event_id=request.source_event_id,
|
||||||
|
source_thread_id=request.source_thread_id,
|
||||||
|
extraction_method=request.extraction_method,
|
||||||
|
is_sensitive=request.is_sensitive,
|
||||||
|
retention=request.retention,
|
||||||
|
ttl_days=request.ttl_days,
|
||||||
|
tags=request.tags,
|
||||||
|
metadata=request.metadata
|
||||||
|
)
|
||||||
|
|
||||||
|
# Index in Qdrant
|
||||||
|
point_id = await vector_store.index_memory(
|
||||||
|
memory_id=memory["memory_id"],
|
||||||
|
text=request.fact_text,
|
||||||
|
org_id=request.org_id,
|
||||||
|
user_id=request.user_id,
|
||||||
|
category=request.category,
|
||||||
|
agent_id=request.agent_id,
|
||||||
|
workspace_id=request.workspace_id,
|
||||||
|
thread_id=request.source_thread_id
|
||||||
|
)
|
||||||
|
|
||||||
|
# Update memory with embedding ID
|
||||||
|
await db.update_memory_embedding_id(memory["memory_id"], point_id)
|
||||||
|
|
||||||
|
return memory
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/memories/{memory_id}", response_model=MemoryResponse)
|
||||||
|
async def get_memory(memory_id: UUID):
|
||||||
|
"""Get memory by ID"""
|
||||||
|
memory = await db.get_memory(memory_id)
|
||||||
|
if not memory:
|
||||||
|
raise HTTPException(status_code=404, detail="Memory not found")
|
||||||
|
return memory
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/memories", response_model=List[MemoryResponse])
|
||||||
|
async def list_memories(
|
||||||
|
user_id: UUID = Query(...),
|
||||||
|
org_id: UUID = Query(...),
|
||||||
|
agent_id: Optional[UUID] = None,
|
||||||
|
workspace_id: Optional[UUID] = None,
|
||||||
|
category: Optional[MemoryCategory] = None,
|
||||||
|
include_global: bool = True,
|
||||||
|
limit: int = Query(default=50, le=200)
|
||||||
|
):
|
||||||
|
"""List memories for user"""
|
||||||
|
memories = await db.list_memories(
|
||||||
|
org_id=org_id,
|
||||||
|
user_id=user_id,
|
||||||
|
agent_id=agent_id,
|
||||||
|
workspace_id=workspace_id,
|
||||||
|
category=category,
|
||||||
|
include_global=include_global,
|
||||||
|
limit=limit
|
||||||
|
)
|
||||||
|
return memories
|
||||||
|
|
||||||
|
|
||||||
|
@app.post("/memories/{memory_id}/feedback")
|
||||||
|
async def memory_feedback(memory_id: UUID, request: MemoryFeedbackRequest):
|
||||||
|
"""User feedback on memory (confirm/reject/edit/delete)"""
|
||||||
|
memory = await db.get_memory(memory_id)
|
||||||
|
if not memory:
|
||||||
|
raise HTTPException(status_code=404, detail="Memory not found")
|
||||||
|
|
||||||
|
# Record feedback
|
||||||
|
await db.add_memory_feedback(
|
||||||
|
memory_id=memory_id,
|
||||||
|
user_id=request.user_id,
|
||||||
|
action=request.action,
|
||||||
|
old_value=memory["fact_text"],
|
||||||
|
new_value=request.new_value,
|
||||||
|
reason=request.reason
|
||||||
|
)
|
||||||
|
|
||||||
|
# Apply action
|
||||||
|
if request.action == FeedbackAction.CONFIRM:
|
||||||
|
new_confidence = min(1.0, memory["confidence"] + settings.memory_confirm_boost)
|
||||||
|
await db.update_memory_confidence(memory_id, new_confidence, verified=True)
|
||||||
|
|
||||||
|
elif request.action == FeedbackAction.REJECT:
|
||||||
|
new_confidence = max(0.0, memory["confidence"] - settings.memory_reject_penalty)
|
||||||
|
if new_confidence < settings.memory_min_confidence:
|
||||||
|
# Mark as invalid
|
||||||
|
await db.invalidate_memory(memory_id)
|
||||||
|
await vector_store.delete_memory(memory_id)
|
||||||
|
else:
|
||||||
|
await db.update_memory_confidence(memory_id, new_confidence)
|
||||||
|
|
||||||
|
elif request.action == FeedbackAction.EDIT:
|
||||||
|
if request.new_value:
|
||||||
|
await db.update_memory_text(memory_id, request.new_value)
|
||||||
|
# Re-index with new text
|
||||||
|
await vector_store.delete_memory(memory_id)
|
||||||
|
await vector_store.index_memory(
|
||||||
|
memory_id=memory_id,
|
||||||
|
text=request.new_value,
|
||||||
|
org_id=memory["org_id"],
|
||||||
|
user_id=memory["user_id"],
|
||||||
|
category=memory["category"],
|
||||||
|
agent_id=memory.get("agent_id"),
|
||||||
|
workspace_id=memory.get("workspace_id")
|
||||||
)
|
)
|
||||||
|
|
||||||
# Перевірка прав доступу (користувач може змінювати тільки свої факти)
|
elif request.action == FeedbackAction.DELETE:
|
||||||
if fact_request.user_id != user_id:
|
await db.invalidate_memory(memory_id)
|
||||||
raise HTTPException(status_code=403, detail="Cannot modify other user's facts")
|
await vector_store.delete_memory(memory_id)
|
||||||
|
|
||||||
fact, created = upsert_user_fact(db, fact_request)
|
return {"status": "ok", "action": request.action.value}
|
||||||
|
|
||||||
return UserFactUpsertResponse(
|
|
||||||
fact=UserFactResponse.model_validate(fact),
|
# ============================================================================
|
||||||
created=created
|
# RETRIEVAL (Semantic Search)
|
||||||
|
# ============================================================================
|
||||||
|
|
||||||
|
@app.post("/retrieve", response_model=RetrievalResponse)
|
||||||
|
async def retrieve_memories(request: RetrievalRequest):
|
||||||
|
"""
|
||||||
|
Semantic retrieval of relevant memories.
|
||||||
|
|
||||||
|
Performs multiple queries and deduplicates results.
|
||||||
|
"""
|
||||||
|
all_results = []
|
||||||
|
seen_ids = set()
|
||||||
|
|
||||||
|
for query in request.queries:
|
||||||
|
results = await vector_store.search_memories(
|
||||||
|
query=query,
|
||||||
|
org_id=request.org_id,
|
||||||
|
user_id=request.user_id,
|
||||||
|
agent_id=request.agent_id,
|
||||||
|
workspace_id=request.workspace_id,
|
||||||
|
categories=request.categories,
|
||||||
|
include_global=request.include_global,
|
||||||
|
top_k=request.top_k
|
||||||
|
)
|
||||||
|
|
||||||
|
for r in results:
|
||||||
|
memory_id = r.get("memory_id")
|
||||||
|
if memory_id and memory_id not in seen_ids:
|
||||||
|
seen_ids.add(memory_id)
|
||||||
|
|
||||||
|
# Get full memory from DB for confidence check
|
||||||
|
memory = await db.get_memory(UUID(memory_id))
|
||||||
|
if memory and memory["confidence"] >= request.min_confidence:
|
||||||
|
all_results.append(RetrievalResult(
|
||||||
|
memory_id=UUID(memory_id),
|
||||||
|
fact_text=r["text"],
|
||||||
|
category=MemoryCategory(r["category"]),
|
||||||
|
confidence=memory["confidence"],
|
||||||
|
relevance_score=r["score"],
|
||||||
|
agent_id=UUID(r["agent_id"]) if r.get("agent_id") else None,
|
||||||
|
is_global=r.get("agent_id") is None
|
||||||
|
))
|
||||||
|
|
||||||
|
# Update usage stats
|
||||||
|
await db.increment_memory_usage(UUID(memory_id))
|
||||||
|
|
||||||
|
# Sort by relevance
|
||||||
|
all_results.sort(key=lambda x: x.relevance_score, reverse=True)
|
||||||
|
|
||||||
|
return RetrievalResponse(
|
||||||
|
results=all_results[:request.top_k],
|
||||||
|
query_count=len(request.queries),
|
||||||
|
total_results=len(all_results)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@app.get("/facts", response_model=List[UserFactResponse])
|
# ============================================================================
|
||||||
async def list_facts(
|
# SUMMARIES (Mid-term Memory)
|
||||||
team_id: Optional[str] = Query(None),
|
# ============================================================================
|
||||||
fact_keys: Optional[str] = Query(None, description="Comma-separated list of fact keys"),
|
|
||||||
skip: int = Query(0, ge=0),
|
|
||||||
limit: int = Query(100, ge=1, le=1000),
|
|
||||||
db: Session = Depends(get_db),
|
|
||||||
user_id: str = Depends(verify_token)
|
|
||||||
):
|
|
||||||
"""Отримати список фактів користувача"""
|
|
||||||
fact_keys_list = None
|
|
||||||
if fact_keys:
|
|
||||||
fact_keys_list = [k.strip() for k in fact_keys.split(",")]
|
|
||||||
|
|
||||||
facts = get_user_facts(db, user_id, team_id, fact_keys_list, skip, limit)
|
@app.post("/threads/{thread_id}/summarize", response_model=SummaryResponse)
|
||||||
return [UserFactResponse.model_validate(f) for f in facts]
|
async def create_summary(thread_id: UUID, request: SummaryRequest):
|
||||||
|
|
||||||
|
|
||||||
@app.get("/facts/{fact_key}", response_model=UserFactResponse)
|
|
||||||
async def get_fact(
|
|
||||||
fact_key: str,
|
|
||||||
team_id: Optional[str] = Query(None),
|
|
||||||
db: Session = Depends(get_db),
|
|
||||||
user_id: str = Depends(verify_token)
|
|
||||||
):
|
|
||||||
"""Отримати конкретний факт за ключем"""
|
|
||||||
fact = get_user_fact(db, user_id, fact_key, team_id)
|
|
||||||
if not fact:
|
|
||||||
raise HTTPException(status_code=404, detail="Fact not found")
|
|
||||||
return UserFactResponse.model_validate(fact)
|
|
||||||
|
|
||||||
|
|
||||||
@app.post("/facts", response_model=UserFactResponse)
|
|
||||||
async def create_fact(
|
|
||||||
fact: UserFactCreate,
|
|
||||||
db: Session = Depends(get_db),
|
|
||||||
user_id: str = Depends(verify_token)
|
|
||||||
):
|
|
||||||
"""Створити новий факт"""
|
|
||||||
if fact.user_id != user_id:
|
|
||||||
raise HTTPException(status_code=403, detail="Cannot create fact for other user")
|
|
||||||
|
|
||||||
db_fact = create_user_fact(db, fact)
|
|
||||||
return UserFactResponse.model_validate(db_fact)
|
|
||||||
|
|
||||||
|
|
||||||
@app.patch("/facts/{fact_id}", response_model=UserFactResponse)
|
|
||||||
async def update_fact(
|
|
||||||
fact_id: str,
|
|
||||||
fact_update: UserFactUpdate,
|
|
||||||
db: Session = Depends(get_db),
|
|
||||||
user_id: str = Depends(verify_token)
|
|
||||||
):
|
|
||||||
"""Оновити факт"""
|
|
||||||
fact = db.query(UserFact).filter(UserFact.id == fact_id).first()
|
|
||||||
if not fact:
|
|
||||||
raise HTTPException(status_code=404, detail="Fact not found")
|
|
||||||
|
|
||||||
if fact.user_id != user_id:
|
|
||||||
raise HTTPException(status_code=403, detail="Cannot modify other user's fact")
|
|
||||||
|
|
||||||
updated_fact = update_user_fact(db, fact_id, fact_update)
|
|
||||||
if not updated_fact:
|
|
||||||
raise HTTPException(status_code=404, detail="Fact not found")
|
|
||||||
|
|
||||||
return UserFactResponse.model_validate(updated_fact)
|
|
||||||
|
|
||||||
|
|
||||||
@app.delete("/facts/{fact_id}")
|
|
||||||
async def delete_fact(
|
|
||||||
fact_id: str,
|
|
||||||
db: Session = Depends(get_db),
|
|
||||||
user_id: str = Depends(verify_token)
|
|
||||||
):
|
|
||||||
"""Видалити факт"""
|
|
||||||
fact = db.query(UserFact).filter(UserFact.id == fact_id).first()
|
|
||||||
if not fact:
|
|
||||||
raise HTTPException(status_code=404, detail="Fact not found")
|
|
||||||
|
|
||||||
if fact.user_id != user_id:
|
|
||||||
raise HTTPException(status_code=403, detail="Cannot delete other user's fact")
|
|
||||||
|
|
||||||
success = delete_user_fact(db, fact_id)
|
|
||||||
if not success:
|
|
||||||
raise HTTPException(status_code=404, detail="Fact not found")
|
|
||||||
|
|
||||||
return {"success": True}
|
|
||||||
|
|
||||||
|
|
||||||
@app.get("/facts/token-gated", response_model=List[UserFactResponse])
|
|
||||||
async def list_token_gated_facts(
|
|
||||||
team_id: Optional[str] = Query(None),
|
|
||||||
db: Session = Depends(get_db),
|
|
||||||
user_id: str = Depends(verify_token)
|
|
||||||
):
|
|
||||||
"""Отримати токен-гейт факти користувача"""
|
|
||||||
facts = get_user_facts_by_token_gate(db, user_id, team_id)
|
|
||||||
return [UserFactResponse.model_validate(f) for f in facts]
|
|
||||||
|
|
||||||
|
|
||||||
# ========== Dialog Summary Endpoints ==========
|
|
||||||
|
|
||||||
@app.post("/summaries", response_model=DialogSummaryResponse)
|
|
||||||
async def create_summary(
|
|
||||||
summary: DialogSummaryCreate,
|
|
||||||
db: Session = Depends(get_db),
|
|
||||||
user_id: str = Depends(verify_token)
|
|
||||||
):
|
|
||||||
"""
|
"""
|
||||||
Створити підсумок діалогу
|
Generate rolling summary for thread.
|
||||||
|
|
||||||
Використовується для масштабування без переповнення контексту.
|
Compresses old events into a structured summary.
|
||||||
Агрегує інформацію про сесії/діалоги.
|
|
||||||
"""
|
"""
|
||||||
db_summary = create_dialog_summary(db, summary)
|
thread = await db.get_thread(thread_id)
|
||||||
return DialogSummaryResponse.model_validate(db_summary)
|
if not thread:
|
||||||
|
raise HTTPException(status_code=404, detail="Thread not found")
|
||||||
|
|
||||||
|
# Check if summary is needed
|
||||||
|
if not request.force and thread["total_tokens"] < settings.summary_trigger_tokens:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=400,
|
||||||
|
detail=f"Token count ({thread['total_tokens']}) below threshold ({settings.summary_trigger_tokens})"
|
||||||
|
)
|
||||||
|
|
||||||
@app.get("/summaries", response_model=DialogSummaryListResponse)
|
# Get events to summarize
|
||||||
async def list_summaries(
|
events = await db.get_events_for_summary(thread_id)
|
||||||
team_id: Optional[str] = Query(None),
|
|
||||||
channel_id: Optional[str] = Query(None),
|
# TODO: Call LLM to generate summary
|
||||||
agent_id: Optional[str] = Query(None),
|
# For now, create a placeholder
|
||||||
user_id_param: Optional[str] = Query(None, alias="user_id"),
|
summary_text = f"Summary of {len(events)} events. [Implement LLM summarization]"
|
||||||
skip: int = Query(0, ge=0),
|
state = {
|
||||||
limit: int = Query(50, ge=1, le=200),
|
"goals": [],
|
||||||
cursor: Optional[str] = Query(None),
|
"decisions": [],
|
||||||
db: Session = Depends(get_db),
|
"open_questions": [],
|
||||||
user_id: str = Depends(verify_token)
|
"next_steps": [],
|
||||||
):
|
"key_facts": []
|
||||||
"""Отримати список підсумків діалогів"""
|
}
|
||||||
summaries, next_cursor = get_dialog_summaries(
|
|
||||||
db, team_id, channel_id, agent_id, user_id_param, skip, limit, cursor
|
# Create summary
|
||||||
|
summary = await db.create_summary(
|
||||||
|
thread_id=thread_id,
|
||||||
|
summary_text=summary_text,
|
||||||
|
state=state,
|
||||||
|
events_from_seq=events[0]["sequence_num"] if events else 0,
|
||||||
|
events_to_seq=events[-1]["sequence_num"] if events else 0,
|
||||||
|
events_count=len(events)
|
||||||
)
|
)
|
||||||
|
|
||||||
return DialogSummaryListResponse(
|
# Index summary in Qdrant
|
||||||
items=[DialogSummaryResponse.model_validate(s) for s in summaries],
|
await vector_store.index_summary(
|
||||||
total=len(summaries),
|
summary_id=summary["summary_id"],
|
||||||
cursor=next_cursor
|
text=summary_text,
|
||||||
|
thread_id=thread_id,
|
||||||
|
org_id=thread["org_id"],
|
||||||
|
user_id=thread["user_id"],
|
||||||
|
agent_id=thread.get("agent_id"),
|
||||||
|
workspace_id=thread.get("workspace_id")
|
||||||
|
)
|
||||||
|
|
||||||
|
return summary
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/threads/{thread_id}/summary", response_model=Optional[SummaryResponse])
|
||||||
|
async def get_latest_summary(thread_id: UUID):
|
||||||
|
"""Get latest summary for thread"""
|
||||||
|
summary = await db.get_latest_summary(thread_id)
|
||||||
|
return summary
|
||||||
|
|
||||||
|
|
||||||
|
# ============================================================================
|
||||||
|
# CONTEXT (Full context for agent)
|
||||||
|
# ============================================================================
|
||||||
|
|
||||||
|
@app.get("/threads/{thread_id}/context", response_model=ContextResponse)
|
||||||
|
async def get_context(
|
||||||
|
thread_id: UUID,
|
||||||
|
queries: List[str] = Query(default=[]),
|
||||||
|
top_k: int = Query(default=10)
|
||||||
|
):
|
||||||
|
"""
|
||||||
|
Get full context for agent prompt.
|
||||||
|
|
||||||
|
Combines:
|
||||||
|
- Latest summary (mid-term)
|
||||||
|
- Recent messages (short-term)
|
||||||
|
- Retrieved memories (long-term)
|
||||||
|
"""
|
||||||
|
thread = await db.get_thread(thread_id)
|
||||||
|
if not thread:
|
||||||
|
raise HTTPException(status_code=404, detail="Thread not found")
|
||||||
|
|
||||||
|
# Get summary
|
||||||
|
summary = await db.get_latest_summary(thread_id)
|
||||||
|
|
||||||
|
# Get recent messages
|
||||||
|
recent = await db.get_events(
|
||||||
|
thread_id,
|
||||||
|
limit=settings.short_term_window_messages
|
||||||
|
)
|
||||||
|
|
||||||
|
# Retrieve memories if queries provided
|
||||||
|
retrieved = []
|
||||||
|
if queries:
|
||||||
|
retrieval_response = await retrieve_memories(RetrievalRequest(
|
||||||
|
org_id=thread["org_id"],
|
||||||
|
user_id=thread["user_id"],
|
||||||
|
agent_id=thread.get("agent_id"),
|
||||||
|
workspace_id=thread.get("workspace_id"),
|
||||||
|
queries=queries,
|
||||||
|
top_k=top_k,
|
||||||
|
include_global=True
|
||||||
|
))
|
||||||
|
retrieved = retrieval_response.results
|
||||||
|
|
||||||
|
# Estimate tokens
|
||||||
|
token_estimate = sum(e.get("token_count", 0) or 0 for e in recent)
|
||||||
|
if summary:
|
||||||
|
token_estimate += summary.get("summary_tokens", 0) or 0
|
||||||
|
|
||||||
|
return ContextResponse(
|
||||||
|
thread_id=thread_id,
|
||||||
|
summary=summary,
|
||||||
|
recent_messages=recent,
|
||||||
|
retrieved_memories=retrieved,
|
||||||
|
token_estimate=token_estimate
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@app.get("/summaries/{summary_id}", response_model=DialogSummaryResponse)
|
# ============================================================================
|
||||||
async def get_summary(
|
# ADMIN
|
||||||
summary_id: str,
|
# ============================================================================
|
||||||
db: Session = Depends(get_db),
|
|
||||||
user_id: str = Depends(verify_token)
|
|
||||||
):
|
|
||||||
"""Отримати підсумок за ID"""
|
|
||||||
summary = get_dialog_summary(db, summary_id)
|
|
||||||
if not summary:
|
|
||||||
raise HTTPException(status_code=404, detail="Summary not found")
|
|
||||||
return DialogSummaryResponse.model_validate(summary)
|
|
||||||
|
|
||||||
|
@app.get("/stats")
|
||||||
@app.delete("/summaries/{summary_id}")
|
async def get_stats():
|
||||||
async def delete_summary(
|
"""Get service statistics"""
|
||||||
summary_id: str,
|
return {
|
||||||
db: Session = Depends(get_db),
|
"vector_store": await vector_store.get_collection_stats(),
|
||||||
user_id: str = Depends(verify_token)
|
"database": await db.get_stats()
|
||||||
):
|
}
|
||||||
"""Видалити підсумок"""
|
|
||||||
success = delete_dialog_summary(db, summary_id)
|
|
||||||
if not success:
|
|
||||||
raise HTTPException(status_code=404, detail="Summary not found")
|
|
||||||
return {"success": True}
|
|
||||||
|
|
||||||
|
|
||||||
# ========== Agent Memory Event Endpoints ==========
|
|
||||||
|
|
||||||
@app.post("/agents/{agent_id}/memory", response_model=AgentMemoryEventResponse)
|
|
||||||
async def create_memory_event(
|
|
||||||
agent_id: str,
|
|
||||||
event: AgentMemoryEventCreate,
|
|
||||||
db: Session = Depends(get_db),
|
|
||||||
user_id: str = Depends(verify_token)
|
|
||||||
):
|
|
||||||
"""Створити подію пам'яті агента"""
|
|
||||||
# Перевірка що agent_id збігається
|
|
||||||
if event.agent_id != agent_id:
|
|
||||||
raise HTTPException(status_code=400, detail="agent_id mismatch")
|
|
||||||
|
|
||||||
db_event = create_agent_memory_event(db, event)
|
|
||||||
return AgentMemoryEventResponse.model_validate(db_event)
|
|
||||||
|
|
||||||
|
|
||||||
@app.get("/agents/{agent_id}/memory", response_model=AgentMemoryEventListResponse)
|
|
||||||
async def list_memory_events(
|
|
||||||
agent_id: str,
|
|
||||||
team_id: Optional[str] = Query(None),
|
|
||||||
channel_id: Optional[str] = Query(None),
|
|
||||||
scope: Optional[str] = Query(None, description="short_term | mid_term | long_term"),
|
|
||||||
kind: Optional[str] = Query(None, description="message | fact | summary | note"),
|
|
||||||
skip: int = Query(0, ge=0),
|
|
||||||
limit: int = Query(50, ge=1, le=200),
|
|
||||||
cursor: Optional[str] = Query(None),
|
|
||||||
db: Session = Depends(get_db),
|
|
||||||
user_id: str = Depends(verify_token)
|
|
||||||
):
|
|
||||||
"""Отримати список подій пам'яті агента"""
|
|
||||||
events, next_cursor = get_agent_memory_events(
|
|
||||||
db, agent_id, team_id, channel_id, scope, kind, skip, limit, cursor
|
|
||||||
)
|
|
||||||
|
|
||||||
return AgentMemoryEventListResponse(
|
|
||||||
items=[AgentMemoryEventResponse.model_validate(e) for e in events],
|
|
||||||
total=len(events),
|
|
||||||
cursor=next_cursor
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
@app.delete("/agents/{agent_id}/memory/{event_id}")
|
|
||||||
async def delete_memory_event(
|
|
||||||
agent_id: str,
|
|
||||||
event_id: str,
|
|
||||||
db: Session = Depends(get_db),
|
|
||||||
user_id: str = Depends(verify_token)
|
|
||||||
):
|
|
||||||
"""Видалити подію пам'яті"""
|
|
||||||
success = delete_agent_memory_event(db, event_id)
|
|
||||||
if not success:
|
|
||||||
raise HTTPException(status_code=404, detail="Memory event not found")
|
|
||||||
return {"success": True}
|
|
||||||
|
|
||||||
|
|
||||||
# ========== Monitor Events Endpoints (Batch Processing) ==========
|
|
||||||
|
|
||||||
from app.monitor_events import MonitorEventBatch, MonitorEventResponse, save_monitor_events_batch, save_monitor_event_single
|
|
||||||
|
|
||||||
@app.post("/api/memory/monitor-events/batch", response_model=MonitorEventResponse)
|
|
||||||
async def save_monitor_events_batch_endpoint(
|
|
||||||
batch: MonitorEventBatch,
|
|
||||||
db: Session = Depends(get_db),
|
|
||||||
authorization: Optional[str] = Header(None)
|
|
||||||
):
|
|
||||||
"""
|
|
||||||
Зберегти батч подій Monitor Agent
|
|
||||||
Оптимізовано для збору метрик з багатьох нод
|
|
||||||
"""
|
|
||||||
return await save_monitor_events_batch(batch, db, authorization)
|
|
||||||
|
|
||||||
@app.post("/api/memory/monitor-events/{node_id}", response_model=AgentMemoryEventResponse)
|
|
||||||
async def save_monitor_event_endpoint(
|
|
||||||
node_id: str,
|
|
||||||
event: Dict[str, Any],
|
|
||||||
db: Session = Depends(get_db),
|
|
||||||
authorization: Optional[str] = Header(None)
|
|
||||||
):
|
|
||||||
"""
|
|
||||||
Зберегти одну подію Monitor Agent
|
|
||||||
"""
|
|
||||||
return await save_monitor_event_single(node_id, event, db, authorization)
|
|
||||||
|
|
||||||
|
|
||||||
# ========== Token Gate Integration Endpoint ==========
|
|
||||||
|
|
||||||
@app.post("/token-gate/check", response_model=TokenGateCheckResponse)
|
|
||||||
async def check_token_gate_endpoint(
|
|
||||||
check: TokenGateCheck,
|
|
||||||
db: Session = Depends(get_db),
|
|
||||||
user_id: str = Depends(verify_token)
|
|
||||||
):
|
|
||||||
"""
|
|
||||||
Перевірка токен-гейту для факту
|
|
||||||
|
|
||||||
Інтеграція з RBAC/Wallet Service для перевірки доступу
|
|
||||||
"""
|
|
||||||
if check.user_id != user_id:
|
|
||||||
raise HTTPException(status_code=403, detail="Cannot check token gate for other user")
|
|
||||||
|
|
||||||
return await check_token_gate(user_id, check.token_requirements, db)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
import uvicorn
|
import uvicorn
|
||||||
uvicorn.run("app.main:app", host="0.0.0.0", port=8000, reload=True)
|
uvicorn.run(app, host="0.0.0.0", port=8000)
|
||||||
|
|
||||||
|
|||||||
@@ -1,178 +1,249 @@
|
|||||||
"""
|
"""
|
||||||
SQLAlchemy моделі для Memory Service
|
DAARION Memory Service - Pydantic Models
|
||||||
Підтримує: user_facts, dialog_summaries, agent_memory_events
|
|
||||||
"""
|
"""
|
||||||
|
from datetime import datetime
|
||||||
from sqlalchemy import (
|
from typing import Optional, List, Any
|
||||||
Column, String, Text, JSON, TIMESTAMP,
|
from uuid import UUID
|
||||||
CheckConstraint, Index, Boolean, Integer
|
from enum import Enum
|
||||||
)
|
from pydantic import BaseModel, Field
|
||||||
from sqlalchemy.ext.declarative import declarative_base
|
|
||||||
from sqlalchemy.sql import func
|
|
||||||
import os
|
|
||||||
|
|
||||||
# Перевірка типу бази даних
|
|
||||||
DATABASE_URL = os.getenv("DATABASE_URL", "sqlite:///./memory.db")
|
|
||||||
IS_SQLITE = "sqlite" in DATABASE_URL.lower()
|
|
||||||
|
|
||||||
if IS_SQLITE:
|
|
||||||
# Для SQLite використовуємо стандартні типи
|
|
||||||
from sqlalchemy import JSON as JSONB_TYPE
|
|
||||||
UUID_TYPE = String # SQLite не має UUID, використовуємо String
|
|
||||||
else:
|
|
||||||
# Для PostgreSQL використовуємо специфічні типи
|
|
||||||
from sqlalchemy.dialects.postgresql import UUID, JSONB
|
|
||||||
UUID_TYPE = UUID
|
|
||||||
JSONB_TYPE = JSONB
|
|
||||||
|
|
||||||
try:
|
|
||||||
from pgvector.sqlalchemy import Vector
|
|
||||||
HAS_PGVECTOR = True
|
|
||||||
except ImportError:
|
|
||||||
HAS_PGVECTOR = False
|
|
||||||
# Заглушка для SQLite
|
|
||||||
class Vector:
|
|
||||||
def __init__(self, *args, **kwargs):
|
|
||||||
pass
|
|
||||||
|
|
||||||
Base = declarative_base()
|
|
||||||
|
|
||||||
|
|
||||||
class UserFact(Base):
|
# ============================================================================
|
||||||
"""
|
# ENUMS
|
||||||
Довгострокові факти про користувача
|
# ============================================================================
|
||||||
Використовується для контрольованої довгострокової пам'яті
|
|
||||||
(мови, вподобання, тип користувача, токен-статуси)
|
|
||||||
"""
|
|
||||||
__tablename__ = "user_facts"
|
|
||||||
|
|
||||||
id = Column(UUID_TYPE(as_uuid=False) if not IS_SQLITE else String, primary_key=True, server_default=func.gen_random_uuid() if not IS_SQLITE else None)
|
class EventType(str, Enum):
|
||||||
user_id = Column(String, nullable=False, index=True) # Без FK constraint для тестування
|
MESSAGE = "message"
|
||||||
team_id = Column(String, nullable=True, index=True) # Без FK constraint, оскільки teams може не існувати
|
TOOL_CALL = "tool_call"
|
||||||
|
TOOL_RESULT = "tool_result"
|
||||||
# Ключ факту (наприклад: "language", "is_donor", "is_validator", "top_contributor")
|
DECISION = "decision"
|
||||||
fact_key = Column(String, nullable=False, index=True)
|
SUMMARY = "summary"
|
||||||
|
MEMORY_WRITE = "memory_write"
|
||||||
# Значення факту (може бути текст, число, boolean, JSON)
|
MEMORY_RETRACT = "memory_retract"
|
||||||
fact_value = Column(Text, nullable=True)
|
ERROR = "error"
|
||||||
fact_value_json = Column(JSONB_TYPE, nullable=True)
|
|
||||||
|
|
||||||
# Метадані: джерело, впевненість, термін дії
|
|
||||||
meta = Column(JSONB_TYPE, nullable=False, server_default="{}")
|
|
||||||
|
|
||||||
# Токен-гейт: чи залежить факт від токенів/активності
|
|
||||||
token_gated = Column(Boolean, nullable=False, server_default="false")
|
|
||||||
token_requirements = Column(JSONB_TYPE, nullable=True) # {"token": "DAAR", "min_balance": 1}
|
|
||||||
|
|
||||||
created_at = Column(TIMESTAMP(timezone=True) if not IS_SQLITE else TIMESTAMP, nullable=False, server_default=func.now())
|
|
||||||
updated_at = Column(TIMESTAMP(timezone=True) if not IS_SQLITE else TIMESTAMP, nullable=True, onupdate=func.now())
|
|
||||||
expires_at = Column(TIMESTAMP(timezone=True) if not IS_SQLITE else TIMESTAMP, nullable=True) # Для тимчасових фактів
|
|
||||||
|
|
||||||
__table_args__ = (
|
|
||||||
Index("idx_user_facts_user_key", "user_id", "fact_key"),
|
|
||||||
Index("idx_user_facts_team", "team_id"),
|
|
||||||
Index("idx_user_facts_token_gated", "token_gated"),
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class DialogSummary(Base):
|
class MessageRole(str, Enum):
|
||||||
"""
|
USER = "user"
|
||||||
Підсумки діалогів для масштабування без переповнення контексту
|
ASSISTANT = "assistant"
|
||||||
Зберігає агреговану інформацію про сесії/діалоги
|
SYSTEM = "system"
|
||||||
"""
|
TOOL = "tool"
|
||||||
__tablename__ = "dialog_summaries"
|
|
||||||
|
|
||||||
id = Column(UUID_TYPE(as_uuid=False) if not IS_SQLITE else String, primary_key=True, server_default=func.gen_random_uuid() if not IS_SQLITE else None)
|
|
||||||
|
|
||||||
# Контекст діалогу (без FK constraints для тестування)
|
|
||||||
team_id = Column(String, nullable=False, index=True)
|
|
||||||
channel_id = Column(String, nullable=True, index=True)
|
|
||||||
agent_id = Column(String, nullable=True, index=True)
|
|
||||||
user_id = Column(String, nullable=True, index=True)
|
|
||||||
|
|
||||||
# Період, який охоплює підсумок
|
|
||||||
period_start = Column(TIMESTAMP(timezone=True) if not IS_SQLITE else TIMESTAMP, nullable=False)
|
|
||||||
period_end = Column(TIMESTAMP(timezone=True) if not IS_SQLITE else TIMESTAMP, nullable=False)
|
|
||||||
|
|
||||||
# Підсумок
|
|
||||||
summary_text = Column(Text, nullable=False)
|
|
||||||
summary_json = Column(JSONB_TYPE, nullable=True) # Структуровані дані
|
|
||||||
|
|
||||||
# Статистика
|
|
||||||
message_count = Column(Integer, nullable=False, server_default="0")
|
|
||||||
participant_count = Column(Integer, nullable=False, server_default="0")
|
|
||||||
|
|
||||||
# Ключові теми/теги
|
|
||||||
topics = Column(JSONB_TYPE, nullable=True) # ["project-planning", "bug-fix", ...]
|
|
||||||
|
|
||||||
# Метадані
|
|
||||||
meta = Column(JSONB_TYPE, nullable=False, server_default="{}")
|
|
||||||
|
|
||||||
created_at = Column(TIMESTAMP(timezone=True) if not IS_SQLITE else TIMESTAMP, nullable=False, server_default=func.now())
|
|
||||||
|
|
||||||
__table_args__ = (
|
|
||||||
Index("idx_dialog_summaries_team_period", "team_id", "period_start", "period_end"),
|
|
||||||
Index("idx_dialog_summaries_channel", "channel_id"),
|
|
||||||
Index("idx_dialog_summaries_agent", "agent_id"),
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class AgentMemoryEvent(Base):
|
class MemoryCategory(str, Enum):
|
||||||
"""
|
PREFERENCE = "preference"
|
||||||
Події пам'яті агентів (short-term, mid-term, long-term)
|
IDENTITY = "identity"
|
||||||
Базується на документації: docs/cursor/13_agent_memory_system.md
|
CONSTRAINT = "constraint"
|
||||||
"""
|
PROJECT_FACT = "project_fact"
|
||||||
__tablename__ = "agent_memory_events"
|
RELATIONSHIP = "relationship"
|
||||||
|
SKILL = "skill"
|
||||||
id = Column(UUID_TYPE(as_uuid=False) if not IS_SQLITE else String, primary_key=True, server_default=func.gen_random_uuid() if not IS_SQLITE else None)
|
GOAL = "goal"
|
||||||
|
CONTEXT = "context"
|
||||||
# Без FK constraints для тестування
|
FEEDBACK = "feedback"
|
||||||
agent_id = Column(String, nullable=False, index=True)
|
|
||||||
team_id = Column(String, nullable=False, index=True)
|
|
||||||
channel_id = Column(String, nullable=True, index=True)
|
|
||||||
user_id = Column(String, nullable=True, index=True)
|
|
||||||
|
|
||||||
# Scope: short_term, mid_term, long_term
|
|
||||||
scope = Column(String, nullable=False)
|
|
||||||
|
|
||||||
# Kind: message, fact, summary, note
|
|
||||||
kind = Column(String, nullable=False)
|
|
||||||
|
|
||||||
# Тіло події
|
|
||||||
body_text = Column(Text, nullable=True)
|
|
||||||
body_json = Column(JSONB_TYPE, nullable=True)
|
|
||||||
|
|
||||||
created_at = Column(TIMESTAMP(timezone=True) if not IS_SQLITE else TIMESTAMP, nullable=False, server_default=func.now())
|
|
||||||
|
|
||||||
__table_args__ = (
|
|
||||||
CheckConstraint("scope IN ('short_term', 'mid_term', 'long_term')", name="ck_agent_memory_scope"),
|
|
||||||
CheckConstraint("kind IN ('message', 'fact', 'summary', 'note')", name="ck_agent_memory_kind"),
|
|
||||||
Index("idx_agent_memory_events_agent_team_scope", "agent_id", "team_id", "scope"),
|
|
||||||
Index("idx_agent_memory_events_channel", "agent_id", "channel_id"),
|
|
||||||
Index("idx_agent_memory_events_created_at", "created_at"),
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class AgentMemoryFactsVector(Base):
|
class RetentionPolicy(str, Enum):
|
||||||
"""
|
PERMANENT = "permanent"
|
||||||
Векторні представлення фактів для RAG (Retrieval-Augmented Generation)
|
SESSION = "session"
|
||||||
"""
|
TTL_DAYS = "ttl_days"
|
||||||
__tablename__ = "agent_memory_facts_vector"
|
UNTIL_REVOKED = "until_revoked"
|
||||||
|
|
||||||
id = Column(UUID_TYPE(as_uuid=False) if not IS_SQLITE else String, primary_key=True, server_default=func.gen_random_uuid() if not IS_SQLITE else None)
|
|
||||||
|
|
||||||
# Без FK constraints для тестування
|
class FeedbackAction(str, Enum):
|
||||||
agent_id = Column(String, nullable=False, index=True)
|
CONFIRM = "confirm"
|
||||||
team_id = Column(String, nullable=False, index=True)
|
REJECT = "reject"
|
||||||
|
EDIT = "edit"
|
||||||
|
DELETE = "delete"
|
||||||
|
|
||||||
fact_text = Column(Text, nullable=False)
|
|
||||||
embedding = Column(Vector(1536), nullable=True) if HAS_PGVECTOR else Column(Text, nullable=True) # OpenAI ada-002 embedding size
|
|
||||||
|
|
||||||
meta = Column(JSONB_TYPE, nullable=False, server_default="{}")
|
# ============================================================================
|
||||||
|
# REQUEST MODELS
|
||||||
|
# ============================================================================
|
||||||
|
|
||||||
created_at = Column(TIMESTAMP(timezone=True) if not IS_SQLITE else TIMESTAMP, nullable=False, server_default=func.now())
|
class CreateThreadRequest(BaseModel):
|
||||||
|
"""Create new conversation thread"""
|
||||||
|
org_id: UUID
|
||||||
|
workspace_id: Optional[UUID] = None
|
||||||
|
user_id: UUID
|
||||||
|
agent_id: Optional[UUID] = None
|
||||||
|
title: Optional[str] = None
|
||||||
|
tags: List[str] = []
|
||||||
|
metadata: dict = {}
|
||||||
|
|
||||||
__table_args__ = (
|
|
||||||
Index("idx_agent_memory_facts_vector_agent_team", "agent_id", "team_id"),
|
|
||||||
)
|
|
||||||
|
|
||||||
|
class AddEventRequest(BaseModel):
|
||||||
|
"""Add event to conversation"""
|
||||||
|
thread_id: UUID
|
||||||
|
event_type: EventType
|
||||||
|
role: Optional[MessageRole] = None
|
||||||
|
content: Optional[str] = None
|
||||||
|
tool_name: Optional[str] = None
|
||||||
|
tool_input: Optional[dict] = None
|
||||||
|
tool_output: Optional[dict] = None
|
||||||
|
payload: dict = {}
|
||||||
|
token_count: Optional[int] = None
|
||||||
|
model_used: Optional[str] = None
|
||||||
|
latency_ms: Optional[int] = None
|
||||||
|
metadata: dict = {}
|
||||||
|
|
||||||
|
|
||||||
|
class CreateMemoryRequest(BaseModel):
|
||||||
|
"""Create long-term memory item"""
|
||||||
|
org_id: UUID
|
||||||
|
workspace_id: Optional[UUID] = None
|
||||||
|
user_id: UUID
|
||||||
|
agent_id: Optional[UUID] = None # null = global
|
||||||
|
category: MemoryCategory
|
||||||
|
fact_text: str
|
||||||
|
confidence: float = Field(default=0.8, ge=0, le=1)
|
||||||
|
source_event_id: Optional[UUID] = None
|
||||||
|
source_thread_id: Optional[UUID] = None
|
||||||
|
extraction_method: str = "explicit"
|
||||||
|
is_sensitive: bool = False
|
||||||
|
retention: RetentionPolicy = RetentionPolicy.UNTIL_REVOKED
|
||||||
|
ttl_days: Optional[int] = None
|
||||||
|
tags: List[str] = []
|
||||||
|
metadata: dict = {}
|
||||||
|
|
||||||
|
|
||||||
|
class MemoryFeedbackRequest(BaseModel):
|
||||||
|
"""User feedback on memory"""
|
||||||
|
memory_id: UUID
|
||||||
|
user_id: UUID
|
||||||
|
action: FeedbackAction
|
||||||
|
new_value: Optional[str] = None
|
||||||
|
reason: Optional[str] = None
|
||||||
|
|
||||||
|
|
||||||
|
class RetrievalRequest(BaseModel):
|
||||||
|
"""Semantic retrieval request"""
|
||||||
|
org_id: UUID
|
||||||
|
user_id: UUID
|
||||||
|
agent_id: Optional[UUID] = None
|
||||||
|
workspace_id: Optional[UUID] = None
|
||||||
|
queries: List[str]
|
||||||
|
top_k: int = 10
|
||||||
|
min_confidence: float = 0.5
|
||||||
|
include_global: bool = True
|
||||||
|
categories: Optional[List[MemoryCategory]] = None
|
||||||
|
|
||||||
|
|
||||||
|
class SummaryRequest(BaseModel):
|
||||||
|
"""Generate summary for thread"""
|
||||||
|
thread_id: UUID
|
||||||
|
force: bool = False # force even if under token threshold
|
||||||
|
|
||||||
|
|
||||||
|
# ============================================================================
|
||||||
|
# RESPONSE MODELS
|
||||||
|
# ============================================================================
|
||||||
|
|
||||||
|
class ThreadResponse(BaseModel):
|
||||||
|
thread_id: UUID
|
||||||
|
org_id: UUID
|
||||||
|
workspace_id: Optional[UUID]
|
||||||
|
user_id: UUID
|
||||||
|
agent_id: Optional[UUID]
|
||||||
|
title: Optional[str]
|
||||||
|
status: str
|
||||||
|
message_count: int
|
||||||
|
total_tokens: int
|
||||||
|
created_at: datetime
|
||||||
|
last_activity_at: datetime
|
||||||
|
|
||||||
|
|
||||||
|
class EventResponse(BaseModel):
|
||||||
|
event_id: UUID
|
||||||
|
thread_id: UUID
|
||||||
|
event_type: EventType
|
||||||
|
role: Optional[MessageRole]
|
||||||
|
content: Optional[str]
|
||||||
|
tool_name: Optional[str]
|
||||||
|
tool_input: Optional[dict]
|
||||||
|
tool_output: Optional[dict]
|
||||||
|
payload: dict
|
||||||
|
token_count: Optional[int]
|
||||||
|
sequence_num: int
|
||||||
|
created_at: datetime
|
||||||
|
|
||||||
|
|
||||||
|
class MemoryResponse(BaseModel):
|
||||||
|
memory_id: UUID
|
||||||
|
org_id: UUID
|
||||||
|
workspace_id: Optional[UUID]
|
||||||
|
user_id: UUID
|
||||||
|
agent_id: Optional[UUID]
|
||||||
|
category: MemoryCategory
|
||||||
|
fact_text: str
|
||||||
|
confidence: float
|
||||||
|
is_verified: bool
|
||||||
|
is_sensitive: bool
|
||||||
|
retention: RetentionPolicy
|
||||||
|
valid_from: datetime
|
||||||
|
valid_to: Optional[datetime]
|
||||||
|
last_used_at: Optional[datetime]
|
||||||
|
use_count: int
|
||||||
|
created_at: datetime
|
||||||
|
|
||||||
|
|
||||||
|
class SummaryResponse(BaseModel):
|
||||||
|
summary_id: UUID
|
||||||
|
thread_id: UUID
|
||||||
|
version: int
|
||||||
|
summary_text: str
|
||||||
|
state: dict
|
||||||
|
events_count: int
|
||||||
|
compression_ratio: Optional[float]
|
||||||
|
created_at: datetime
|
||||||
|
|
||||||
|
|
||||||
|
class RetrievalResult(BaseModel):
|
||||||
|
"""Single retrieval result"""
|
||||||
|
memory_id: UUID
|
||||||
|
fact_text: str
|
||||||
|
category: MemoryCategory
|
||||||
|
confidence: float
|
||||||
|
relevance_score: float
|
||||||
|
agent_id: Optional[UUID]
|
||||||
|
is_global: bool
|
||||||
|
|
||||||
|
|
||||||
|
class RetrievalResponse(BaseModel):
|
||||||
|
"""Retrieval response with results"""
|
||||||
|
results: List[RetrievalResult]
|
||||||
|
query_count: int
|
||||||
|
total_results: int
|
||||||
|
|
||||||
|
|
||||||
|
class ContextResponse(BaseModel):
|
||||||
|
"""Full context for agent prompt"""
|
||||||
|
thread_id: UUID
|
||||||
|
summary: Optional[SummaryResponse]
|
||||||
|
recent_messages: List[EventResponse]
|
||||||
|
retrieved_memories: List[RetrievalResult]
|
||||||
|
token_estimate: int
|
||||||
|
|
||||||
|
|
||||||
|
# ============================================================================
|
||||||
|
# INTERNAL MODELS
|
||||||
|
# ============================================================================
|
||||||
|
|
||||||
|
class EmbeddingRequest(BaseModel):
|
||||||
|
"""Internal embedding request"""
|
||||||
|
texts: List[str]
|
||||||
|
input_type: str = "search_document" # or "search_query"
|
||||||
|
|
||||||
|
|
||||||
|
class QdrantPayload(BaseModel):
|
||||||
|
"""Qdrant point payload"""
|
||||||
|
org_id: str
|
||||||
|
workspace_id: Optional[str]
|
||||||
|
user_id: str
|
||||||
|
agent_id: Optional[str]
|
||||||
|
thread_id: Optional[str]
|
||||||
|
memory_id: Optional[str]
|
||||||
|
event_id: Optional[str]
|
||||||
|
type: str # "memory", "summary", "message"
|
||||||
|
category: Optional[str]
|
||||||
|
text: str
|
||||||
|
created_at: str
|
||||||
|
|||||||
325
services/memory-service/app/vector_store.py
Normal file
325
services/memory-service/app/vector_store.py
Normal file
@@ -0,0 +1,325 @@
|
|||||||
|
"""
|
||||||
|
DAARION Memory Service - Qdrant Vector Store
|
||||||
|
"""
|
||||||
|
from typing import List, Optional, Dict, Any
|
||||||
|
from uuid import UUID, uuid4
|
||||||
|
import structlog
|
||||||
|
from qdrant_client import QdrantClient
|
||||||
|
from qdrant_client.http import models as qmodels
|
||||||
|
|
||||||
|
from .config import get_settings
|
||||||
|
from .embedding import get_query_embedding, get_document_embeddings
|
||||||
|
from .models import MemoryCategory, QdrantPayload
|
||||||
|
|
||||||
|
logger = structlog.get_logger()
|
||||||
|
settings = get_settings()
|
||||||
|
|
||||||
|
|
||||||
|
class VectorStore:
|
||||||
|
"""Qdrant vector store for semantic search"""
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
self.client = QdrantClient(
|
||||||
|
host=settings.qdrant_host,
|
||||||
|
port=settings.qdrant_port
|
||||||
|
)
|
||||||
|
self.memories_collection = settings.qdrant_collection_memories
|
||||||
|
self.messages_collection = settings.qdrant_collection_messages
|
||||||
|
|
||||||
|
async def initialize(self):
|
||||||
|
"""Initialize collections if they don't exist"""
|
||||||
|
await self._ensure_collection(
|
||||||
|
self.memories_collection,
|
||||||
|
settings.embedding_dimensions
|
||||||
|
)
|
||||||
|
await self._ensure_collection(
|
||||||
|
self.messages_collection,
|
||||||
|
settings.embedding_dimensions
|
||||||
|
)
|
||||||
|
logger.info("vector_store_initialized")
|
||||||
|
|
||||||
|
async def _ensure_collection(self, name: str, dimensions: int):
|
||||||
|
"""Create collection if it doesn't exist"""
|
||||||
|
collections = self.client.get_collections().collections
|
||||||
|
exists = any(c.name == name for c in collections)
|
||||||
|
|
||||||
|
if not exists:
|
||||||
|
self.client.create_collection(
|
||||||
|
collection_name=name,
|
||||||
|
vectors_config=qmodels.VectorParams(
|
||||||
|
size=dimensions,
|
||||||
|
distance=qmodels.Distance.COSINE
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
# Create payload indexes for filtering
|
||||||
|
self.client.create_payload_index(
|
||||||
|
collection_name=name,
|
||||||
|
field_name="org_id",
|
||||||
|
field_schema=qmodels.PayloadSchemaType.KEYWORD
|
||||||
|
)
|
||||||
|
self.client.create_payload_index(
|
||||||
|
collection_name=name,
|
||||||
|
field_name="user_id",
|
||||||
|
field_schema=qmodels.PayloadSchemaType.KEYWORD
|
||||||
|
)
|
||||||
|
self.client.create_payload_index(
|
||||||
|
collection_name=name,
|
||||||
|
field_name="agent_id",
|
||||||
|
field_schema=qmodels.PayloadSchemaType.KEYWORD
|
||||||
|
)
|
||||||
|
self.client.create_payload_index(
|
||||||
|
collection_name=name,
|
||||||
|
field_name="type",
|
||||||
|
field_schema=qmodels.PayloadSchemaType.KEYWORD
|
||||||
|
)
|
||||||
|
self.client.create_payload_index(
|
||||||
|
collection_name=name,
|
||||||
|
field_name="category",
|
||||||
|
field_schema=qmodels.PayloadSchemaType.KEYWORD
|
||||||
|
)
|
||||||
|
|
||||||
|
logger.info("collection_created", name=name, dimensions=dimensions)
|
||||||
|
|
||||||
|
async def index_memory(
|
||||||
|
self,
|
||||||
|
memory_id: UUID,
|
||||||
|
text: str,
|
||||||
|
org_id: UUID,
|
||||||
|
user_id: UUID,
|
||||||
|
category: MemoryCategory,
|
||||||
|
agent_id: Optional[UUID] = None,
|
||||||
|
workspace_id: Optional[UUID] = None,
|
||||||
|
thread_id: Optional[UUID] = None,
|
||||||
|
metadata: Dict[str, Any] = {}
|
||||||
|
) -> str:
|
||||||
|
"""
|
||||||
|
Index a memory item in Qdrant.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Qdrant point ID
|
||||||
|
"""
|
||||||
|
# Get embedding
|
||||||
|
embeddings = await get_document_embeddings([text])
|
||||||
|
if not embeddings:
|
||||||
|
raise ValueError("Failed to generate embedding")
|
||||||
|
|
||||||
|
vector = embeddings[0]
|
||||||
|
point_id = str(uuid4())
|
||||||
|
|
||||||
|
# Build payload
|
||||||
|
payload = {
|
||||||
|
"org_id": str(org_id),
|
||||||
|
"user_id": str(user_id),
|
||||||
|
"memory_id": str(memory_id),
|
||||||
|
"type": "memory",
|
||||||
|
"category": category.value,
|
||||||
|
"text": text,
|
||||||
|
**metadata
|
||||||
|
}
|
||||||
|
|
||||||
|
if agent_id:
|
||||||
|
payload["agent_id"] = str(agent_id)
|
||||||
|
if workspace_id:
|
||||||
|
payload["workspace_id"] = str(workspace_id)
|
||||||
|
if thread_id:
|
||||||
|
payload["thread_id"] = str(thread_id)
|
||||||
|
|
||||||
|
# Upsert point
|
||||||
|
self.client.upsert(
|
||||||
|
collection_name=self.memories_collection,
|
||||||
|
points=[
|
||||||
|
qmodels.PointStruct(
|
||||||
|
id=point_id,
|
||||||
|
vector=vector,
|
||||||
|
payload=payload
|
||||||
|
)
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
"memory_indexed",
|
||||||
|
memory_id=str(memory_id),
|
||||||
|
point_id=point_id,
|
||||||
|
category=category.value
|
||||||
|
)
|
||||||
|
|
||||||
|
return point_id
|
||||||
|
|
||||||
|
async def index_summary(
|
||||||
|
self,
|
||||||
|
summary_id: UUID,
|
||||||
|
text: str,
|
||||||
|
thread_id: UUID,
|
||||||
|
org_id: UUID,
|
||||||
|
user_id: UUID,
|
||||||
|
agent_id: Optional[UUID] = None,
|
||||||
|
workspace_id: Optional[UUID] = None
|
||||||
|
) -> str:
|
||||||
|
"""Index a thread summary"""
|
||||||
|
embeddings = await get_document_embeddings([text])
|
||||||
|
if not embeddings:
|
||||||
|
raise ValueError("Failed to generate embedding")
|
||||||
|
|
||||||
|
vector = embeddings[0]
|
||||||
|
point_id = str(uuid4())
|
||||||
|
|
||||||
|
payload = {
|
||||||
|
"org_id": str(org_id),
|
||||||
|
"user_id": str(user_id),
|
||||||
|
"thread_id": str(thread_id),
|
||||||
|
"summary_id": str(summary_id),
|
||||||
|
"type": "summary",
|
||||||
|
"text": text
|
||||||
|
}
|
||||||
|
|
||||||
|
if agent_id:
|
||||||
|
payload["agent_id"] = str(agent_id)
|
||||||
|
if workspace_id:
|
||||||
|
payload["workspace_id"] = str(workspace_id)
|
||||||
|
|
||||||
|
self.client.upsert(
|
||||||
|
collection_name=self.memories_collection,
|
||||||
|
points=[
|
||||||
|
qmodels.PointStruct(
|
||||||
|
id=point_id,
|
||||||
|
vector=vector,
|
||||||
|
payload=payload
|
||||||
|
)
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
logger.info("summary_indexed", summary_id=str(summary_id), point_id=point_id)
|
||||||
|
return point_id
|
||||||
|
|
||||||
|
async def search_memories(
|
||||||
|
self,
|
||||||
|
query: str,
|
||||||
|
org_id: UUID,
|
||||||
|
user_id: UUID,
|
||||||
|
agent_id: Optional[UUID] = None,
|
||||||
|
workspace_id: Optional[UUID] = None,
|
||||||
|
categories: Optional[List[MemoryCategory]] = None,
|
||||||
|
include_global: bool = True,
|
||||||
|
top_k: int = 10
|
||||||
|
) -> List[Dict[str, Any]]:
|
||||||
|
"""
|
||||||
|
Semantic search for memories.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
query: Search query
|
||||||
|
org_id: Organization filter
|
||||||
|
user_id: User filter
|
||||||
|
agent_id: Agent filter (None = include global)
|
||||||
|
workspace_id: Workspace filter
|
||||||
|
categories: Filter by memory categories
|
||||||
|
include_global: Include memories without agent_id
|
||||||
|
top_k: Number of results
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
List of results with scores
|
||||||
|
"""
|
||||||
|
# Get query embedding
|
||||||
|
query_vector = await get_query_embedding(query)
|
||||||
|
if not query_vector:
|
||||||
|
return []
|
||||||
|
|
||||||
|
# Build filter
|
||||||
|
must_conditions = [
|
||||||
|
qmodels.FieldCondition(
|
||||||
|
key="org_id",
|
||||||
|
match=qmodels.MatchValue(value=str(org_id))
|
||||||
|
),
|
||||||
|
qmodels.FieldCondition(
|
||||||
|
key="user_id",
|
||||||
|
match=qmodels.MatchValue(value=str(user_id))
|
||||||
|
)
|
||||||
|
]
|
||||||
|
|
||||||
|
if workspace_id:
|
||||||
|
must_conditions.append(
|
||||||
|
qmodels.FieldCondition(
|
||||||
|
key="workspace_id",
|
||||||
|
match=qmodels.MatchValue(value=str(workspace_id))
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
if categories:
|
||||||
|
must_conditions.append(
|
||||||
|
qmodels.FieldCondition(
|
||||||
|
key="category",
|
||||||
|
match=qmodels.MatchAny(any=[c.value for c in categories])
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
# Agent filter with global option
|
||||||
|
if agent_id and not include_global:
|
||||||
|
must_conditions.append(
|
||||||
|
qmodels.FieldCondition(
|
||||||
|
key="agent_id",
|
||||||
|
match=qmodels.MatchValue(value=str(agent_id))
|
||||||
|
)
|
||||||
|
)
|
||||||
|
elif agent_id and include_global:
|
||||||
|
# Include both agent-specific and global (no agent_id)
|
||||||
|
# This requires a should clause
|
||||||
|
pass # Will handle in separate query if needed
|
||||||
|
|
||||||
|
search_filter = qmodels.Filter(must=must_conditions)
|
||||||
|
|
||||||
|
# Search
|
||||||
|
results = self.client.search(
|
||||||
|
collection_name=self.memories_collection,
|
||||||
|
query_vector=query_vector,
|
||||||
|
query_filter=search_filter,
|
||||||
|
limit=top_k,
|
||||||
|
with_payload=True
|
||||||
|
)
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
"memory_search",
|
||||||
|
query_preview=query[:50],
|
||||||
|
results_count=len(results)
|
||||||
|
)
|
||||||
|
|
||||||
|
return [
|
||||||
|
{
|
||||||
|
"point_id": str(r.id),
|
||||||
|
"score": r.score,
|
||||||
|
**r.payload
|
||||||
|
}
|
||||||
|
for r in results
|
||||||
|
]
|
||||||
|
|
||||||
|
async def delete_memory(self, memory_id: UUID):
|
||||||
|
"""Delete memory from index by memory_id"""
|
||||||
|
self.client.delete(
|
||||||
|
collection_name=self.memories_collection,
|
||||||
|
points_selector=qmodels.FilterSelector(
|
||||||
|
filter=qmodels.Filter(
|
||||||
|
must=[
|
||||||
|
qmodels.FieldCondition(
|
||||||
|
key="memory_id",
|
||||||
|
match=qmodels.MatchValue(value=str(memory_id))
|
||||||
|
)
|
||||||
|
]
|
||||||
|
)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
logger.info("memory_deleted_from_index", memory_id=str(memory_id))
|
||||||
|
|
||||||
|
async def get_collection_stats(self) -> Dict[str, Any]:
|
||||||
|
"""Get collection statistics"""
|
||||||
|
memories_info = self.client.get_collection(self.memories_collection)
|
||||||
|
|
||||||
|
return {
|
||||||
|
"memories": {
|
||||||
|
"points_count": memories_info.points_count,
|
||||||
|
"vectors_count": memories_info.vectors_count,
|
||||||
|
"indexed_vectors_count": memories_info.indexed_vectors_count
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
# Global instance
|
||||||
|
vector_store = VectorStore()
|
||||||
@@ -1,6 +1,32 @@
|
|||||||
fastapi>=0.104.0
|
# DAARION Memory Service
|
||||||
uvicorn[standard]>=0.24.0
|
# Agent memory management with PostgreSQL + Qdrant + Cohere
|
||||||
sqlalchemy>=2.0.0
|
|
||||||
psycopg2-binary>=2.9.0
|
# Web framework
|
||||||
pydantic>=2.0.0
|
fastapi==0.109.0
|
||||||
python-dotenv>=1.0.0
|
uvicorn[standard]==0.27.0
|
||||||
|
pydantic==2.5.3
|
||||||
|
pydantic-settings==2.1.0
|
||||||
|
|
||||||
|
# Database
|
||||||
|
asyncpg==0.29.0
|
||||||
|
sqlalchemy[asyncio]==2.0.25
|
||||||
|
alembic==1.13.1
|
||||||
|
|
||||||
|
# Vector database
|
||||||
|
qdrant-client==1.7.3
|
||||||
|
|
||||||
|
# Embeddings
|
||||||
|
cohere==4.44
|
||||||
|
|
||||||
|
# Utilities
|
||||||
|
python-dotenv==1.0.0
|
||||||
|
httpx==0.26.0
|
||||||
|
tenacity==8.2.3
|
||||||
|
structlog==24.1.0
|
||||||
|
|
||||||
|
# Token counting
|
||||||
|
tiktoken==0.5.2
|
||||||
|
|
||||||
|
# Testing
|
||||||
|
pytest==7.4.4
|
||||||
|
pytest-asyncio==0.23.3
|
||||||
|
|||||||
Reference in New Issue
Block a user