🧠 Add Agent Memory System with PostgreSQL + Qdrant + Cohere
Features: - Three-tier memory architecture (short/mid/long-term) - PostgreSQL schema for conversations, events, memories - Qdrant vector database for semantic search - Cohere embeddings (embed-multilingual-v3.0, 1024 dims) - FastAPI Memory Service with full CRUD - External Secrets integration with Vault - Kubernetes deployment manifests Components: - infrastructure/database/agent-memory-schema.sql - infrastructure/kubernetes/apps/qdrant/ - infrastructure/kubernetes/apps/memory-service/ - services/memory-service/ (FastAPI app) Also includes: - External Secrets Operator - Traefik Ingress Controller - Cert-Manager with Let's Encrypt - ArgoCD for GitOps
This commit is contained in:
Apple
257
infrastructure/ansible/playbooks/postgres-ha.yml
Normal file
257
infrastructure/ansible/playbooks/postgres-ha.yml
Normal file
@@ -0,0 +1,257 @@
|
||||
# DAARION Network - PostgreSQL HA Setup
|
||||
# Patroni + PgBouncer + pgBackRest + Monitoring
|
||||
---
|
||||
# =============================================================================
|
||||
# POSTGRESQL HA WITH PATRONI
|
||||
# =============================================================================
|
||||
- name: Setup PostgreSQL HA Cluster
|
||||
hosts: database_nodes
|
||||
become: yes
|
||||
|
||||
vars:
|
||||
# Patroni
|
||||
patroni_version: "3.2.0"
|
||||
patroni_scope: "daarion-cluster"
|
||||
patroni_namespace: "/daarion"
|
||||
|
||||
# PostgreSQL
|
||||
postgres_version: "16"
|
||||
postgres_data_dir: "/var/lib/postgresql/{{ postgres_version }}/main"
|
||||
postgres_config_dir: "/etc/postgresql/{{ postgres_version }}/main"
|
||||
|
||||
# PgBouncer
|
||||
pgbouncer_port: 6432
|
||||
pgbouncer_max_client_conn: 1000
|
||||
pgbouncer_default_pool_size: 50
|
||||
|
||||
# Backup
|
||||
pgbackrest_repo_path: "/var/lib/pgbackrest"
|
||||
pgbackrest_s3_bucket: "daarion-backups"
|
||||
|
||||
# Consul for DCS
|
||||
consul_host: "{{ hostvars[groups['masters'][0]].ansible_host }}"
|
||||
|
||||
tasks:
|
||||
# =========================================================================
|
||||
# PREREQUISITES
|
||||
# =========================================================================
|
||||
- name: Add PostgreSQL APT repository
|
||||
shell: |
|
||||
curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor -o /usr/share/keyrings/postgresql-keyring.gpg
|
||||
echo "deb [signed-by=/usr/share/keyrings/postgresql-keyring.gpg] http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list
|
||||
args:
|
||||
creates: /etc/apt/sources.list.d/pgdg.list
|
||||
|
||||
- name: Update apt cache
|
||||
apt:
|
||||
update_cache: yes
|
||||
|
||||
- name: Install PostgreSQL and dependencies
|
||||
apt:
|
||||
name:
|
||||
- postgresql-{{ postgres_version }}
|
||||
- postgresql-contrib-{{ postgres_version }}
|
||||
- python3-pip
|
||||
- python3-psycopg2
|
||||
- python3-consul
|
||||
- pgbouncer
|
||||
- pgbackrest
|
||||
state: present
|
||||
|
||||
# =========================================================================
|
||||
# PATRONI INSTALLATION
|
||||
# =========================================================================
|
||||
- name: Install Patroni
|
||||
pip:
|
||||
name:
|
||||
- patroni[consul]=={{ patroni_version }}
|
||||
- python-consul
|
||||
state: present
|
||||
|
||||
- name: Create Patroni directories
|
||||
file:
|
||||
path: "{{ item }}"
|
||||
state: directory
|
||||
owner: postgres
|
||||
group: postgres
|
||||
mode: '0750'
|
||||
loop:
|
||||
- /etc/patroni
|
||||
- /var/log/patroni
|
||||
|
||||
- name: Configure Patroni
|
||||
template:
|
||||
src: templates/patroni.yml.j2
|
||||
dest: /etc/patroni/patroni.yml
|
||||
owner: postgres
|
||||
group: postgres
|
||||
mode: '0640'
|
||||
notify: restart patroni
|
||||
|
||||
- name: Create Patroni systemd service
|
||||
copy:
|
||||
dest: /etc/systemd/system/patroni.service
|
||||
content: |
|
||||
[Unit]
|
||||
Description=Patroni PostgreSQL Cluster Manager
|
||||
After=network.target consul.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=postgres
|
||||
Group=postgres
|
||||
ExecStart=/usr/local/bin/patroni /etc/patroni/patroni.yml
|
||||
ExecReload=/bin/kill -HUP $MAINPID
|
||||
KillMode=process
|
||||
TimeoutSec=30
|
||||
Restart=on-failure
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
notify:
|
||||
- reload systemd
|
||||
- restart patroni
|
||||
|
||||
# =========================================================================
|
||||
# PGBOUNCER
|
||||
# =========================================================================
|
||||
- name: Configure PgBouncer
|
||||
template:
|
||||
src: templates/pgbouncer.ini.j2
|
||||
dest: /etc/pgbouncer/pgbouncer.ini
|
||||
owner: postgres
|
||||
group: postgres
|
||||
mode: '0640'
|
||||
notify: restart pgbouncer
|
||||
|
||||
- name: Configure PgBouncer userlist
|
||||
copy:
|
||||
dest: /etc/pgbouncer/userlist.txt
|
||||
content: |
|
||||
"{{ postgres_user }}" "{{ postgres_password }}"
|
||||
"pgbouncer" "{{ pgbouncer_password | default('pgbouncer_secret') }}"
|
||||
owner: postgres
|
||||
group: postgres
|
||||
mode: '0600'
|
||||
notify: restart pgbouncer
|
||||
|
||||
- name: Enable PgBouncer
|
||||
service:
|
||||
name: pgbouncer
|
||||
enabled: yes
|
||||
state: started
|
||||
|
||||
# =========================================================================
|
||||
# PGBACKREST
|
||||
# =========================================================================
|
||||
- name: Create pgBackRest directories
|
||||
file:
|
||||
path: "{{ item }}"
|
||||
state: directory
|
||||
owner: postgres
|
||||
group: postgres
|
||||
mode: '0750'
|
||||
loop:
|
||||
- "{{ pgbackrest_repo_path }}"
|
||||
- /var/log/pgbackrest
|
||||
|
||||
- name: Configure pgBackRest
|
||||
template:
|
||||
src: templates/pgbackrest.conf.j2
|
||||
dest: /etc/pgbackrest.conf
|
||||
owner: postgres
|
||||
group: postgres
|
||||
mode: '0640'
|
||||
|
||||
- name: Setup backup cron
|
||||
cron:
|
||||
name: "Daily PostgreSQL backup"
|
||||
hour: "2"
|
||||
minute: "0"
|
||||
user: postgres
|
||||
job: "pgbackrest --stanza={{ patroni_scope }} --type=diff backup >> /var/log/pgbackrest/backup.log 2>&1"
|
||||
|
||||
- name: Setup weekly full backup
|
||||
cron:
|
||||
name: "Weekly full PostgreSQL backup"
|
||||
weekday: "0"
|
||||
hour: "3"
|
||||
minute: "0"
|
||||
user: postgres
|
||||
job: "pgbackrest --stanza={{ patroni_scope }} --type=full backup >> /var/log/pgbackrest/backup.log 2>&1"
|
||||
|
||||
# =========================================================================
|
||||
# MONITORING
|
||||
# =========================================================================
|
||||
- name: Install postgres_exporter
|
||||
shell: |
|
||||
curl -sL https://github.com/prometheus-community/postgres_exporter/releases/download/v0.15.0/postgres_exporter-0.15.0.linux-amd64.tar.gz | tar xz
|
||||
mv postgres_exporter-0.15.0.linux-amd64/postgres_exporter /usr/local/bin/
|
||||
rm -rf postgres_exporter-0.15.0.linux-amd64
|
||||
args:
|
||||
creates: /usr/local/bin/postgres_exporter
|
||||
|
||||
- name: Create postgres_exporter systemd service
|
||||
copy:
|
||||
dest: /etc/systemd/system/postgres_exporter.service
|
||||
content: |
|
||||
[Unit]
|
||||
Description=Prometheus PostgreSQL Exporter
|
||||
After=network.target postgresql.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=postgres
|
||||
Environment="DATA_SOURCE_NAME=postgresql://{{ postgres_user }}:{{ postgres_password }}@localhost:5432/{{ postgres_db }}?sslmode=disable"
|
||||
ExecStart=/usr/local/bin/postgres_exporter --web.listen-address=:9187
|
||||
Restart=on-failure
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
notify:
|
||||
- reload systemd
|
||||
- restart postgres_exporter
|
||||
|
||||
- name: Enable postgres_exporter
|
||||
service:
|
||||
name: postgres_exporter
|
||||
enabled: yes
|
||||
state: started
|
||||
|
||||
# =========================================================================
|
||||
# VERIFICATION
|
||||
# =========================================================================
|
||||
- name: Show PostgreSQL HA status
|
||||
debug:
|
||||
msg: |
|
||||
PostgreSQL HA Setup Complete!
|
||||
|
||||
Components:
|
||||
- Patroni: Cluster management
|
||||
- PgBouncer: Connection pooling (port {{ pgbouncer_port }})
|
||||
- pgBackRest: Backups
|
||||
- postgres_exporter: Metrics (port 9187)
|
||||
|
||||
Connection strings:
|
||||
- Direct: postgresql://{{ postgres_user }}@{{ ansible_host }}:5432/{{ postgres_db }}
|
||||
- Pooled: postgresql://{{ postgres_user }}@{{ ansible_host }}:{{ pgbouncer_port }}/{{ postgres_db }}
|
||||
|
||||
handlers:
|
||||
- name: reload systemd
|
||||
systemd:
|
||||
daemon_reload: yes
|
||||
|
||||
- name: restart patroni
|
||||
service:
|
||||
name: patroni
|
||||
state: restarted
|
||||
|
||||
- name: restart pgbouncer
|
||||
service:
|
||||
name: pgbouncer
|
||||
state: restarted
|
||||
|
||||
- name: restart postgres_exporter
|
||||
service:
|
||||
name: postgres_exporter
|
||||
state: restarted
|
||||
120
infrastructure/ansible/templates/patroni.yml.j2
Normal file
120
infrastructure/ansible/templates/patroni.yml.j2
Normal file
@@ -0,0 +1,120 @@
|
||||
# Patroni Configuration for {{ inventory_hostname }}
|
||||
# Generated by Ansible
|
||||
|
||||
scope: {{ patroni_scope }}
|
||||
namespace: {{ patroni_namespace }}
|
||||
name: {{ inventory_hostname }}
|
||||
|
||||
restapi:
|
||||
listen: 0.0.0.0:8008
|
||||
connect_address: {{ ansible_host }}:8008
|
||||
|
||||
consul:
|
||||
host: {{ consul_host }}:8500
|
||||
register_service: true
|
||||
|
||||
bootstrap:
|
||||
dcs:
|
||||
ttl: 30
|
||||
loop_wait: 10
|
||||
retry_timeout: 10
|
||||
maximum_lag_on_failover: 1048576
|
||||
postgresql:
|
||||
use_pg_rewind: true
|
||||
use_slots: true
|
||||
parameters:
|
||||
# Performance
|
||||
max_connections: 200
|
||||
shared_buffers: 256MB
|
||||
effective_cache_size: 768MB
|
||||
maintenance_work_mem: 64MB
|
||||
checkpoint_completion_target: 0.9
|
||||
wal_buffers: 16MB
|
||||
default_statistics_target: 100
|
||||
random_page_cost: 1.1
|
||||
effective_io_concurrency: 200
|
||||
work_mem: 2621kB
|
||||
huge_pages: off
|
||||
min_wal_size: 1GB
|
||||
max_wal_size: 4GB
|
||||
max_worker_processes: 4
|
||||
max_parallel_workers_per_gather: 2
|
||||
max_parallel_workers: 4
|
||||
max_parallel_maintenance_workers: 2
|
||||
|
||||
# Replication
|
||||
wal_level: replica
|
||||
hot_standby: "on"
|
||||
max_wal_senders: 10
|
||||
max_replication_slots: 10
|
||||
hot_standby_feedback: "on"
|
||||
|
||||
# Logging
|
||||
log_destination: 'stderr'
|
||||
logging_collector: 'on'
|
||||
log_directory: 'log'
|
||||
log_filename: 'postgresql-%Y-%m-%d_%H%M%S.log'
|
||||
log_rotation_age: '1d'
|
||||
log_rotation_size: '100MB'
|
||||
log_min_duration_statement: 1000
|
||||
log_checkpoints: 'on'
|
||||
log_connections: 'on'
|
||||
log_disconnections: 'on'
|
||||
log_lock_waits: 'on'
|
||||
|
||||
# Archive (for pgBackRest)
|
||||
archive_mode: "on"
|
||||
archive_command: 'pgbackrest --stanza={{ patroni_scope }} archive-push %p'
|
||||
|
||||
initdb:
|
||||
- encoding: UTF8
|
||||
- data-checksums
|
||||
|
||||
pg_hba:
|
||||
- host replication replicator 0.0.0.0/0 scram-sha-256
|
||||
- host all all 0.0.0.0/0 scram-sha-256
|
||||
|
||||
users:
|
||||
{{ postgres_user }}:
|
||||
password: {{ postgres_password }}
|
||||
options:
|
||||
- createrole
|
||||
- createdb
|
||||
replicator:
|
||||
password: {{ replicator_password | default('replicator_secret') }}
|
||||
options:
|
||||
- replication
|
||||
|
||||
postgresql:
|
||||
listen: 0.0.0.0:5432
|
||||
connect_address: {{ ansible_host }}:5432
|
||||
data_dir: {{ postgres_data_dir }}
|
||||
bin_dir: /usr/lib/postgresql/{{ postgres_version }}/bin
|
||||
config_dir: {{ postgres_config_dir }}
|
||||
pgpass: /var/lib/postgresql/.pgpass
|
||||
|
||||
authentication:
|
||||
replication:
|
||||
username: replicator
|
||||
password: {{ replicator_password | default('replicator_secret') }}
|
||||
superuser:
|
||||
username: postgres
|
||||
password: {{ postgres_superuser_password | default('postgres_secret') }}
|
||||
rewind:
|
||||
username: rewind
|
||||
password: {{ rewind_password | default('rewind_secret') }}
|
||||
|
||||
parameters:
|
||||
unix_socket_directories: '/var/run/postgresql'
|
||||
|
||||
pg_hba:
|
||||
- local all all peer
|
||||
- host all all 127.0.0.1/32 scram-sha-256
|
||||
- host all all 0.0.0.0/0 scram-sha-256
|
||||
- host replication replicator 0.0.0.0/0 scram-sha-256
|
||||
|
||||
tags:
|
||||
nofailover: false
|
||||
noloadbalance: false
|
||||
clonefrom: false
|
||||
nosync: false
|
||||
40
infrastructure/ansible/templates/pgbackrest.conf.j2
Normal file
40
infrastructure/ansible/templates/pgbackrest.conf.j2
Normal file
@@ -0,0 +1,40 @@
|
||||
# pgBackRest Configuration for {{ inventory_hostname }}
|
||||
# Generated by Ansible
|
||||
|
||||
[global]
|
||||
# Repository
|
||||
repo1-path={{ pgbackrest_repo_path }}
|
||||
repo1-retention-full=2
|
||||
repo1-retention-diff=7
|
||||
|
||||
# S3 (optional - uncomment for cloud backups)
|
||||
# repo2-type=s3
|
||||
# repo2-path=/backup
|
||||
# repo2-s3-bucket={{ pgbackrest_s3_bucket }}
|
||||
# repo2-s3-endpoint=s3.eu-central-1.amazonaws.com
|
||||
# repo2-s3-region=eu-central-1
|
||||
# repo2-s3-key={{ pgbackrest_s3_key | default('') }}
|
||||
# repo2-s3-key-secret={{ pgbackrest_s3_secret | default('') }}
|
||||
# repo2-retention-full=4
|
||||
# repo2-retention-diff=14
|
||||
|
||||
# Compression
|
||||
compress-type=zst
|
||||
compress-level=3
|
||||
|
||||
# Parallel
|
||||
process-max=4
|
||||
|
||||
# Logging
|
||||
log-level-console=info
|
||||
log-level-file=detail
|
||||
log-path=/var/log/pgbackrest
|
||||
|
||||
# Archive
|
||||
archive-async=y
|
||||
archive-push-queue-max=4GB
|
||||
|
||||
[{{ patroni_scope }}]
|
||||
pg1-path={{ postgres_data_dir }}
|
||||
pg1-port=5432
|
||||
pg1-user=postgres
|
||||
44
infrastructure/ansible/templates/pgbouncer.ini.j2
Normal file
44
infrastructure/ansible/templates/pgbouncer.ini.j2
Normal file
@@ -0,0 +1,44 @@
|
||||
# PgBouncer Configuration for {{ inventory_hostname }}
|
||||
# Generated by Ansible
|
||||
|
||||
[databases]
|
||||
{{ postgres_db }} = host=127.0.0.1 port=5432 dbname={{ postgres_db }}
|
||||
* = host=127.0.0.1 port=5432
|
||||
|
||||
[pgbouncer]
|
||||
listen_addr = 0.0.0.0
|
||||
listen_port = {{ pgbouncer_port }}
|
||||
unix_socket_dir = /var/run/postgresql
|
||||
|
||||
auth_type = scram-sha-256
|
||||
auth_file = /etc/pgbouncer/userlist.txt
|
||||
|
||||
# Pool settings
|
||||
pool_mode = transaction
|
||||
max_client_conn = {{ pgbouncer_max_client_conn }}
|
||||
default_pool_size = {{ pgbouncer_default_pool_size }}
|
||||
min_pool_size = 10
|
||||
reserve_pool_size = 5
|
||||
reserve_pool_timeout = 3
|
||||
|
||||
# Timeouts
|
||||
server_connect_timeout = 15
|
||||
server_idle_timeout = 600
|
||||
server_lifetime = 3600
|
||||
client_idle_timeout = 0
|
||||
client_login_timeout = 60
|
||||
query_timeout = 0
|
||||
query_wait_timeout = 120
|
||||
|
||||
# Logging
|
||||
log_connections = 1
|
||||
log_disconnections = 1
|
||||
log_pooler_errors = 1
|
||||
stats_period = 60
|
||||
|
||||
# Admin
|
||||
admin_users = pgbouncer,{{ postgres_user }}
|
||||
stats_users = pgbouncer,{{ postgres_user }}
|
||||
|
||||
# Security
|
||||
ignore_startup_parameters = extra_float_digits
|
||||
458
infrastructure/database/agent-memory-schema.sql
Normal file
458
infrastructure/database/agent-memory-schema.sql
Normal file
@@ -0,0 +1,458 @@
|
||||
-- ============================================================================
|
||||
-- DAARION Agent Memory System - PostgreSQL Schema
|
||||
-- Version: 1.0.0
|
||||
-- Date: 2026-01-10
|
||||
--
|
||||
-- Трирівнева пам'ять агентів:
|
||||
-- 1. Short-term: conversation_events (робочий буфер)
|
||||
-- 2. Mid-term: thread_summaries (сесійна/тематична)
|
||||
-- 3. Long-term: long_term_memory_items (персональна/проектна)
|
||||
-- ============================================================================
|
||||
|
||||
-- Extensions
|
||||
CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
|
||||
CREATE EXTENSION IF NOT EXISTS "pgcrypto";
|
||||
|
||||
-- ============================================================================
|
||||
-- CORE ENTITIES
|
||||
-- ============================================================================
|
||||
|
||||
-- Organizations (top-level tenant)
|
||||
CREATE TABLE IF NOT EXISTS organizations (
|
||||
org_id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
name VARCHAR(255) NOT NULL,
|
||||
settings JSONB DEFAULT '{}',
|
||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
|
||||
-- Workspaces (projects within org)
|
||||
CREATE TABLE IF NOT EXISTS workspaces (
|
||||
workspace_id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
org_id UUID NOT NULL REFERENCES organizations(org_id) ON DELETE CASCADE,
|
||||
name VARCHAR(255) NOT NULL,
|
||||
description TEXT,
|
||||
settings JSONB DEFAULT '{}',
|
||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE INDEX idx_workspaces_org ON workspaces(org_id);
|
||||
|
||||
-- Users
|
||||
CREATE TABLE IF NOT EXISTS users (
|
||||
user_id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
org_id UUID NOT NULL REFERENCES organizations(org_id) ON DELETE CASCADE,
|
||||
external_id VARCHAR(255), -- for SSO/OAuth mapping
|
||||
email VARCHAR(255),
|
||||
display_name VARCHAR(255),
|
||||
preferences JSONB DEFAULT '{}',
|
||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
UNIQUE(org_id, external_id)
|
||||
);
|
||||
|
||||
CREATE INDEX idx_users_org ON users(org_id);
|
||||
CREATE INDEX idx_users_email ON users(email);
|
||||
|
||||
-- Agents
|
||||
CREATE TABLE IF NOT EXISTS agents (
|
||||
agent_id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
org_id UUID NOT NULL REFERENCES organizations(org_id) ON DELETE CASCADE,
|
||||
name VARCHAR(255) NOT NULL,
|
||||
type VARCHAR(50) NOT NULL, -- 'assistant', 'specialist', 'coordinator'
|
||||
model VARCHAR(100), -- 'claude-3-opus', 'gpt-4', etc.
|
||||
system_prompt TEXT,
|
||||
capabilities JSONB DEFAULT '[]', -- ['code', 'search', 'memory', 'tools']
|
||||
settings JSONB DEFAULT '{}',
|
||||
is_active BOOLEAN DEFAULT true,
|
||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE INDEX idx_agents_org ON agents(org_id);
|
||||
CREATE INDEX idx_agents_type ON agents(type);
|
||||
|
||||
-- ============================================================================
|
||||
-- CONVERSATION LAYER (Short-term Memory)
|
||||
-- ============================================================================
|
||||
|
||||
-- Conversation threads
|
||||
CREATE TABLE IF NOT EXISTS conversation_threads (
|
||||
thread_id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
org_id UUID NOT NULL REFERENCES organizations(org_id) ON DELETE CASCADE,
|
||||
workspace_id UUID REFERENCES workspaces(workspace_id) ON DELETE SET NULL,
|
||||
user_id UUID NOT NULL REFERENCES users(user_id) ON DELETE CASCADE,
|
||||
agent_id UUID REFERENCES agents(agent_id) ON DELETE SET NULL,
|
||||
|
||||
title VARCHAR(500),
|
||||
status VARCHAR(50) DEFAULT 'active', -- 'active', 'archived', 'completed'
|
||||
|
||||
-- Metadata
|
||||
tags JSONB DEFAULT '[]',
|
||||
metadata JSONB DEFAULT '{}',
|
||||
|
||||
-- Stats
|
||||
message_count INTEGER DEFAULT 0,
|
||||
total_tokens INTEGER DEFAULT 0,
|
||||
|
||||
-- Timestamps
|
||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
last_activity_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
archived_at TIMESTAMPTZ
|
||||
);
|
||||
|
||||
CREATE INDEX idx_threads_org ON conversation_threads(org_id);
|
||||
CREATE INDEX idx_threads_workspace ON conversation_threads(workspace_id);
|
||||
CREATE INDEX idx_threads_user ON conversation_threads(user_id);
|
||||
CREATE INDEX idx_threads_agent ON conversation_threads(agent_id);
|
||||
CREATE INDEX idx_threads_status ON conversation_threads(status);
|
||||
CREATE INDEX idx_threads_last_activity ON conversation_threads(last_activity_at DESC);
|
||||
|
||||
-- Conversation events (Event Log - source of truth)
|
||||
CREATE TABLE IF NOT EXISTS conversation_events (
|
||||
event_id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
thread_id UUID NOT NULL REFERENCES conversation_threads(thread_id) ON DELETE CASCADE,
|
||||
|
||||
-- Event type
|
||||
event_type VARCHAR(50) NOT NULL, -- 'message', 'tool_call', 'tool_result', 'decision', 'summary', 'memory_write', 'memory_retract', 'error'
|
||||
|
||||
-- For messages
|
||||
role VARCHAR(20), -- 'user', 'assistant', 'system', 'tool'
|
||||
content TEXT,
|
||||
|
||||
-- For tool calls
|
||||
tool_name VARCHAR(100),
|
||||
tool_input JSONB,
|
||||
tool_output JSONB,
|
||||
|
||||
-- Structured payload (flexible)
|
||||
payload JSONB DEFAULT '{}',
|
||||
|
||||
-- Token tracking
|
||||
token_count INTEGER,
|
||||
|
||||
-- Metadata
|
||||
model_used VARCHAR(100),
|
||||
latency_ms INTEGER,
|
||||
metadata JSONB DEFAULT '{}',
|
||||
|
||||
-- Ordering
|
||||
sequence_num SERIAL,
|
||||
created_at TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE INDEX idx_events_thread ON conversation_events(thread_id);
|
||||
CREATE INDEX idx_events_type ON conversation_events(event_type);
|
||||
CREATE INDEX idx_events_created ON conversation_events(created_at DESC);
|
||||
CREATE INDEX idx_events_thread_seq ON conversation_events(thread_id, sequence_num);
|
||||
|
||||
-- ============================================================================
|
||||
-- SUMMARY LAYER (Mid-term Memory)
|
||||
-- ============================================================================
|
||||
|
||||
-- Thread summaries (rolling compression)
|
||||
CREATE TABLE IF NOT EXISTS thread_summaries (
|
||||
summary_id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
thread_id UUID NOT NULL REFERENCES conversation_threads(thread_id) ON DELETE CASCADE,
|
||||
|
||||
-- Version tracking
|
||||
version INTEGER NOT NULL DEFAULT 1,
|
||||
|
||||
-- Summary content
|
||||
summary_text TEXT NOT NULL,
|
||||
|
||||
-- Structured state
|
||||
state JSONB DEFAULT '{}', -- {goals: [], decisions: [], open_questions: [], next_steps: [], key_facts: []}
|
||||
|
||||
-- Coverage
|
||||
events_from_seq INTEGER, -- first event sequence included
|
||||
events_to_seq INTEGER, -- last event sequence included
|
||||
events_count INTEGER,
|
||||
|
||||
-- Token info
|
||||
original_tokens INTEGER, -- tokens before compression
|
||||
summary_tokens INTEGER, -- tokens after compression
|
||||
compression_ratio FLOAT,
|
||||
|
||||
-- Timestamps
|
||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
|
||||
UNIQUE(thread_id, version)
|
||||
);
|
||||
|
||||
CREATE INDEX idx_summaries_thread ON thread_summaries(thread_id);
|
||||
CREATE INDEX idx_summaries_thread_version ON thread_summaries(thread_id, version DESC);
|
||||
|
||||
-- ============================================================================
|
||||
-- LONG-TERM MEMORY LAYER
|
||||
-- ============================================================================
|
||||
|
||||
-- Memory categories enum
|
||||
CREATE TYPE memory_category AS ENUM (
|
||||
'preference', -- user likes/dislikes
|
||||
'identity', -- who the user is
|
||||
'constraint', -- limitations, rules
|
||||
'project_fact', -- project-specific knowledge
|
||||
'relationship', -- connections between entities
|
||||
'skill', -- user capabilities
|
||||
'goal', -- user objectives
|
||||
'context', -- situational info
|
||||
'feedback' -- user corrections/confirmations
|
||||
);
|
||||
|
||||
-- Retention policy enum
|
||||
CREATE TYPE retention_policy AS ENUM (
|
||||
'permanent', -- keep until explicitly deleted
|
||||
'session', -- delete after session
|
||||
'ttl_days', -- delete after N days
|
||||
'until_revoked' -- keep until user revokes
|
||||
);
|
||||
|
||||
-- Long-term memory items
|
||||
CREATE TABLE IF NOT EXISTS long_term_memory_items (
|
||||
memory_id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
|
||||
-- Scope (all nullable for flexibility)
|
||||
org_id UUID REFERENCES organizations(org_id) ON DELETE CASCADE,
|
||||
workspace_id UUID REFERENCES workspaces(workspace_id) ON DELETE SET NULL,
|
||||
user_id UUID REFERENCES users(user_id) ON DELETE CASCADE,
|
||||
agent_id UUID REFERENCES agents(agent_id) ON DELETE SET NULL, -- null = global for user
|
||||
|
||||
-- Content
|
||||
category memory_category NOT NULL,
|
||||
fact_text TEXT NOT NULL, -- atomic statement
|
||||
fact_embedding_id VARCHAR(100), -- reference to Qdrant point ID
|
||||
|
||||
-- Confidence & validation
|
||||
confidence FLOAT DEFAULT 0.8 CHECK (confidence >= 0 AND confidence <= 1),
|
||||
is_verified BOOLEAN DEFAULT false,
|
||||
verification_count INTEGER DEFAULT 0,
|
||||
|
||||
-- Source tracking
|
||||
source_event_id UUID REFERENCES conversation_events(event_id) ON DELETE SET NULL,
|
||||
source_thread_id UUID REFERENCES conversation_threads(thread_id) ON DELETE SET NULL,
|
||||
extraction_method VARCHAR(50), -- 'explicit', 'inferred', 'confirmed', 'imported'
|
||||
|
||||
-- Lifecycle
|
||||
valid_from TIMESTAMPTZ DEFAULT NOW(),
|
||||
valid_to TIMESTAMPTZ, -- null = currently valid
|
||||
last_confirmed_at TIMESTAMPTZ,
|
||||
last_used_at TIMESTAMPTZ,
|
||||
use_count INTEGER DEFAULT 0,
|
||||
|
||||
-- Privacy & retention
|
||||
is_sensitive BOOLEAN DEFAULT false,
|
||||
retention retention_policy DEFAULT 'until_revoked',
|
||||
ttl_days INTEGER, -- if retention = 'ttl_days'
|
||||
|
||||
-- Metadata
|
||||
tags JSONB DEFAULT '[]',
|
||||
metadata JSONB DEFAULT '{}',
|
||||
|
||||
-- Timestamps
|
||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
|
||||
-- Indexes for memory retrieval
|
||||
CREATE INDEX idx_memory_org ON long_term_memory_items(org_id);
|
||||
CREATE INDEX idx_memory_workspace ON long_term_memory_items(workspace_id);
|
||||
CREATE INDEX idx_memory_user ON long_term_memory_items(user_id);
|
||||
CREATE INDEX idx_memory_agent ON long_term_memory_items(agent_id);
|
||||
CREATE INDEX idx_memory_category ON long_term_memory_items(category);
|
||||
CREATE INDEX idx_memory_user_agent ON long_term_memory_items(user_id, agent_id);
|
||||
CREATE INDEX idx_memory_valid ON long_term_memory_items(valid_from, valid_to);
|
||||
CREATE INDEX idx_memory_confidence ON long_term_memory_items(confidence DESC);
|
||||
CREATE INDEX idx_memory_created ON long_term_memory_items(created_at DESC);
|
||||
|
||||
-- GIN index for tags search
|
||||
CREATE INDEX idx_memory_tags ON long_term_memory_items USING GIN (tags);
|
||||
|
||||
-- Memory feedback (user corrections)
|
||||
CREATE TABLE IF NOT EXISTS memory_feedback (
|
||||
feedback_id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
memory_id UUID NOT NULL REFERENCES long_term_memory_items(memory_id) ON DELETE CASCADE,
|
||||
user_id UUID NOT NULL REFERENCES users(user_id) ON DELETE CASCADE,
|
||||
|
||||
action VARCHAR(20) NOT NULL, -- 'confirm', 'reject', 'edit', 'delete'
|
||||
old_value TEXT,
|
||||
new_value TEXT,
|
||||
reason TEXT,
|
||||
|
||||
created_at TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE INDEX idx_feedback_memory ON memory_feedback(memory_id);
|
||||
CREATE INDEX idx_feedback_user ON memory_feedback(user_id);
|
||||
|
||||
-- ============================================================================
|
||||
-- HELPER VIEWS
|
||||
-- ============================================================================
|
||||
|
||||
-- Active memories for a user (across all agents)
|
||||
CREATE OR REPLACE VIEW v_active_user_memories AS
|
||||
SELECT
|
||||
m.*,
|
||||
a.name as agent_name
|
||||
FROM long_term_memory_items m
|
||||
LEFT JOIN agents a ON m.agent_id = a.agent_id
|
||||
WHERE m.valid_to IS NULL
|
||||
AND m.confidence >= 0.5
|
||||
ORDER BY m.confidence DESC, m.last_used_at DESC NULLS LAST;
|
||||
|
||||
-- Recent conversations with summaries
|
||||
CREATE OR REPLACE VIEW v_recent_conversations AS
|
||||
SELECT
|
||||
t.thread_id,
|
||||
t.title,
|
||||
t.user_id,
|
||||
t.agent_id,
|
||||
t.message_count,
|
||||
t.last_activity_at,
|
||||
s.summary_text,
|
||||
s.state
|
||||
FROM conversation_threads t
|
||||
LEFT JOIN LATERAL (
|
||||
SELECT summary_text, state
|
||||
FROM thread_summaries
|
||||
WHERE thread_id = t.thread_id
|
||||
ORDER BY version DESC
|
||||
LIMIT 1
|
||||
) s ON true
|
||||
WHERE t.status = 'active'
|
||||
ORDER BY t.last_activity_at DESC;
|
||||
|
||||
-- ============================================================================
|
||||
-- FUNCTIONS
|
||||
-- ============================================================================
|
||||
|
||||
-- Update thread stats after new event
|
||||
CREATE OR REPLACE FUNCTION update_thread_stats()
|
||||
RETURNS TRIGGER AS $$
|
||||
BEGIN
|
||||
UPDATE conversation_threads
|
||||
SET
|
||||
message_count = message_count + CASE WHEN NEW.event_type = 'message' THEN 1 ELSE 0 END,
|
||||
total_tokens = total_tokens + COALESCE(NEW.token_count, 0),
|
||||
last_activity_at = NOW()
|
||||
WHERE thread_id = NEW.thread_id;
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
CREATE TRIGGER trg_update_thread_stats
|
||||
AFTER INSERT ON conversation_events
|
||||
FOR EACH ROW EXECUTE FUNCTION update_thread_stats();
|
||||
|
||||
-- Update memory usage stats
|
||||
CREATE OR REPLACE FUNCTION update_memory_usage()
|
||||
RETURNS TRIGGER AS $$
|
||||
BEGIN
|
||||
UPDATE long_term_memory_items
|
||||
SET
|
||||
use_count = use_count + 1,
|
||||
last_used_at = NOW()
|
||||
WHERE memory_id = NEW.memory_id;
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
-- Auto-update updated_at
|
||||
CREATE OR REPLACE FUNCTION update_updated_at()
|
||||
RETURNS TRIGGER AS $$
|
||||
BEGIN
|
||||
NEW.updated_at = NOW();
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
CREATE TRIGGER trg_organizations_updated
|
||||
BEFORE UPDATE ON organizations
|
||||
FOR EACH ROW EXECUTE FUNCTION update_updated_at();
|
||||
|
||||
CREATE TRIGGER trg_workspaces_updated
|
||||
BEFORE UPDATE ON workspaces
|
||||
FOR EACH ROW EXECUTE FUNCTION update_updated_at();
|
||||
|
||||
CREATE TRIGGER trg_users_updated
|
||||
BEFORE UPDATE ON users
|
||||
FOR EACH ROW EXECUTE FUNCTION update_updated_at();
|
||||
|
||||
CREATE TRIGGER trg_agents_updated
|
||||
BEFORE UPDATE ON agents
|
||||
FOR EACH ROW EXECUTE FUNCTION update_updated_at();
|
||||
|
||||
CREATE TRIGGER trg_memory_updated
|
||||
BEFORE UPDATE ON long_term_memory_items
|
||||
FOR EACH ROW EXECUTE FUNCTION update_updated_at();
|
||||
|
||||
-- ============================================================================
|
||||
-- INITIAL DATA
|
||||
-- ============================================================================
|
||||
|
||||
-- Default organization
|
||||
INSERT INTO organizations (org_id, name, settings)
|
||||
VALUES (
|
||||
'a0000000-0000-0000-0000-000000000001',
|
||||
'DAARION',
|
||||
'{"tier": "enterprise", "features": ["memory", "multi-agent", "knowledge-base"]}'
|
||||
) ON CONFLICT DO NOTHING;
|
||||
|
||||
-- Default workspace
|
||||
INSERT INTO workspaces (workspace_id, org_id, name, description)
|
||||
VALUES (
|
||||
'b0000000-0000-0000-0000-000000000001',
|
||||
'a0000000-0000-0000-0000-000000000001',
|
||||
'MicroDAO',
|
||||
'Main development workspace for DAARION project'
|
||||
) ON CONFLICT DO NOTHING;
|
||||
|
||||
-- Default user (Ivan)
|
||||
INSERT INTO users (user_id, org_id, external_id, display_name, preferences)
|
||||
VALUES (
|
||||
'c0000000-0000-0000-0000-000000000001',
|
||||
'a0000000-0000-0000-0000-000000000001',
|
||||
'ivan',
|
||||
'Ivan Tytar',
|
||||
'{"language": "uk", "timezone": "Europe/Kyiv"}'
|
||||
) ON CONFLICT DO NOTHING;
|
||||
|
||||
-- Default agents
|
||||
INSERT INTO agents (agent_id, org_id, name, type, model, capabilities) VALUES
|
||||
(
|
||||
'd0000000-0000-0000-0000-000000000001',
|
||||
'a0000000-0000-0000-0000-000000000001',
|
||||
'Claude Assistant',
|
||||
'assistant',
|
||||
'claude-3-opus',
|
||||
'["code", "memory", "tools", "search"]'
|
||||
),
|
||||
(
|
||||
'd0000000-0000-0000-0000-000000000002',
|
||||
'a0000000-0000-0000-0000-000000000001',
|
||||
'Code Specialist',
|
||||
'specialist',
|
||||
'claude-3-opus',
|
||||
'["code", "review", "refactor"]'
|
||||
),
|
||||
(
|
||||
'd0000000-0000-0000-0000-000000000003',
|
||||
'a0000000-0000-0000-0000-000000000001',
|
||||
'DevOps Agent',
|
||||
'specialist',
|
||||
'claude-3-opus',
|
||||
'["infrastructure", "deployment", "monitoring"]'
|
||||
)
|
||||
ON CONFLICT DO NOTHING;
|
||||
|
||||
-- ============================================================================
|
||||
-- COMMENTS
|
||||
-- ============================================================================
|
||||
|
||||
COMMENT ON TABLE conversation_events IS 'Event log for all conversation activities - source of truth';
|
||||
COMMENT ON TABLE thread_summaries IS 'Rolling summaries for context compression (mid-term memory)';
|
||||
COMMENT ON TABLE long_term_memory_items IS 'Persistent facts about users/projects (long-term memory)';
|
||||
COMMENT ON COLUMN long_term_memory_items.fact_text IS 'Atomic statement - one fact per row';
|
||||
COMMENT ON COLUMN long_term_memory_items.confidence IS 'Confidence score 0-1, increases with confirmations';
|
||||
COMMENT ON COLUMN long_term_memory_items.fact_embedding_id IS 'Reference to Qdrant vector point ID';
|
||||
105
infrastructure/kubernetes/apps/memory-service/deployment.yaml
Normal file
105
infrastructure/kubernetes/apps/memory-service/deployment.yaml
Normal file
@@ -0,0 +1,105 @@
|
||||
---
|
||||
# DAARION Memory Service
|
||||
# Agent memory management with PostgreSQL + Qdrant + Cohere
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: memory-service
|
||||
namespace: daarion
|
||||
labels:
|
||||
app: memory-service
|
||||
component: memory
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: memory-service
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: memory-service
|
||||
component: memory
|
||||
spec:
|
||||
nodeSelector:
|
||||
node-role.kubernetes.io/control-plane: "true"
|
||||
containers:
|
||||
- name: memory-service
|
||||
image: ghcr.io/ivantytar/memory-service:latest
|
||||
imagePullPolicy: Always
|
||||
ports:
|
||||
- containerPort: 8000
|
||||
name: http
|
||||
envFrom:
|
||||
- secretRef:
|
||||
name: memory-service-secrets
|
||||
env:
|
||||
- name: MEMORY_DEBUG
|
||||
value: "false"
|
||||
resources:
|
||||
requests:
|
||||
memory: "256Mi"
|
||||
cpu: "100m"
|
||||
limits:
|
||||
memory: "512Mi"
|
||||
cpu: "500m"
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: 8000
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 10
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: 8000
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 5
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: memory-service
|
||||
namespace: daarion
|
||||
spec:
|
||||
selector:
|
||||
app: memory-service
|
||||
ports:
|
||||
- name: http
|
||||
port: 8000
|
||||
targetPort: 8000
|
||||
type: ClusterIP
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: memory-service-external
|
||||
namespace: daarion
|
||||
spec:
|
||||
selector:
|
||||
app: memory-service
|
||||
ports:
|
||||
- name: http
|
||||
port: 8000
|
||||
targetPort: 8000
|
||||
nodePort: 30800
|
||||
type: NodePort
|
||||
---
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: memory-service-ingress
|
||||
namespace: daarion
|
||||
annotations:
|
||||
traefik.ingress.kubernetes.io/router.entrypoints: web
|
||||
spec:
|
||||
rules:
|
||||
- host: memory.daarion.local
|
||||
http:
|
||||
paths:
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: memory-service
|
||||
port:
|
||||
number: 8000
|
||||
133
infrastructure/kubernetes/apps/qdrant/deployment.yaml
Normal file
133
infrastructure/kubernetes/apps/qdrant/deployment.yaml
Normal file
@@ -0,0 +1,133 @@
|
||||
---
|
||||
# DAARION Qdrant Vector Database
|
||||
# For semantic search in agent memory system
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: qdrant
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: qdrant-storage
|
||||
namespace: qdrant
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
resources:
|
||||
requests:
|
||||
storage: 20Gi
|
||||
storageClassName: local-path
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: qdrant-config
|
||||
namespace: qdrant
|
||||
data:
|
||||
config.yaml: |
|
||||
log_level: INFO
|
||||
|
||||
storage:
|
||||
storage_path: /qdrant/storage
|
||||
snapshots_path: /qdrant/snapshots
|
||||
|
||||
service:
|
||||
http_port: 6333
|
||||
grpc_port: 6334
|
||||
|
||||
cluster:
|
||||
enabled: false
|
||||
|
||||
telemetry_disabled: true
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: qdrant
|
||||
namespace: qdrant
|
||||
labels:
|
||||
app: qdrant
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: qdrant
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: qdrant
|
||||
spec:
|
||||
nodeSelector:
|
||||
node-role.kubernetes.io/control-plane: "true"
|
||||
containers:
|
||||
- name: qdrant
|
||||
image: qdrant/qdrant:v1.7.4
|
||||
ports:
|
||||
- containerPort: 6333
|
||||
name: http
|
||||
- containerPort: 6334
|
||||
name: grpc
|
||||
volumeMounts:
|
||||
- name: storage
|
||||
mountPath: /qdrant/storage
|
||||
- name: config
|
||||
mountPath: /qdrant/config
|
||||
resources:
|
||||
requests:
|
||||
memory: "512Mi"
|
||||
cpu: "250m"
|
||||
limits:
|
||||
memory: "2Gi"
|
||||
cpu: "1"
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /
|
||||
port: 6333
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 10
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /readyz
|
||||
port: 6333
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 5
|
||||
volumes:
|
||||
- name: storage
|
||||
persistentVolumeClaim:
|
||||
claimName: qdrant-storage
|
||||
- name: config
|
||||
configMap:
|
||||
name: qdrant-config
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: qdrant
|
||||
namespace: qdrant
|
||||
spec:
|
||||
selector:
|
||||
app: qdrant
|
||||
ports:
|
||||
- name: http
|
||||
port: 6333
|
||||
targetPort: 6333
|
||||
- name: grpc
|
||||
port: 6334
|
||||
targetPort: 6334
|
||||
type: ClusterIP
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: qdrant-external
|
||||
namespace: qdrant
|
||||
spec:
|
||||
selector:
|
||||
app: qdrant
|
||||
ports:
|
||||
- name: http
|
||||
port: 6333
|
||||
targetPort: 6333
|
||||
nodePort: 30633
|
||||
type: NodePort
|
||||
@@ -1,24 +1,23 @@
|
||||
# DAARION Memory Service
|
||||
FROM python:3.11-slim
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Встановлюємо системні залежності
|
||||
RUN apt-get update && apt-get install -y \
|
||||
gcc \
|
||||
postgresql-client \
|
||||
curl \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Копіюємо requirements та встановлюємо залежності
|
||||
# Install dependencies
|
||||
COPY requirements.txt .
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
|
||||
# Копіюємо код
|
||||
COPY . .
|
||||
# Copy application
|
||||
COPY app/ ./app/
|
||||
|
||||
# Відкриваємо порт
|
||||
# Environment
|
||||
ENV PYTHONPATH=/app
|
||||
ENV PYTHONUNBUFFERED=1
|
||||
|
||||
# Health check
|
||||
HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
|
||||
CMD curl -f http://localhost:8000/health || exit 1
|
||||
|
||||
# Run
|
||||
EXPOSE 8000
|
||||
|
||||
# Запускаємо додаток
|
||||
CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"]
|
||||
|
||||
|
||||
@@ -1 +1 @@
|
||||
# Memory Service for DAARION.city
|
||||
"""DAARION Memory Service"""
|
||||
|
||||
56
services/memory-service/app/config.py
Normal file
56
services/memory-service/app/config.py
Normal file
@@ -0,0 +1,56 @@
|
||||
"""
|
||||
DAARION Memory Service Configuration
|
||||
"""
|
||||
from pydantic_settings import BaseSettings
|
||||
from functools import lru_cache
|
||||
|
||||
|
||||
class Settings(BaseSettings):
|
||||
"""Application settings"""
|
||||
|
||||
# Service
|
||||
service_name: str = "memory-service"
|
||||
debug: bool = False
|
||||
|
||||
# PostgreSQL
|
||||
postgres_host: str = "daarion-pooler.daarion"
|
||||
postgres_port: int = 5432
|
||||
postgres_user: str = "daarion"
|
||||
postgres_password: str = "DaarionDB2026!"
|
||||
postgres_db: str = "daarion_main"
|
||||
|
||||
@property
|
||||
def postgres_url(self) -> str:
|
||||
return f"postgresql+asyncpg://{self.postgres_user}:{self.postgres_password}@{self.postgres_host}:{self.postgres_port}/{self.postgres_db}"
|
||||
|
||||
# Qdrant
|
||||
qdrant_host: str = "qdrant.qdrant"
|
||||
qdrant_port: int = 6333
|
||||
qdrant_collection_memories: str = "memories"
|
||||
qdrant_collection_messages: str = "messages"
|
||||
|
||||
# Cohere (embeddings)
|
||||
cohere_api_key: str = "nOdOXnuepLku2ipJWpe6acWgAsJCsDhMO0RnaEJB"
|
||||
cohere_model: str = "embed-multilingual-v3.0" # 1024 dimensions
|
||||
embedding_dimensions: int = 1024
|
||||
|
||||
# Memory settings
|
||||
short_term_window_messages: int = 20
|
||||
short_term_window_minutes: int = 60
|
||||
summary_trigger_tokens: int = 4000
|
||||
summary_target_tokens: int = 500
|
||||
retrieval_top_k: int = 10
|
||||
|
||||
# Confidence thresholds
|
||||
memory_min_confidence: float = 0.5
|
||||
memory_confirm_boost: float = 0.1
|
||||
memory_reject_penalty: float = 0.3
|
||||
|
||||
class Config:
|
||||
env_prefix = "MEMORY_"
|
||||
env_file = ".env"
|
||||
|
||||
|
||||
@lru_cache()
|
||||
def get_settings() -> Settings:
|
||||
return Settings()
|
||||
430
services/memory-service/app/database.py
Normal file
430
services/memory-service/app/database.py
Normal file
@@ -0,0 +1,430 @@
|
||||
"""
|
||||
DAARION Memory Service - PostgreSQL Database Layer
|
||||
"""
|
||||
from typing import List, Optional, Dict, Any
|
||||
from uuid import UUID, uuid4
|
||||
from datetime import datetime
|
||||
import structlog
|
||||
import asyncpg
|
||||
|
||||
from .config import get_settings
|
||||
from .models import EventType, MessageRole, MemoryCategory, RetentionPolicy, FeedbackAction
|
||||
|
||||
logger = structlog.get_logger()
|
||||
settings = get_settings()
|
||||
|
||||
|
||||
class Database:
|
||||
"""PostgreSQL database operations"""
|
||||
|
||||
def __init__(self):
|
||||
self.pool: Optional[asyncpg.Pool] = None
|
||||
|
||||
async def connect(self):
|
||||
"""Connect to database"""
|
||||
self.pool = await asyncpg.create_pool(
|
||||
host=settings.postgres_host,
|
||||
port=settings.postgres_port,
|
||||
user=settings.postgres_user,
|
||||
password=settings.postgres_password,
|
||||
database=settings.postgres_db,
|
||||
min_size=5,
|
||||
max_size=20
|
||||
)
|
||||
logger.info("database_connected")
|
||||
|
||||
async def disconnect(self):
|
||||
"""Disconnect from database"""
|
||||
if self.pool:
|
||||
await self.pool.close()
|
||||
logger.info("database_disconnected")
|
||||
|
||||
# ========================================================================
|
||||
# THREADS
|
||||
# ========================================================================
|
||||
|
||||
async def create_thread(
|
||||
self,
|
||||
org_id: UUID,
|
||||
user_id: UUID,
|
||||
workspace_id: Optional[UUID] = None,
|
||||
agent_id: Optional[UUID] = None,
|
||||
title: Optional[str] = None,
|
||||
tags: List[str] = [],
|
||||
metadata: dict = {}
|
||||
) -> Dict[str, Any]:
|
||||
"""Create new conversation thread"""
|
||||
thread_id = uuid4()
|
||||
|
||||
async with self.pool.acquire() as conn:
|
||||
row = await conn.fetchrow("""
|
||||
INSERT INTO conversation_threads
|
||||
(thread_id, org_id, workspace_id, user_id, agent_id, title, tags, metadata)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
|
||||
RETURNING *
|
||||
""", thread_id, org_id, workspace_id, user_id, agent_id, title, tags, metadata)
|
||||
|
||||
logger.info("thread_created", thread_id=str(thread_id))
|
||||
return dict(row)
|
||||
|
||||
async def get_thread(self, thread_id: UUID) -> Optional[Dict[str, Any]]:
|
||||
"""Get thread by ID"""
|
||||
async with self.pool.acquire() as conn:
|
||||
row = await conn.fetchrow("""
|
||||
SELECT * FROM conversation_threads WHERE thread_id = $1
|
||||
""", thread_id)
|
||||
return dict(row) if row else None
|
||||
|
||||
async def list_threads(
|
||||
self,
|
||||
org_id: UUID,
|
||||
user_id: UUID,
|
||||
workspace_id: Optional[UUID] = None,
|
||||
agent_id: Optional[UUID] = None,
|
||||
limit: int = 20
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""List threads for user"""
|
||||
async with self.pool.acquire() as conn:
|
||||
query = """
|
||||
SELECT * FROM conversation_threads
|
||||
WHERE org_id = $1 AND user_id = $2 AND status = 'active'
|
||||
"""
|
||||
params = [org_id, user_id]
|
||||
|
||||
if workspace_id:
|
||||
query += f" AND workspace_id = ${len(params) + 1}"
|
||||
params.append(workspace_id)
|
||||
if agent_id:
|
||||
query += f" AND agent_id = ${len(params) + 1}"
|
||||
params.append(agent_id)
|
||||
|
||||
query += f" ORDER BY last_activity_at DESC LIMIT ${len(params) + 1}"
|
||||
params.append(limit)
|
||||
|
||||
rows = await conn.fetch(query, *params)
|
||||
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
# ========================================================================
|
||||
# EVENTS
|
||||
# ========================================================================
|
||||
|
||||
async def add_event(
|
||||
self,
|
||||
thread_id: UUID,
|
||||
event_type: EventType,
|
||||
role: Optional[MessageRole] = None,
|
||||
content: Optional[str] = None,
|
||||
tool_name: Optional[str] = None,
|
||||
tool_input: Optional[dict] = None,
|
||||
tool_output: Optional[dict] = None,
|
||||
payload: dict = {},
|
||||
token_count: Optional[int] = None,
|
||||
model_used: Optional[str] = None,
|
||||
latency_ms: Optional[int] = None,
|
||||
metadata: dict = {}
|
||||
) -> Dict[str, Any]:
|
||||
"""Add event to conversation"""
|
||||
event_id = uuid4()
|
||||
|
||||
async with self.pool.acquire() as conn:
|
||||
row = await conn.fetchrow("""
|
||||
INSERT INTO conversation_events
|
||||
(event_id, thread_id, event_type, role, content, tool_name,
|
||||
tool_input, tool_output, payload, token_count, model_used,
|
||||
latency_ms, metadata)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)
|
||||
RETURNING *
|
||||
""", event_id, thread_id, event_type.value,
|
||||
role.value if role else None, content, tool_name,
|
||||
tool_input, tool_output, payload, token_count, model_used,
|
||||
latency_ms, metadata)
|
||||
|
||||
logger.info("event_added", event_id=str(event_id), type=event_type.value)
|
||||
return dict(row)
|
||||
|
||||
async def get_events(
|
||||
self,
|
||||
thread_id: UUID,
|
||||
limit: int = 50,
|
||||
offset: int = 0
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Get events for thread"""
|
||||
async with self.pool.acquire() as conn:
|
||||
rows = await conn.fetch("""
|
||||
SELECT * FROM conversation_events
|
||||
WHERE thread_id = $1
|
||||
ORDER BY sequence_num DESC
|
||||
LIMIT $2 OFFSET $3
|
||||
""", thread_id, limit, offset)
|
||||
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
async def get_events_for_summary(
|
||||
self,
|
||||
thread_id: UUID,
|
||||
after_seq: Optional[int] = None
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Get events for summarization"""
|
||||
async with self.pool.acquire() as conn:
|
||||
if after_seq:
|
||||
rows = await conn.fetch("""
|
||||
SELECT * FROM conversation_events
|
||||
WHERE thread_id = $1 AND sequence_num > $2
|
||||
ORDER BY sequence_num ASC
|
||||
""", thread_id, after_seq)
|
||||
else:
|
||||
rows = await conn.fetch("""
|
||||
SELECT * FROM conversation_events
|
||||
WHERE thread_id = $1
|
||||
ORDER BY sequence_num ASC
|
||||
""", thread_id)
|
||||
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
# ========================================================================
|
||||
# MEMORIES
|
||||
# ========================================================================
|
||||
|
||||
async def create_memory(
|
||||
self,
|
||||
org_id: UUID,
|
||||
user_id: UUID,
|
||||
category: MemoryCategory,
|
||||
fact_text: str,
|
||||
workspace_id: Optional[UUID] = None,
|
||||
agent_id: Optional[UUID] = None,
|
||||
confidence: float = 0.8,
|
||||
source_event_id: Optional[UUID] = None,
|
||||
source_thread_id: Optional[UUID] = None,
|
||||
extraction_method: str = "explicit",
|
||||
is_sensitive: bool = False,
|
||||
retention: RetentionPolicy = RetentionPolicy.UNTIL_REVOKED,
|
||||
ttl_days: Optional[int] = None,
|
||||
tags: List[str] = [],
|
||||
metadata: dict = {}
|
||||
) -> Dict[str, Any]:
|
||||
"""Create long-term memory item"""
|
||||
memory_id = uuid4()
|
||||
|
||||
async with self.pool.acquire() as conn:
|
||||
row = await conn.fetchrow("""
|
||||
INSERT INTO long_term_memory_items
|
||||
(memory_id, org_id, workspace_id, user_id, agent_id, category,
|
||||
fact_text, confidence, source_event_id, source_thread_id,
|
||||
extraction_method, is_sensitive, retention, ttl_days, tags, metadata)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16)
|
||||
RETURNING *
|
||||
""", memory_id, org_id, workspace_id, user_id, agent_id, category.value,
|
||||
fact_text, confidence, source_event_id, source_thread_id,
|
||||
extraction_method, is_sensitive, retention.value, ttl_days, tags, metadata)
|
||||
|
||||
logger.info("memory_created", memory_id=str(memory_id), category=category.value)
|
||||
return dict(row)
|
||||
|
||||
async def get_memory(self, memory_id: UUID) -> Optional[Dict[str, Any]]:
|
||||
"""Get memory by ID"""
|
||||
async with self.pool.acquire() as conn:
|
||||
row = await conn.fetchrow("""
|
||||
SELECT * FROM long_term_memory_items
|
||||
WHERE memory_id = $1 AND valid_to IS NULL
|
||||
""", memory_id)
|
||||
return dict(row) if row else None
|
||||
|
||||
async def list_memories(
|
||||
self,
|
||||
org_id: UUID,
|
||||
user_id: UUID,
|
||||
agent_id: Optional[UUID] = None,
|
||||
workspace_id: Optional[UUID] = None,
|
||||
category: Optional[MemoryCategory] = None,
|
||||
include_global: bool = True,
|
||||
limit: int = 50
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""List memories for user"""
|
||||
async with self.pool.acquire() as conn:
|
||||
query = """
|
||||
SELECT * FROM long_term_memory_items
|
||||
WHERE org_id = $1 AND user_id = $2 AND valid_to IS NULL
|
||||
AND confidence >= $3
|
||||
"""
|
||||
params = [org_id, user_id, settings.memory_min_confidence]
|
||||
|
||||
if workspace_id:
|
||||
query += f" AND (workspace_id = ${len(params) + 1} OR workspace_id IS NULL)"
|
||||
params.append(workspace_id)
|
||||
|
||||
if agent_id:
|
||||
if include_global:
|
||||
query += f" AND (agent_id = ${len(params) + 1} OR agent_id IS NULL)"
|
||||
else:
|
||||
query += f" AND agent_id = ${len(params) + 1}"
|
||||
params.append(agent_id)
|
||||
|
||||
if category:
|
||||
query += f" AND category = ${len(params) + 1}"
|
||||
params.append(category.value)
|
||||
|
||||
query += f" ORDER BY confidence DESC, last_used_at DESC NULLS LAST LIMIT ${len(params) + 1}"
|
||||
params.append(limit)
|
||||
|
||||
rows = await conn.fetch(query, *params)
|
||||
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
async def update_memory_embedding_id(self, memory_id: UUID, embedding_id: str):
|
||||
"""Update memory with Qdrant point ID"""
|
||||
async with self.pool.acquire() as conn:
|
||||
await conn.execute("""
|
||||
UPDATE long_term_memory_items
|
||||
SET fact_embedding_id = $2
|
||||
WHERE memory_id = $1
|
||||
""", memory_id, embedding_id)
|
||||
|
||||
async def update_memory_confidence(
|
||||
self,
|
||||
memory_id: UUID,
|
||||
confidence: float,
|
||||
verified: bool = False
|
||||
):
|
||||
"""Update memory confidence"""
|
||||
async with self.pool.acquire() as conn:
|
||||
await conn.execute("""
|
||||
UPDATE long_term_memory_items
|
||||
SET confidence = $2,
|
||||
is_verified = CASE WHEN $3 THEN true ELSE is_verified END,
|
||||
verification_count = verification_count + CASE WHEN $3 THEN 1 ELSE 0 END,
|
||||
last_confirmed_at = CASE WHEN $3 THEN NOW() ELSE last_confirmed_at END
|
||||
WHERE memory_id = $1
|
||||
""", memory_id, confidence, verified)
|
||||
|
||||
async def update_memory_text(self, memory_id: UUID, new_text: str):
|
||||
"""Update memory text"""
|
||||
async with self.pool.acquire() as conn:
|
||||
await conn.execute("""
|
||||
UPDATE long_term_memory_items
|
||||
SET fact_text = $2
|
||||
WHERE memory_id = $1
|
||||
""", memory_id, new_text)
|
||||
|
||||
async def invalidate_memory(self, memory_id: UUID):
|
||||
"""Mark memory as invalid (soft delete)"""
|
||||
async with self.pool.acquire() as conn:
|
||||
await conn.execute("""
|
||||
UPDATE long_term_memory_items
|
||||
SET valid_to = NOW()
|
||||
WHERE memory_id = $1
|
||||
""", memory_id)
|
||||
logger.info("memory_invalidated", memory_id=str(memory_id))
|
||||
|
||||
async def increment_memory_usage(self, memory_id: UUID):
|
||||
"""Increment memory usage counter"""
|
||||
async with self.pool.acquire() as conn:
|
||||
await conn.execute("""
|
||||
UPDATE long_term_memory_items
|
||||
SET use_count = use_count + 1, last_used_at = NOW()
|
||||
WHERE memory_id = $1
|
||||
""", memory_id)
|
||||
|
||||
# ========================================================================
|
||||
# FEEDBACK
|
||||
# ========================================================================
|
||||
|
||||
async def add_memory_feedback(
|
||||
self,
|
||||
memory_id: UUID,
|
||||
user_id: UUID,
|
||||
action: FeedbackAction,
|
||||
old_value: Optional[str] = None,
|
||||
new_value: Optional[str] = None,
|
||||
reason: Optional[str] = None
|
||||
):
|
||||
"""Record user feedback on memory"""
|
||||
feedback_id = uuid4()
|
||||
|
||||
async with self.pool.acquire() as conn:
|
||||
await conn.execute("""
|
||||
INSERT INTO memory_feedback
|
||||
(feedback_id, memory_id, user_id, action, old_value, new_value, reason)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7)
|
||||
""", feedback_id, memory_id, user_id, action.value, old_value, new_value, reason)
|
||||
|
||||
logger.info("feedback_recorded", memory_id=str(memory_id), action=action.value)
|
||||
|
||||
# ========================================================================
|
||||
# SUMMARIES
|
||||
# ========================================================================
|
||||
|
||||
async def create_summary(
|
||||
self,
|
||||
thread_id: UUID,
|
||||
summary_text: str,
|
||||
state: dict,
|
||||
events_from_seq: int,
|
||||
events_to_seq: int,
|
||||
events_count: int,
|
||||
original_tokens: Optional[int] = None,
|
||||
summary_tokens: Optional[int] = None
|
||||
) -> Dict[str, Any]:
|
||||
"""Create thread summary"""
|
||||
summary_id = uuid4()
|
||||
|
||||
# Get next version
|
||||
async with self.pool.acquire() as conn:
|
||||
version_row = await conn.fetchrow("""
|
||||
SELECT COALESCE(MAX(version), 0) + 1 as next_version
|
||||
FROM thread_summaries WHERE thread_id = $1
|
||||
""", thread_id)
|
||||
version = version_row["next_version"]
|
||||
|
||||
compression_ratio = None
|
||||
if original_tokens and summary_tokens:
|
||||
compression_ratio = summary_tokens / original_tokens
|
||||
|
||||
row = await conn.fetchrow("""
|
||||
INSERT INTO thread_summaries
|
||||
(summary_id, thread_id, version, summary_text, state,
|
||||
events_from_seq, events_to_seq, events_count,
|
||||
original_tokens, summary_tokens, compression_ratio)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)
|
||||
RETURNING *
|
||||
""", summary_id, thread_id, version, summary_text, state,
|
||||
events_from_seq, events_to_seq, events_count,
|
||||
original_tokens, summary_tokens, compression_ratio)
|
||||
|
||||
logger.info("summary_created", summary_id=str(summary_id), version=version)
|
||||
return dict(row)
|
||||
|
||||
async def get_latest_summary(self, thread_id: UUID) -> Optional[Dict[str, Any]]:
|
||||
"""Get latest summary for thread"""
|
||||
async with self.pool.acquire() as conn:
|
||||
row = await conn.fetchrow("""
|
||||
SELECT * FROM thread_summaries
|
||||
WHERE thread_id = $1
|
||||
ORDER BY version DESC
|
||||
LIMIT 1
|
||||
""", thread_id)
|
||||
return dict(row) if row else None
|
||||
|
||||
# ========================================================================
|
||||
# STATS
|
||||
# ========================================================================
|
||||
|
||||
async def get_stats(self) -> Dict[str, Any]:
|
||||
"""Get database statistics"""
|
||||
async with self.pool.acquire() as conn:
|
||||
threads = await conn.fetchval("SELECT COUNT(*) FROM conversation_threads")
|
||||
events = await conn.fetchval("SELECT COUNT(*) FROM conversation_events")
|
||||
memories = await conn.fetchval("SELECT COUNT(*) FROM long_term_memory_items WHERE valid_to IS NULL")
|
||||
summaries = await conn.fetchval("SELECT COUNT(*) FROM thread_summaries")
|
||||
|
||||
return {
|
||||
"threads": threads,
|
||||
"events": events,
|
||||
"active_memories": memories,
|
||||
"summaries": summaries
|
||||
}
|
||||
|
||||
|
||||
# Global instance
|
||||
db = Database()
|
||||
86
services/memory-service/app/embedding.py
Normal file
86
services/memory-service/app/embedding.py
Normal file
@@ -0,0 +1,86 @@
|
||||
"""
|
||||
DAARION Memory Service - Embedding Layer (Cohere)
|
||||
"""
|
||||
import cohere
|
||||
from typing import List
|
||||
import structlog
|
||||
from tenacity import retry, stop_after_attempt, wait_exponential
|
||||
|
||||
from .config import get_settings
|
||||
|
||||
logger = structlog.get_logger()
|
||||
settings = get_settings()
|
||||
|
||||
# Initialize Cohere client
|
||||
co = cohere.Client(settings.cohere_api_key)
|
||||
|
||||
|
||||
@retry(
|
||||
stop=stop_after_attempt(3),
|
||||
wait=wait_exponential(multiplier=1, min=1, max=10)
|
||||
)
|
||||
async def get_embeddings(
|
||||
texts: List[str],
|
||||
input_type: str = "search_document"
|
||||
) -> List[List[float]]:
|
||||
"""
|
||||
Get embeddings from Cohere API.
|
||||
|
||||
Args:
|
||||
texts: List of texts to embed
|
||||
input_type: "search_document" for indexing, "search_query" for queries
|
||||
|
||||
Returns:
|
||||
List of embedding vectors (1024 dimensions for embed-multilingual-v3.0)
|
||||
"""
|
||||
if not texts:
|
||||
return []
|
||||
|
||||
logger.info("generating_embeddings", count=len(texts), input_type=input_type)
|
||||
|
||||
response = co.embed(
|
||||
texts=texts,
|
||||
model=settings.cohere_model,
|
||||
input_type=input_type,
|
||||
truncate="END"
|
||||
)
|
||||
|
||||
embeddings = response.embeddings
|
||||
|
||||
logger.info(
|
||||
"embeddings_generated",
|
||||
count=len(embeddings),
|
||||
dimensions=len(embeddings[0]) if embeddings else 0
|
||||
)
|
||||
|
||||
return embeddings
|
||||
|
||||
|
||||
async def get_query_embedding(query: str) -> List[float]:
|
||||
"""Get embedding for a search query"""
|
||||
embeddings = await get_embeddings([query], input_type="search_query")
|
||||
return embeddings[0] if embeddings else []
|
||||
|
||||
|
||||
async def get_document_embeddings(texts: List[str]) -> List[List[float]]:
|
||||
"""Get embeddings for documents (memories, summaries)"""
|
||||
return await get_embeddings(texts, input_type="search_document")
|
||||
|
||||
|
||||
# Batch processing for large sets
|
||||
async def batch_embed(
|
||||
texts: List[str],
|
||||
input_type: str = "search_document",
|
||||
batch_size: int = 96 # Cohere limit
|
||||
) -> List[List[float]]:
|
||||
"""
|
||||
Embed large number of texts in batches.
|
||||
"""
|
||||
all_embeddings = []
|
||||
|
||||
for i in range(0, len(texts), batch_size):
|
||||
batch = texts[i:i + batch_size]
|
||||
embeddings = await get_embeddings(batch, input_type)
|
||||
all_embeddings.extend(embeddings)
|
||||
|
||||
return all_embeddings
|
||||
@@ -1,443 +1,483 @@
|
||||
"""
|
||||
Memory Service - FastAPI додаток
|
||||
Підтримує: user_facts, dialog_summaries, agent_memory_events
|
||||
Інтеграція з token-gate через RBAC
|
||||
DAARION Memory Service - FastAPI Application
|
||||
|
||||
Трирівнева пам'ять агентів:
|
||||
- Short-term: conversation events (робочий буфер)
|
||||
- Mid-term: thread summaries (сесійна/тематична)
|
||||
- Long-term: memory items (персональна/проектна)
|
||||
"""
|
||||
|
||||
import os
|
||||
from typing import Optional, List, Dict, Any
|
||||
from datetime import datetime
|
||||
|
||||
from fastapi import FastAPI, Depends, HTTPException, Query, Header
|
||||
from contextlib import asynccontextmanager
|
||||
from typing import List, Optional
|
||||
from uuid import UUID
|
||||
import structlog
|
||||
from fastapi import FastAPI, HTTPException, Query
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from sqlalchemy.orm import Session
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
from app.models import Base, UserFact, DialogSummary, AgentMemoryEvent
|
||||
from app.schemas import (
|
||||
UserFactCreate, UserFactUpdate, UserFactResponse, UserFactUpsertRequest, UserFactUpsertResponse,
|
||||
DialogSummaryCreate, DialogSummaryResponse, DialogSummaryListResponse,
|
||||
AgentMemoryEventCreate, AgentMemoryEventResponse, AgentMemoryEventListResponse,
|
||||
TokenGateCheck, TokenGateCheckResponse
|
||||
)
|
||||
from app.crud import (
|
||||
get_user_fact, get_user_facts, create_user_fact, update_user_fact,
|
||||
upsert_user_fact, delete_user_fact, get_user_facts_by_token_gate,
|
||||
create_dialog_summary, get_dialog_summaries, get_dialog_summary, delete_dialog_summary,
|
||||
create_agent_memory_event, get_agent_memory_events, delete_agent_memory_event
|
||||
from .config import get_settings
|
||||
from .models import (
|
||||
CreateThreadRequest, AddEventRequest, CreateMemoryRequest,
|
||||
MemoryFeedbackRequest, RetrievalRequest, SummaryRequest,
|
||||
ThreadResponse, EventResponse, MemoryResponse,
|
||||
SummaryResponse, RetrievalResponse, RetrievalResult,
|
||||
ContextResponse, MemoryCategory, FeedbackAction
|
||||
)
|
||||
from .vector_store import vector_store
|
||||
from .database import db
|
||||
|
||||
# ========== Configuration ==========
|
||||
logger = structlog.get_logger()
|
||||
settings = get_settings()
|
||||
|
||||
DATABASE_URL = os.getenv(
|
||||
"DATABASE_URL",
|
||||
"sqlite:///./memory.db" # SQLite для розробки, PostgreSQL для продакшену
|
||||
)
|
||||
|
||||
# Створюємо engine та sessionmaker
|
||||
engine = create_engine(DATABASE_URL)
|
||||
SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
"""Startup and shutdown events"""
|
||||
# Startup
|
||||
logger.info("starting_memory_service")
|
||||
await db.connect()
|
||||
await vector_store.initialize()
|
||||
yield
|
||||
# Shutdown
|
||||
await db.disconnect()
|
||||
logger.info("memory_service_stopped")
|
||||
|
||||
# Створюємо таблиці (для dev, в продакшені використовуйте міграції)
|
||||
Base.metadata.create_all(bind=engine)
|
||||
|
||||
# ========== FastAPI App ==========
|
||||
|
||||
app = FastAPI(
|
||||
title="Memory Service",
|
||||
description="Сервіс пам'яті для MicroDAO: user_facts, dialog_summaries, agent_memory_events",
|
||||
version="1.0.0"
|
||||
title="DAARION Memory Service",
|
||||
description="Agent memory management with PostgreSQL + Qdrant + Cohere",
|
||||
version="1.0.0",
|
||||
lifespan=lifespan
|
||||
)
|
||||
|
||||
# CORS middleware
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=["*"], # В продакшені обмежте це
|
||||
allow_origins=["*"],
|
||||
allow_credentials=True,
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
# ========== Dependencies ==========
|
||||
|
||||
def get_db():
|
||||
"""Dependency для отримання DB сесії"""
|
||||
db = SessionLocal()
|
||||
try:
|
||||
yield db
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
async def verify_token(authorization: Optional[str] = Header(None)) -> Optional[str]:
|
||||
"""
|
||||
Перевірка JWT токену (заглушка)
|
||||
В продакшені інтегруйте з вашою системою авторизації
|
||||
"""
|
||||
if not authorization:
|
||||
raise HTTPException(status_code=401, detail="Missing authorization header")
|
||||
|
||||
# Заглушка: в реальності перевіряйте JWT
|
||||
# token = authorization.replace("Bearer ", "")
|
||||
# user_id = verify_jwt_token(token)
|
||||
# return user_id
|
||||
|
||||
# Для тестування повертаємо user_id з заголовка
|
||||
return "u_test" # TODO: реалізувати реальну перевірку
|
||||
|
||||
|
||||
async def check_token_gate(
|
||||
user_id: str,
|
||||
token_requirements: dict,
|
||||
db: Session
|
||||
) -> TokenGateCheckResponse:
|
||||
"""
|
||||
Перевірка токен-гейту (інтеграція з RBAC/Wallet Service)
|
||||
Заглушка - в продакшені викликайте ваш PDP/Wallet Service
|
||||
"""
|
||||
# TODO: Інтегрувати з:
|
||||
# - PDP Service для перевірки capabilities
|
||||
# - Wallet Service для перевірки балансів
|
||||
# - RBAC для перевірки ролей
|
||||
|
||||
# Приклад логіки:
|
||||
# if "token" in token_requirements:
|
||||
# token_type = token_requirements["token"]
|
||||
# min_balance = token_requirements.get("min_balance", 0)
|
||||
# balance = await wallet_service.get_balance(user_id, token_type)
|
||||
# if balance < min_balance:
|
||||
# return TokenGateCheckResponse(
|
||||
# allowed=False,
|
||||
# reason=f"Insufficient {token_type} balance",
|
||||
# missing_requirements={"token": token_type, "required": min_balance, "current": balance}
|
||||
# )
|
||||
|
||||
# Заглушка: завжди дозволяємо
|
||||
return TokenGateCheckResponse(allowed=True)
|
||||
|
||||
|
||||
# ========== Health Check ==========
|
||||
# ============================================================================
|
||||
# HEALTH
|
||||
# ============================================================================
|
||||
|
||||
@app.get("/health")
|
||||
async def health_check():
|
||||
"""Health check endpoint"""
|
||||
return {"status": "ok", "service": "memory-service"}
|
||||
async def health():
|
||||
"""Health check"""
|
||||
return {
|
||||
"status": "healthy",
|
||||
"service": settings.service_name,
|
||||
"vector_store": await vector_store.get_collection_stats()
|
||||
}
|
||||
|
||||
|
||||
# ========== User Facts Endpoints ==========
|
||||
# ============================================================================
|
||||
# THREADS (Conversations)
|
||||
# ============================================================================
|
||||
|
||||
@app.post("/facts/upsert", response_model=UserFactUpsertResponse)
|
||||
async def upsert_fact(
|
||||
fact_request: UserFactUpsertRequest,
|
||||
db: Session = Depends(get_db),
|
||||
user_id: str = Depends(verify_token)
|
||||
@app.post("/threads", response_model=ThreadResponse)
|
||||
async def create_thread(request: CreateThreadRequest):
|
||||
"""Create new conversation thread"""
|
||||
thread = await db.create_thread(
|
||||
org_id=request.org_id,
|
||||
user_id=request.user_id,
|
||||
workspace_id=request.workspace_id,
|
||||
agent_id=request.agent_id,
|
||||
title=request.title,
|
||||
tags=request.tags,
|
||||
metadata=request.metadata
|
||||
)
|
||||
return thread
|
||||
|
||||
|
||||
@app.get("/threads/{thread_id}", response_model=ThreadResponse)
|
||||
async def get_thread(thread_id: UUID):
|
||||
"""Get thread by ID"""
|
||||
thread = await db.get_thread(thread_id)
|
||||
if not thread:
|
||||
raise HTTPException(status_code=404, detail="Thread not found")
|
||||
return thread
|
||||
|
||||
|
||||
@app.get("/threads", response_model=List[ThreadResponse])
|
||||
async def list_threads(
|
||||
user_id: UUID = Query(...),
|
||||
org_id: UUID = Query(...),
|
||||
workspace_id: Optional[UUID] = None,
|
||||
agent_id: Optional[UUID] = None,
|
||||
limit: int = Query(default=20, le=100)
|
||||
):
|
||||
"""
|
||||
Створити або оновити факт користувача (upsert)
|
||||
"""List threads for user"""
|
||||
threads = await db.list_threads(
|
||||
org_id=org_id,
|
||||
user_id=user_id,
|
||||
workspace_id=workspace_id,
|
||||
agent_id=agent_id,
|
||||
limit=limit
|
||||
)
|
||||
return threads
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# EVENTS (Short-term Memory)
|
||||
# ============================================================================
|
||||
|
||||
@app.post("/events", response_model=EventResponse)
|
||||
async def add_event(request: AddEventRequest):
|
||||
"""Add event to conversation (message, tool call, etc.)"""
|
||||
event = await db.add_event(
|
||||
thread_id=request.thread_id,
|
||||
event_type=request.event_type,
|
||||
role=request.role,
|
||||
content=request.content,
|
||||
tool_name=request.tool_name,
|
||||
tool_input=request.tool_input,
|
||||
tool_output=request.tool_output,
|
||||
payload=request.payload,
|
||||
token_count=request.token_count,
|
||||
model_used=request.model_used,
|
||||
latency_ms=request.latency_ms,
|
||||
metadata=request.metadata
|
||||
)
|
||||
return event
|
||||
|
||||
|
||||
@app.get("/threads/{thread_id}/events", response_model=List[EventResponse])
|
||||
async def get_events(
|
||||
thread_id: UUID,
|
||||
limit: int = Query(default=50, le=200),
|
||||
offset: int = Query(default=0)
|
||||
):
|
||||
"""Get events for thread (most recent first)"""
|
||||
events = await db.get_events(thread_id, limit=limit, offset=offset)
|
||||
return events
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# MEMORIES (Long-term Memory)
|
||||
# ============================================================================
|
||||
|
||||
@app.post("/memories", response_model=MemoryResponse)
|
||||
async def create_memory(request: CreateMemoryRequest):
|
||||
"""Create long-term memory item"""
|
||||
# Create in PostgreSQL
|
||||
memory = await db.create_memory(
|
||||
org_id=request.org_id,
|
||||
user_id=request.user_id,
|
||||
workspace_id=request.workspace_id,
|
||||
agent_id=request.agent_id,
|
||||
category=request.category,
|
||||
fact_text=request.fact_text,
|
||||
confidence=request.confidence,
|
||||
source_event_id=request.source_event_id,
|
||||
source_thread_id=request.source_thread_id,
|
||||
extraction_method=request.extraction_method,
|
||||
is_sensitive=request.is_sensitive,
|
||||
retention=request.retention,
|
||||
ttl_days=request.ttl_days,
|
||||
tags=request.tags,
|
||||
metadata=request.metadata
|
||||
)
|
||||
|
||||
Це основний ендпоінт для контрольованої довгострокової пам'яті.
|
||||
Підтримує токен-гейт інтеграцію.
|
||||
"""
|
||||
# Перевірка токен-гейту якщо потрібно
|
||||
if fact_request.token_gated and fact_request.token_requirements:
|
||||
gate_check = await check_token_gate(
|
||||
fact_request.user_id,
|
||||
fact_request.token_requirements,
|
||||
db
|
||||
)
|
||||
if not gate_check.allowed:
|
||||
raise HTTPException(
|
||||
status_code=403,
|
||||
detail=f"Token gate check failed: {gate_check.reason}"
|
||||
# Index in Qdrant
|
||||
point_id = await vector_store.index_memory(
|
||||
memory_id=memory["memory_id"],
|
||||
text=request.fact_text,
|
||||
org_id=request.org_id,
|
||||
user_id=request.user_id,
|
||||
category=request.category,
|
||||
agent_id=request.agent_id,
|
||||
workspace_id=request.workspace_id,
|
||||
thread_id=request.source_thread_id
|
||||
)
|
||||
|
||||
# Update memory with embedding ID
|
||||
await db.update_memory_embedding_id(memory["memory_id"], point_id)
|
||||
|
||||
return memory
|
||||
|
||||
|
||||
@app.get("/memories/{memory_id}", response_model=MemoryResponse)
|
||||
async def get_memory(memory_id: UUID):
|
||||
"""Get memory by ID"""
|
||||
memory = await db.get_memory(memory_id)
|
||||
if not memory:
|
||||
raise HTTPException(status_code=404, detail="Memory not found")
|
||||
return memory
|
||||
|
||||
|
||||
@app.get("/memories", response_model=List[MemoryResponse])
|
||||
async def list_memories(
|
||||
user_id: UUID = Query(...),
|
||||
org_id: UUID = Query(...),
|
||||
agent_id: Optional[UUID] = None,
|
||||
workspace_id: Optional[UUID] = None,
|
||||
category: Optional[MemoryCategory] = None,
|
||||
include_global: bool = True,
|
||||
limit: int = Query(default=50, le=200)
|
||||
):
|
||||
"""List memories for user"""
|
||||
memories = await db.list_memories(
|
||||
org_id=org_id,
|
||||
user_id=user_id,
|
||||
agent_id=agent_id,
|
||||
workspace_id=workspace_id,
|
||||
category=category,
|
||||
include_global=include_global,
|
||||
limit=limit
|
||||
)
|
||||
return memories
|
||||
|
||||
|
||||
@app.post("/memories/{memory_id}/feedback")
|
||||
async def memory_feedback(memory_id: UUID, request: MemoryFeedbackRequest):
|
||||
"""User feedback on memory (confirm/reject/edit/delete)"""
|
||||
memory = await db.get_memory(memory_id)
|
||||
if not memory:
|
||||
raise HTTPException(status_code=404, detail="Memory not found")
|
||||
|
||||
# Record feedback
|
||||
await db.add_memory_feedback(
|
||||
memory_id=memory_id,
|
||||
user_id=request.user_id,
|
||||
action=request.action,
|
||||
old_value=memory["fact_text"],
|
||||
new_value=request.new_value,
|
||||
reason=request.reason
|
||||
)
|
||||
|
||||
# Apply action
|
||||
if request.action == FeedbackAction.CONFIRM:
|
||||
new_confidence = min(1.0, memory["confidence"] + settings.memory_confirm_boost)
|
||||
await db.update_memory_confidence(memory_id, new_confidence, verified=True)
|
||||
|
||||
elif request.action == FeedbackAction.REJECT:
|
||||
new_confidence = max(0.0, memory["confidence"] - settings.memory_reject_penalty)
|
||||
if new_confidence < settings.memory_min_confidence:
|
||||
# Mark as invalid
|
||||
await db.invalidate_memory(memory_id)
|
||||
await vector_store.delete_memory(memory_id)
|
||||
else:
|
||||
await db.update_memory_confidence(memory_id, new_confidence)
|
||||
|
||||
elif request.action == FeedbackAction.EDIT:
|
||||
if request.new_value:
|
||||
await db.update_memory_text(memory_id, request.new_value)
|
||||
# Re-index with new text
|
||||
await vector_store.delete_memory(memory_id)
|
||||
await vector_store.index_memory(
|
||||
memory_id=memory_id,
|
||||
text=request.new_value,
|
||||
org_id=memory["org_id"],
|
||||
user_id=memory["user_id"],
|
||||
category=memory["category"],
|
||||
agent_id=memory.get("agent_id"),
|
||||
workspace_id=memory.get("workspace_id")
|
||||
)
|
||||
|
||||
elif request.action == FeedbackAction.DELETE:
|
||||
await db.invalidate_memory(memory_id)
|
||||
await vector_store.delete_memory(memory_id)
|
||||
|
||||
return {"status": "ok", "action": request.action.value}
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# RETRIEVAL (Semantic Search)
|
||||
# ============================================================================
|
||||
|
||||
@app.post("/retrieve", response_model=RetrievalResponse)
|
||||
async def retrieve_memories(request: RetrievalRequest):
|
||||
"""
|
||||
Semantic retrieval of relevant memories.
|
||||
|
||||
# Перевірка прав доступу (користувач може змінювати тільки свої факти)
|
||||
if fact_request.user_id != user_id:
|
||||
raise HTTPException(status_code=403, detail="Cannot modify other user's facts")
|
||||
Performs multiple queries and deduplicates results.
|
||||
"""
|
||||
all_results = []
|
||||
seen_ids = set()
|
||||
|
||||
fact, created = upsert_user_fact(db, fact_request)
|
||||
for query in request.queries:
|
||||
results = await vector_store.search_memories(
|
||||
query=query,
|
||||
org_id=request.org_id,
|
||||
user_id=request.user_id,
|
||||
agent_id=request.agent_id,
|
||||
workspace_id=request.workspace_id,
|
||||
categories=request.categories,
|
||||
include_global=request.include_global,
|
||||
top_k=request.top_k
|
||||
)
|
||||
|
||||
for r in results:
|
||||
memory_id = r.get("memory_id")
|
||||
if memory_id and memory_id not in seen_ids:
|
||||
seen_ids.add(memory_id)
|
||||
|
||||
# Get full memory from DB for confidence check
|
||||
memory = await db.get_memory(UUID(memory_id))
|
||||
if memory and memory["confidence"] >= request.min_confidence:
|
||||
all_results.append(RetrievalResult(
|
||||
memory_id=UUID(memory_id),
|
||||
fact_text=r["text"],
|
||||
category=MemoryCategory(r["category"]),
|
||||
confidence=memory["confidence"],
|
||||
relevance_score=r["score"],
|
||||
agent_id=UUID(r["agent_id"]) if r.get("agent_id") else None,
|
||||
is_global=r.get("agent_id") is None
|
||||
))
|
||||
|
||||
# Update usage stats
|
||||
await db.increment_memory_usage(UUID(memory_id))
|
||||
|
||||
return UserFactUpsertResponse(
|
||||
fact=UserFactResponse.model_validate(fact),
|
||||
created=created
|
||||
# Sort by relevance
|
||||
all_results.sort(key=lambda x: x.relevance_score, reverse=True)
|
||||
|
||||
return RetrievalResponse(
|
||||
results=all_results[:request.top_k],
|
||||
query_count=len(request.queries),
|
||||
total_results=len(all_results)
|
||||
)
|
||||
|
||||
|
||||
@app.get("/facts", response_model=List[UserFactResponse])
|
||||
async def list_facts(
|
||||
team_id: Optional[str] = Query(None),
|
||||
fact_keys: Optional[str] = Query(None, description="Comma-separated list of fact keys"),
|
||||
skip: int = Query(0, ge=0),
|
||||
limit: int = Query(100, ge=1, le=1000),
|
||||
db: Session = Depends(get_db),
|
||||
user_id: str = Depends(verify_token)
|
||||
):
|
||||
"""Отримати список фактів користувача"""
|
||||
fact_keys_list = None
|
||||
if fact_keys:
|
||||
fact_keys_list = [k.strip() for k in fact_keys.split(",")]
|
||||
|
||||
facts = get_user_facts(db, user_id, team_id, fact_keys_list, skip, limit)
|
||||
return [UserFactResponse.model_validate(f) for f in facts]
|
||||
# ============================================================================
|
||||
# SUMMARIES (Mid-term Memory)
|
||||
# ============================================================================
|
||||
|
||||
|
||||
@app.get("/facts/{fact_key}", response_model=UserFactResponse)
|
||||
async def get_fact(
|
||||
fact_key: str,
|
||||
team_id: Optional[str] = Query(None),
|
||||
db: Session = Depends(get_db),
|
||||
user_id: str = Depends(verify_token)
|
||||
):
|
||||
"""Отримати конкретний факт за ключем"""
|
||||
fact = get_user_fact(db, user_id, fact_key, team_id)
|
||||
if not fact:
|
||||
raise HTTPException(status_code=404, detail="Fact not found")
|
||||
return UserFactResponse.model_validate(fact)
|
||||
|
||||
|
||||
@app.post("/facts", response_model=UserFactResponse)
|
||||
async def create_fact(
|
||||
fact: UserFactCreate,
|
||||
db: Session = Depends(get_db),
|
||||
user_id: str = Depends(verify_token)
|
||||
):
|
||||
"""Створити новий факт"""
|
||||
if fact.user_id != user_id:
|
||||
raise HTTPException(status_code=403, detail="Cannot create fact for other user")
|
||||
|
||||
db_fact = create_user_fact(db, fact)
|
||||
return UserFactResponse.model_validate(db_fact)
|
||||
|
||||
|
||||
@app.patch("/facts/{fact_id}", response_model=UserFactResponse)
|
||||
async def update_fact(
|
||||
fact_id: str,
|
||||
fact_update: UserFactUpdate,
|
||||
db: Session = Depends(get_db),
|
||||
user_id: str = Depends(verify_token)
|
||||
):
|
||||
"""Оновити факт"""
|
||||
fact = db.query(UserFact).filter(UserFact.id == fact_id).first()
|
||||
if not fact:
|
||||
raise HTTPException(status_code=404, detail="Fact not found")
|
||||
|
||||
if fact.user_id != user_id:
|
||||
raise HTTPException(status_code=403, detail="Cannot modify other user's fact")
|
||||
|
||||
updated_fact = update_user_fact(db, fact_id, fact_update)
|
||||
if not updated_fact:
|
||||
raise HTTPException(status_code=404, detail="Fact not found")
|
||||
|
||||
return UserFactResponse.model_validate(updated_fact)
|
||||
|
||||
|
||||
@app.delete("/facts/{fact_id}")
|
||||
async def delete_fact(
|
||||
fact_id: str,
|
||||
db: Session = Depends(get_db),
|
||||
user_id: str = Depends(verify_token)
|
||||
):
|
||||
"""Видалити факт"""
|
||||
fact = db.query(UserFact).filter(UserFact.id == fact_id).first()
|
||||
if not fact:
|
||||
raise HTTPException(status_code=404, detail="Fact not found")
|
||||
|
||||
if fact.user_id != user_id:
|
||||
raise HTTPException(status_code=403, detail="Cannot delete other user's fact")
|
||||
|
||||
success = delete_user_fact(db, fact_id)
|
||||
if not success:
|
||||
raise HTTPException(status_code=404, detail="Fact not found")
|
||||
|
||||
return {"success": True}
|
||||
|
||||
|
||||
@app.get("/facts/token-gated", response_model=List[UserFactResponse])
|
||||
async def list_token_gated_facts(
|
||||
team_id: Optional[str] = Query(None),
|
||||
db: Session = Depends(get_db),
|
||||
user_id: str = Depends(verify_token)
|
||||
):
|
||||
"""Отримати токен-гейт факти користувача"""
|
||||
facts = get_user_facts_by_token_gate(db, user_id, team_id)
|
||||
return [UserFactResponse.model_validate(f) for f in facts]
|
||||
|
||||
|
||||
# ========== Dialog Summary Endpoints ==========
|
||||
|
||||
@app.post("/summaries", response_model=DialogSummaryResponse)
|
||||
async def create_summary(
|
||||
summary: DialogSummaryCreate,
|
||||
db: Session = Depends(get_db),
|
||||
user_id: str = Depends(verify_token)
|
||||
):
|
||||
@app.post("/threads/{thread_id}/summarize", response_model=SummaryResponse)
|
||||
async def create_summary(thread_id: UUID, request: SummaryRequest):
|
||||
"""
|
||||
Створити підсумок діалогу
|
||||
Generate rolling summary for thread.
|
||||
|
||||
Використовується для масштабування без переповнення контексту.
|
||||
Агрегує інформацію про сесії/діалоги.
|
||||
Compresses old events into a structured summary.
|
||||
"""
|
||||
db_summary = create_dialog_summary(db, summary)
|
||||
return DialogSummaryResponse.model_validate(db_summary)
|
||||
|
||||
|
||||
@app.get("/summaries", response_model=DialogSummaryListResponse)
|
||||
async def list_summaries(
|
||||
team_id: Optional[str] = Query(None),
|
||||
channel_id: Optional[str] = Query(None),
|
||||
agent_id: Optional[str] = Query(None),
|
||||
user_id_param: Optional[str] = Query(None, alias="user_id"),
|
||||
skip: int = Query(0, ge=0),
|
||||
limit: int = Query(50, ge=1, le=200),
|
||||
cursor: Optional[str] = Query(None),
|
||||
db: Session = Depends(get_db),
|
||||
user_id: str = Depends(verify_token)
|
||||
):
|
||||
"""Отримати список підсумків діалогів"""
|
||||
summaries, next_cursor = get_dialog_summaries(
|
||||
db, team_id, channel_id, agent_id, user_id_param, skip, limit, cursor
|
||||
thread = await db.get_thread(thread_id)
|
||||
if not thread:
|
||||
raise HTTPException(status_code=404, detail="Thread not found")
|
||||
|
||||
# Check if summary is needed
|
||||
if not request.force and thread["total_tokens"] < settings.summary_trigger_tokens:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Token count ({thread['total_tokens']}) below threshold ({settings.summary_trigger_tokens})"
|
||||
)
|
||||
|
||||
# Get events to summarize
|
||||
events = await db.get_events_for_summary(thread_id)
|
||||
|
||||
# TODO: Call LLM to generate summary
|
||||
# For now, create a placeholder
|
||||
summary_text = f"Summary of {len(events)} events. [Implement LLM summarization]"
|
||||
state = {
|
||||
"goals": [],
|
||||
"decisions": [],
|
||||
"open_questions": [],
|
||||
"next_steps": [],
|
||||
"key_facts": []
|
||||
}
|
||||
|
||||
# Create summary
|
||||
summary = await db.create_summary(
|
||||
thread_id=thread_id,
|
||||
summary_text=summary_text,
|
||||
state=state,
|
||||
events_from_seq=events[0]["sequence_num"] if events else 0,
|
||||
events_to_seq=events[-1]["sequence_num"] if events else 0,
|
||||
events_count=len(events)
|
||||
)
|
||||
|
||||
return DialogSummaryListResponse(
|
||||
items=[DialogSummaryResponse.model_validate(s) for s in summaries],
|
||||
total=len(summaries),
|
||||
cursor=next_cursor
|
||||
# Index summary in Qdrant
|
||||
await vector_store.index_summary(
|
||||
summary_id=summary["summary_id"],
|
||||
text=summary_text,
|
||||
thread_id=thread_id,
|
||||
org_id=thread["org_id"],
|
||||
user_id=thread["user_id"],
|
||||
agent_id=thread.get("agent_id"),
|
||||
workspace_id=thread.get("workspace_id")
|
||||
)
|
||||
|
||||
return summary
|
||||
|
||||
|
||||
@app.get("/threads/{thread_id}/summary", response_model=Optional[SummaryResponse])
|
||||
async def get_latest_summary(thread_id: UUID):
|
||||
"""Get latest summary for thread"""
|
||||
summary = await db.get_latest_summary(thread_id)
|
||||
return summary
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# CONTEXT (Full context for agent)
|
||||
# ============================================================================
|
||||
|
||||
@app.get("/threads/{thread_id}/context", response_model=ContextResponse)
|
||||
async def get_context(
|
||||
thread_id: UUID,
|
||||
queries: List[str] = Query(default=[]),
|
||||
top_k: int = Query(default=10)
|
||||
):
|
||||
"""
|
||||
Get full context for agent prompt.
|
||||
|
||||
Combines:
|
||||
- Latest summary (mid-term)
|
||||
- Recent messages (short-term)
|
||||
- Retrieved memories (long-term)
|
||||
"""
|
||||
thread = await db.get_thread(thread_id)
|
||||
if not thread:
|
||||
raise HTTPException(status_code=404, detail="Thread not found")
|
||||
|
||||
# Get summary
|
||||
summary = await db.get_latest_summary(thread_id)
|
||||
|
||||
# Get recent messages
|
||||
recent = await db.get_events(
|
||||
thread_id,
|
||||
limit=settings.short_term_window_messages
|
||||
)
|
||||
|
||||
# Retrieve memories if queries provided
|
||||
retrieved = []
|
||||
if queries:
|
||||
retrieval_response = await retrieve_memories(RetrievalRequest(
|
||||
org_id=thread["org_id"],
|
||||
user_id=thread["user_id"],
|
||||
agent_id=thread.get("agent_id"),
|
||||
workspace_id=thread.get("workspace_id"),
|
||||
queries=queries,
|
||||
top_k=top_k,
|
||||
include_global=True
|
||||
))
|
||||
retrieved = retrieval_response.results
|
||||
|
||||
# Estimate tokens
|
||||
token_estimate = sum(e.get("token_count", 0) or 0 for e in recent)
|
||||
if summary:
|
||||
token_estimate += summary.get("summary_tokens", 0) or 0
|
||||
|
||||
return ContextResponse(
|
||||
thread_id=thread_id,
|
||||
summary=summary,
|
||||
recent_messages=recent,
|
||||
retrieved_memories=retrieved,
|
||||
token_estimate=token_estimate
|
||||
)
|
||||
|
||||
|
||||
@app.get("/summaries/{summary_id}", response_model=DialogSummaryResponse)
|
||||
async def get_summary(
|
||||
summary_id: str,
|
||||
db: Session = Depends(get_db),
|
||||
user_id: str = Depends(verify_token)
|
||||
):
|
||||
"""Отримати підсумок за ID"""
|
||||
summary = get_dialog_summary(db, summary_id)
|
||||
if not summary:
|
||||
raise HTTPException(status_code=404, detail="Summary not found")
|
||||
return DialogSummaryResponse.model_validate(summary)
|
||||
# ============================================================================
|
||||
# ADMIN
|
||||
# ============================================================================
|
||||
|
||||
|
||||
@app.delete("/summaries/{summary_id}")
|
||||
async def delete_summary(
|
||||
summary_id: str,
|
||||
db: Session = Depends(get_db),
|
||||
user_id: str = Depends(verify_token)
|
||||
):
|
||||
"""Видалити підсумок"""
|
||||
success = delete_dialog_summary(db, summary_id)
|
||||
if not success:
|
||||
raise HTTPException(status_code=404, detail="Summary not found")
|
||||
return {"success": True}
|
||||
|
||||
|
||||
# ========== Agent Memory Event Endpoints ==========
|
||||
|
||||
@app.post("/agents/{agent_id}/memory", response_model=AgentMemoryEventResponse)
|
||||
async def create_memory_event(
|
||||
agent_id: str,
|
||||
event: AgentMemoryEventCreate,
|
||||
db: Session = Depends(get_db),
|
||||
user_id: str = Depends(verify_token)
|
||||
):
|
||||
"""Створити подію пам'яті агента"""
|
||||
# Перевірка що agent_id збігається
|
||||
if event.agent_id != agent_id:
|
||||
raise HTTPException(status_code=400, detail="agent_id mismatch")
|
||||
|
||||
db_event = create_agent_memory_event(db, event)
|
||||
return AgentMemoryEventResponse.model_validate(db_event)
|
||||
|
||||
|
||||
@app.get("/agents/{agent_id}/memory", response_model=AgentMemoryEventListResponse)
|
||||
async def list_memory_events(
|
||||
agent_id: str,
|
||||
team_id: Optional[str] = Query(None),
|
||||
channel_id: Optional[str] = Query(None),
|
||||
scope: Optional[str] = Query(None, description="short_term | mid_term | long_term"),
|
||||
kind: Optional[str] = Query(None, description="message | fact | summary | note"),
|
||||
skip: int = Query(0, ge=0),
|
||||
limit: int = Query(50, ge=1, le=200),
|
||||
cursor: Optional[str] = Query(None),
|
||||
db: Session = Depends(get_db),
|
||||
user_id: str = Depends(verify_token)
|
||||
):
|
||||
"""Отримати список подій пам'яті агента"""
|
||||
events, next_cursor = get_agent_memory_events(
|
||||
db, agent_id, team_id, channel_id, scope, kind, skip, limit, cursor
|
||||
)
|
||||
|
||||
return AgentMemoryEventListResponse(
|
||||
items=[AgentMemoryEventResponse.model_validate(e) for e in events],
|
||||
total=len(events),
|
||||
cursor=next_cursor
|
||||
)
|
||||
|
||||
|
||||
@app.delete("/agents/{agent_id}/memory/{event_id}")
|
||||
async def delete_memory_event(
|
||||
agent_id: str,
|
||||
event_id: str,
|
||||
db: Session = Depends(get_db),
|
||||
user_id: str = Depends(verify_token)
|
||||
):
|
||||
"""Видалити подію пам'яті"""
|
||||
success = delete_agent_memory_event(db, event_id)
|
||||
if not success:
|
||||
raise HTTPException(status_code=404, detail="Memory event not found")
|
||||
return {"success": True}
|
||||
|
||||
|
||||
# ========== Monitor Events Endpoints (Batch Processing) ==========
|
||||
|
||||
from app.monitor_events import MonitorEventBatch, MonitorEventResponse, save_monitor_events_batch, save_monitor_event_single
|
||||
|
||||
@app.post("/api/memory/monitor-events/batch", response_model=MonitorEventResponse)
|
||||
async def save_monitor_events_batch_endpoint(
|
||||
batch: MonitorEventBatch,
|
||||
db: Session = Depends(get_db),
|
||||
authorization: Optional[str] = Header(None)
|
||||
):
|
||||
"""
|
||||
Зберегти батч подій Monitor Agent
|
||||
Оптимізовано для збору метрик з багатьох нод
|
||||
"""
|
||||
return await save_monitor_events_batch(batch, db, authorization)
|
||||
|
||||
@app.post("/api/memory/monitor-events/{node_id}", response_model=AgentMemoryEventResponse)
|
||||
async def save_monitor_event_endpoint(
|
||||
node_id: str,
|
||||
event: Dict[str, Any],
|
||||
db: Session = Depends(get_db),
|
||||
authorization: Optional[str] = Header(None)
|
||||
):
|
||||
"""
|
||||
Зберегти одну подію Monitor Agent
|
||||
"""
|
||||
return await save_monitor_event_single(node_id, event, db, authorization)
|
||||
|
||||
|
||||
# ========== Token Gate Integration Endpoint ==========
|
||||
|
||||
@app.post("/token-gate/check", response_model=TokenGateCheckResponse)
|
||||
async def check_token_gate_endpoint(
|
||||
check: TokenGateCheck,
|
||||
db: Session = Depends(get_db),
|
||||
user_id: str = Depends(verify_token)
|
||||
):
|
||||
"""
|
||||
Перевірка токен-гейту для факту
|
||||
|
||||
Інтеграція з RBAC/Wallet Service для перевірки доступу
|
||||
"""
|
||||
if check.user_id != user_id:
|
||||
raise HTTPException(status_code=403, detail="Cannot check token gate for other user")
|
||||
|
||||
return await check_token_gate(user_id, check.token_requirements, db)
|
||||
@app.get("/stats")
|
||||
async def get_stats():
|
||||
"""Get service statistics"""
|
||||
return {
|
||||
"vector_store": await vector_store.get_collection_stats(),
|
||||
"database": await db.get_stats()
|
||||
}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import uvicorn
|
||||
uvicorn.run("app.main:app", host="0.0.0.0", port=8000, reload=True)
|
||||
|
||||
uvicorn.run(app, host="0.0.0.0", port=8000)
|
||||
|
||||
@@ -1,178 +1,249 @@
|
||||
"""
|
||||
SQLAlchemy моделі для Memory Service
|
||||
Підтримує: user_facts, dialog_summaries, agent_memory_events
|
||||
DAARION Memory Service - Pydantic Models
|
||||
"""
|
||||
|
||||
from sqlalchemy import (
|
||||
Column, String, Text, JSON, TIMESTAMP,
|
||||
CheckConstraint, Index, Boolean, Integer
|
||||
)
|
||||
from sqlalchemy.ext.declarative import declarative_base
|
||||
from sqlalchemy.sql import func
|
||||
import os
|
||||
|
||||
# Перевірка типу бази даних
|
||||
DATABASE_URL = os.getenv("DATABASE_URL", "sqlite:///./memory.db")
|
||||
IS_SQLITE = "sqlite" in DATABASE_URL.lower()
|
||||
|
||||
if IS_SQLITE:
|
||||
# Для SQLite використовуємо стандартні типи
|
||||
from sqlalchemy import JSON as JSONB_TYPE
|
||||
UUID_TYPE = String # SQLite не має UUID, використовуємо String
|
||||
else:
|
||||
# Для PostgreSQL використовуємо специфічні типи
|
||||
from sqlalchemy.dialects.postgresql import UUID, JSONB
|
||||
UUID_TYPE = UUID
|
||||
JSONB_TYPE = JSONB
|
||||
|
||||
try:
|
||||
from pgvector.sqlalchemy import Vector
|
||||
HAS_PGVECTOR = True
|
||||
except ImportError:
|
||||
HAS_PGVECTOR = False
|
||||
# Заглушка для SQLite
|
||||
class Vector:
|
||||
def __init__(self, *args, **kwargs):
|
||||
pass
|
||||
|
||||
Base = declarative_base()
|
||||
from datetime import datetime
|
||||
from typing import Optional, List, Any
|
||||
from uuid import UUID
|
||||
from enum import Enum
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class UserFact(Base):
|
||||
"""
|
||||
Довгострокові факти про користувача
|
||||
Використовується для контрольованої довгострокової пам'яті
|
||||
(мови, вподобання, тип користувача, токен-статуси)
|
||||
"""
|
||||
__tablename__ = "user_facts"
|
||||
# ============================================================================
|
||||
# ENUMS
|
||||
# ============================================================================
|
||||
|
||||
id = Column(UUID_TYPE(as_uuid=False) if not IS_SQLITE else String, primary_key=True, server_default=func.gen_random_uuid() if not IS_SQLITE else None)
|
||||
user_id = Column(String, nullable=False, index=True) # Без FK constraint для тестування
|
||||
team_id = Column(String, nullable=True, index=True) # Без FK constraint, оскільки teams може не існувати
|
||||
|
||||
# Ключ факту (наприклад: "language", "is_donor", "is_validator", "top_contributor")
|
||||
fact_key = Column(String, nullable=False, index=True)
|
||||
|
||||
# Значення факту (може бути текст, число, boolean, JSON)
|
||||
fact_value = Column(Text, nullable=True)
|
||||
fact_value_json = Column(JSONB_TYPE, nullable=True)
|
||||
|
||||
# Метадані: джерело, впевненість, термін дії
|
||||
meta = Column(JSONB_TYPE, nullable=False, server_default="{}")
|
||||
|
||||
# Токен-гейт: чи залежить факт від токенів/активності
|
||||
token_gated = Column(Boolean, nullable=False, server_default="false")
|
||||
token_requirements = Column(JSONB_TYPE, nullable=True) # {"token": "DAAR", "min_balance": 1}
|
||||
|
||||
created_at = Column(TIMESTAMP(timezone=True) if not IS_SQLITE else TIMESTAMP, nullable=False, server_default=func.now())
|
||||
updated_at = Column(TIMESTAMP(timezone=True) if not IS_SQLITE else TIMESTAMP, nullable=True, onupdate=func.now())
|
||||
expires_at = Column(TIMESTAMP(timezone=True) if not IS_SQLITE else TIMESTAMP, nullable=True) # Для тимчасових фактів
|
||||
|
||||
__table_args__ = (
|
||||
Index("idx_user_facts_user_key", "user_id", "fact_key"),
|
||||
Index("idx_user_facts_team", "team_id"),
|
||||
Index("idx_user_facts_token_gated", "token_gated"),
|
||||
)
|
||||
class EventType(str, Enum):
|
||||
MESSAGE = "message"
|
||||
TOOL_CALL = "tool_call"
|
||||
TOOL_RESULT = "tool_result"
|
||||
DECISION = "decision"
|
||||
SUMMARY = "summary"
|
||||
MEMORY_WRITE = "memory_write"
|
||||
MEMORY_RETRACT = "memory_retract"
|
||||
ERROR = "error"
|
||||
|
||||
|
||||
class DialogSummary(Base):
|
||||
"""
|
||||
Підсумки діалогів для масштабування без переповнення контексту
|
||||
Зберігає агреговану інформацію про сесії/діалоги
|
||||
"""
|
||||
__tablename__ = "dialog_summaries"
|
||||
|
||||
id = Column(UUID_TYPE(as_uuid=False) if not IS_SQLITE else String, primary_key=True, server_default=func.gen_random_uuid() if not IS_SQLITE else None)
|
||||
|
||||
# Контекст діалогу (без FK constraints для тестування)
|
||||
team_id = Column(String, nullable=False, index=True)
|
||||
channel_id = Column(String, nullable=True, index=True)
|
||||
agent_id = Column(String, nullable=True, index=True)
|
||||
user_id = Column(String, nullable=True, index=True)
|
||||
|
||||
# Період, який охоплює підсумок
|
||||
period_start = Column(TIMESTAMP(timezone=True) if not IS_SQLITE else TIMESTAMP, nullable=False)
|
||||
period_end = Column(TIMESTAMP(timezone=True) if not IS_SQLITE else TIMESTAMP, nullable=False)
|
||||
|
||||
# Підсумок
|
||||
summary_text = Column(Text, nullable=False)
|
||||
summary_json = Column(JSONB_TYPE, nullable=True) # Структуровані дані
|
||||
|
||||
# Статистика
|
||||
message_count = Column(Integer, nullable=False, server_default="0")
|
||||
participant_count = Column(Integer, nullable=False, server_default="0")
|
||||
|
||||
# Ключові теми/теги
|
||||
topics = Column(JSONB_TYPE, nullable=True) # ["project-planning", "bug-fix", ...]
|
||||
|
||||
# Метадані
|
||||
meta = Column(JSONB_TYPE, nullable=False, server_default="{}")
|
||||
|
||||
created_at = Column(TIMESTAMP(timezone=True) if not IS_SQLITE else TIMESTAMP, nullable=False, server_default=func.now())
|
||||
|
||||
__table_args__ = (
|
||||
Index("idx_dialog_summaries_team_period", "team_id", "period_start", "period_end"),
|
||||
Index("idx_dialog_summaries_channel", "channel_id"),
|
||||
Index("idx_dialog_summaries_agent", "agent_id"),
|
||||
)
|
||||
class MessageRole(str, Enum):
|
||||
USER = "user"
|
||||
ASSISTANT = "assistant"
|
||||
SYSTEM = "system"
|
||||
TOOL = "tool"
|
||||
|
||||
|
||||
class AgentMemoryEvent(Base):
|
||||
"""
|
||||
Події пам'яті агентів (short-term, mid-term, long-term)
|
||||
Базується на документації: docs/cursor/13_agent_memory_system.md
|
||||
"""
|
||||
__tablename__ = "agent_memory_events"
|
||||
|
||||
id = Column(UUID_TYPE(as_uuid=False) if not IS_SQLITE else String, primary_key=True, server_default=func.gen_random_uuid() if not IS_SQLITE else None)
|
||||
|
||||
# Без FK constraints для тестування
|
||||
agent_id = Column(String, nullable=False, index=True)
|
||||
team_id = Column(String, nullable=False, index=True)
|
||||
channel_id = Column(String, nullable=True, index=True)
|
||||
user_id = Column(String, nullable=True, index=True)
|
||||
|
||||
# Scope: short_term, mid_term, long_term
|
||||
scope = Column(String, nullable=False)
|
||||
|
||||
# Kind: message, fact, summary, note
|
||||
kind = Column(String, nullable=False)
|
||||
|
||||
# Тіло події
|
||||
body_text = Column(Text, nullable=True)
|
||||
body_json = Column(JSONB_TYPE, nullable=True)
|
||||
|
||||
created_at = Column(TIMESTAMP(timezone=True) if not IS_SQLITE else TIMESTAMP, nullable=False, server_default=func.now())
|
||||
|
||||
__table_args__ = (
|
||||
CheckConstraint("scope IN ('short_term', 'mid_term', 'long_term')", name="ck_agent_memory_scope"),
|
||||
CheckConstraint("kind IN ('message', 'fact', 'summary', 'note')", name="ck_agent_memory_kind"),
|
||||
Index("idx_agent_memory_events_agent_team_scope", "agent_id", "team_id", "scope"),
|
||||
Index("idx_agent_memory_events_channel", "agent_id", "channel_id"),
|
||||
Index("idx_agent_memory_events_created_at", "created_at"),
|
||||
)
|
||||
class MemoryCategory(str, Enum):
|
||||
PREFERENCE = "preference"
|
||||
IDENTITY = "identity"
|
||||
CONSTRAINT = "constraint"
|
||||
PROJECT_FACT = "project_fact"
|
||||
RELATIONSHIP = "relationship"
|
||||
SKILL = "skill"
|
||||
GOAL = "goal"
|
||||
CONTEXT = "context"
|
||||
FEEDBACK = "feedback"
|
||||
|
||||
|
||||
class AgentMemoryFactsVector(Base):
|
||||
"""
|
||||
Векторні представлення фактів для RAG (Retrieval-Augmented Generation)
|
||||
"""
|
||||
__tablename__ = "agent_memory_facts_vector"
|
||||
class RetentionPolicy(str, Enum):
|
||||
PERMANENT = "permanent"
|
||||
SESSION = "session"
|
||||
TTL_DAYS = "ttl_days"
|
||||
UNTIL_REVOKED = "until_revoked"
|
||||
|
||||
id = Column(UUID_TYPE(as_uuid=False) if not IS_SQLITE else String, primary_key=True, server_default=func.gen_random_uuid() if not IS_SQLITE else None)
|
||||
|
||||
# Без FK constraints для тестування
|
||||
agent_id = Column(String, nullable=False, index=True)
|
||||
team_id = Column(String, nullable=False, index=True)
|
||||
|
||||
fact_text = Column(Text, nullable=False)
|
||||
embedding = Column(Vector(1536), nullable=True) if HAS_PGVECTOR else Column(Text, nullable=True) # OpenAI ada-002 embedding size
|
||||
|
||||
meta = Column(JSONB_TYPE, nullable=False, server_default="{}")
|
||||
|
||||
created_at = Column(TIMESTAMP(timezone=True) if not IS_SQLITE else TIMESTAMP, nullable=False, server_default=func.now())
|
||||
|
||||
__table_args__ = (
|
||||
Index("idx_agent_memory_facts_vector_agent_team", "agent_id", "team_id"),
|
||||
)
|
||||
class FeedbackAction(str, Enum):
|
||||
CONFIRM = "confirm"
|
||||
REJECT = "reject"
|
||||
EDIT = "edit"
|
||||
DELETE = "delete"
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# REQUEST MODELS
|
||||
# ============================================================================
|
||||
|
||||
class CreateThreadRequest(BaseModel):
|
||||
"""Create new conversation thread"""
|
||||
org_id: UUID
|
||||
workspace_id: Optional[UUID] = None
|
||||
user_id: UUID
|
||||
agent_id: Optional[UUID] = None
|
||||
title: Optional[str] = None
|
||||
tags: List[str] = []
|
||||
metadata: dict = {}
|
||||
|
||||
|
||||
class AddEventRequest(BaseModel):
|
||||
"""Add event to conversation"""
|
||||
thread_id: UUID
|
||||
event_type: EventType
|
||||
role: Optional[MessageRole] = None
|
||||
content: Optional[str] = None
|
||||
tool_name: Optional[str] = None
|
||||
tool_input: Optional[dict] = None
|
||||
tool_output: Optional[dict] = None
|
||||
payload: dict = {}
|
||||
token_count: Optional[int] = None
|
||||
model_used: Optional[str] = None
|
||||
latency_ms: Optional[int] = None
|
||||
metadata: dict = {}
|
||||
|
||||
|
||||
class CreateMemoryRequest(BaseModel):
|
||||
"""Create long-term memory item"""
|
||||
org_id: UUID
|
||||
workspace_id: Optional[UUID] = None
|
||||
user_id: UUID
|
||||
agent_id: Optional[UUID] = None # null = global
|
||||
category: MemoryCategory
|
||||
fact_text: str
|
||||
confidence: float = Field(default=0.8, ge=0, le=1)
|
||||
source_event_id: Optional[UUID] = None
|
||||
source_thread_id: Optional[UUID] = None
|
||||
extraction_method: str = "explicit"
|
||||
is_sensitive: bool = False
|
||||
retention: RetentionPolicy = RetentionPolicy.UNTIL_REVOKED
|
||||
ttl_days: Optional[int] = None
|
||||
tags: List[str] = []
|
||||
metadata: dict = {}
|
||||
|
||||
|
||||
class MemoryFeedbackRequest(BaseModel):
|
||||
"""User feedback on memory"""
|
||||
memory_id: UUID
|
||||
user_id: UUID
|
||||
action: FeedbackAction
|
||||
new_value: Optional[str] = None
|
||||
reason: Optional[str] = None
|
||||
|
||||
|
||||
class RetrievalRequest(BaseModel):
|
||||
"""Semantic retrieval request"""
|
||||
org_id: UUID
|
||||
user_id: UUID
|
||||
agent_id: Optional[UUID] = None
|
||||
workspace_id: Optional[UUID] = None
|
||||
queries: List[str]
|
||||
top_k: int = 10
|
||||
min_confidence: float = 0.5
|
||||
include_global: bool = True
|
||||
categories: Optional[List[MemoryCategory]] = None
|
||||
|
||||
|
||||
class SummaryRequest(BaseModel):
|
||||
"""Generate summary for thread"""
|
||||
thread_id: UUID
|
||||
force: bool = False # force even if under token threshold
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# RESPONSE MODELS
|
||||
# ============================================================================
|
||||
|
||||
class ThreadResponse(BaseModel):
|
||||
thread_id: UUID
|
||||
org_id: UUID
|
||||
workspace_id: Optional[UUID]
|
||||
user_id: UUID
|
||||
agent_id: Optional[UUID]
|
||||
title: Optional[str]
|
||||
status: str
|
||||
message_count: int
|
||||
total_tokens: int
|
||||
created_at: datetime
|
||||
last_activity_at: datetime
|
||||
|
||||
|
||||
class EventResponse(BaseModel):
|
||||
event_id: UUID
|
||||
thread_id: UUID
|
||||
event_type: EventType
|
||||
role: Optional[MessageRole]
|
||||
content: Optional[str]
|
||||
tool_name: Optional[str]
|
||||
tool_input: Optional[dict]
|
||||
tool_output: Optional[dict]
|
||||
payload: dict
|
||||
token_count: Optional[int]
|
||||
sequence_num: int
|
||||
created_at: datetime
|
||||
|
||||
|
||||
class MemoryResponse(BaseModel):
|
||||
memory_id: UUID
|
||||
org_id: UUID
|
||||
workspace_id: Optional[UUID]
|
||||
user_id: UUID
|
||||
agent_id: Optional[UUID]
|
||||
category: MemoryCategory
|
||||
fact_text: str
|
||||
confidence: float
|
||||
is_verified: bool
|
||||
is_sensitive: bool
|
||||
retention: RetentionPolicy
|
||||
valid_from: datetime
|
||||
valid_to: Optional[datetime]
|
||||
last_used_at: Optional[datetime]
|
||||
use_count: int
|
||||
created_at: datetime
|
||||
|
||||
|
||||
class SummaryResponse(BaseModel):
|
||||
summary_id: UUID
|
||||
thread_id: UUID
|
||||
version: int
|
||||
summary_text: str
|
||||
state: dict
|
||||
events_count: int
|
||||
compression_ratio: Optional[float]
|
||||
created_at: datetime
|
||||
|
||||
|
||||
class RetrievalResult(BaseModel):
|
||||
"""Single retrieval result"""
|
||||
memory_id: UUID
|
||||
fact_text: str
|
||||
category: MemoryCategory
|
||||
confidence: float
|
||||
relevance_score: float
|
||||
agent_id: Optional[UUID]
|
||||
is_global: bool
|
||||
|
||||
|
||||
class RetrievalResponse(BaseModel):
|
||||
"""Retrieval response with results"""
|
||||
results: List[RetrievalResult]
|
||||
query_count: int
|
||||
total_results: int
|
||||
|
||||
|
||||
class ContextResponse(BaseModel):
|
||||
"""Full context for agent prompt"""
|
||||
thread_id: UUID
|
||||
summary: Optional[SummaryResponse]
|
||||
recent_messages: List[EventResponse]
|
||||
retrieved_memories: List[RetrievalResult]
|
||||
token_estimate: int
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# INTERNAL MODELS
|
||||
# ============================================================================
|
||||
|
||||
class EmbeddingRequest(BaseModel):
|
||||
"""Internal embedding request"""
|
||||
texts: List[str]
|
||||
input_type: str = "search_document" # or "search_query"
|
||||
|
||||
|
||||
class QdrantPayload(BaseModel):
|
||||
"""Qdrant point payload"""
|
||||
org_id: str
|
||||
workspace_id: Optional[str]
|
||||
user_id: str
|
||||
agent_id: Optional[str]
|
||||
thread_id: Optional[str]
|
||||
memory_id: Optional[str]
|
||||
event_id: Optional[str]
|
||||
type: str # "memory", "summary", "message"
|
||||
category: Optional[str]
|
||||
text: str
|
||||
created_at: str
|
||||
|
||||
325
services/memory-service/app/vector_store.py
Normal file
325
services/memory-service/app/vector_store.py
Normal file
@@ -0,0 +1,325 @@
|
||||
"""
|
||||
DAARION Memory Service - Qdrant Vector Store
|
||||
"""
|
||||
from typing import List, Optional, Dict, Any
|
||||
from uuid import UUID, uuid4
|
||||
import structlog
|
||||
from qdrant_client import QdrantClient
|
||||
from qdrant_client.http import models as qmodels
|
||||
|
||||
from .config import get_settings
|
||||
from .embedding import get_query_embedding, get_document_embeddings
|
||||
from .models import MemoryCategory, QdrantPayload
|
||||
|
||||
logger = structlog.get_logger()
|
||||
settings = get_settings()
|
||||
|
||||
|
||||
class VectorStore:
|
||||
"""Qdrant vector store for semantic search"""
|
||||
|
||||
def __init__(self):
|
||||
self.client = QdrantClient(
|
||||
host=settings.qdrant_host,
|
||||
port=settings.qdrant_port
|
||||
)
|
||||
self.memories_collection = settings.qdrant_collection_memories
|
||||
self.messages_collection = settings.qdrant_collection_messages
|
||||
|
||||
async def initialize(self):
|
||||
"""Initialize collections if they don't exist"""
|
||||
await self._ensure_collection(
|
||||
self.memories_collection,
|
||||
settings.embedding_dimensions
|
||||
)
|
||||
await self._ensure_collection(
|
||||
self.messages_collection,
|
||||
settings.embedding_dimensions
|
||||
)
|
||||
logger.info("vector_store_initialized")
|
||||
|
||||
async def _ensure_collection(self, name: str, dimensions: int):
|
||||
"""Create collection if it doesn't exist"""
|
||||
collections = self.client.get_collections().collections
|
||||
exists = any(c.name == name for c in collections)
|
||||
|
||||
if not exists:
|
||||
self.client.create_collection(
|
||||
collection_name=name,
|
||||
vectors_config=qmodels.VectorParams(
|
||||
size=dimensions,
|
||||
distance=qmodels.Distance.COSINE
|
||||
)
|
||||
)
|
||||
|
||||
# Create payload indexes for filtering
|
||||
self.client.create_payload_index(
|
||||
collection_name=name,
|
||||
field_name="org_id",
|
||||
field_schema=qmodels.PayloadSchemaType.KEYWORD
|
||||
)
|
||||
self.client.create_payload_index(
|
||||
collection_name=name,
|
||||
field_name="user_id",
|
||||
field_schema=qmodels.PayloadSchemaType.KEYWORD
|
||||
)
|
||||
self.client.create_payload_index(
|
||||
collection_name=name,
|
||||
field_name="agent_id",
|
||||
field_schema=qmodels.PayloadSchemaType.KEYWORD
|
||||
)
|
||||
self.client.create_payload_index(
|
||||
collection_name=name,
|
||||
field_name="type",
|
||||
field_schema=qmodels.PayloadSchemaType.KEYWORD
|
||||
)
|
||||
self.client.create_payload_index(
|
||||
collection_name=name,
|
||||
field_name="category",
|
||||
field_schema=qmodels.PayloadSchemaType.KEYWORD
|
||||
)
|
||||
|
||||
logger.info("collection_created", name=name, dimensions=dimensions)
|
||||
|
||||
async def index_memory(
|
||||
self,
|
||||
memory_id: UUID,
|
||||
text: str,
|
||||
org_id: UUID,
|
||||
user_id: UUID,
|
||||
category: MemoryCategory,
|
||||
agent_id: Optional[UUID] = None,
|
||||
workspace_id: Optional[UUID] = None,
|
||||
thread_id: Optional[UUID] = None,
|
||||
metadata: Dict[str, Any] = {}
|
||||
) -> str:
|
||||
"""
|
||||
Index a memory item in Qdrant.
|
||||
|
||||
Returns:
|
||||
Qdrant point ID
|
||||
"""
|
||||
# Get embedding
|
||||
embeddings = await get_document_embeddings([text])
|
||||
if not embeddings:
|
||||
raise ValueError("Failed to generate embedding")
|
||||
|
||||
vector = embeddings[0]
|
||||
point_id = str(uuid4())
|
||||
|
||||
# Build payload
|
||||
payload = {
|
||||
"org_id": str(org_id),
|
||||
"user_id": str(user_id),
|
||||
"memory_id": str(memory_id),
|
||||
"type": "memory",
|
||||
"category": category.value,
|
||||
"text": text,
|
||||
**metadata
|
||||
}
|
||||
|
||||
if agent_id:
|
||||
payload["agent_id"] = str(agent_id)
|
||||
if workspace_id:
|
||||
payload["workspace_id"] = str(workspace_id)
|
||||
if thread_id:
|
||||
payload["thread_id"] = str(thread_id)
|
||||
|
||||
# Upsert point
|
||||
self.client.upsert(
|
||||
collection_name=self.memories_collection,
|
||||
points=[
|
||||
qmodels.PointStruct(
|
||||
id=point_id,
|
||||
vector=vector,
|
||||
payload=payload
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"memory_indexed",
|
||||
memory_id=str(memory_id),
|
||||
point_id=point_id,
|
||||
category=category.value
|
||||
)
|
||||
|
||||
return point_id
|
||||
|
||||
async def index_summary(
|
||||
self,
|
||||
summary_id: UUID,
|
||||
text: str,
|
||||
thread_id: UUID,
|
||||
org_id: UUID,
|
||||
user_id: UUID,
|
||||
agent_id: Optional[UUID] = None,
|
||||
workspace_id: Optional[UUID] = None
|
||||
) -> str:
|
||||
"""Index a thread summary"""
|
||||
embeddings = await get_document_embeddings([text])
|
||||
if not embeddings:
|
||||
raise ValueError("Failed to generate embedding")
|
||||
|
||||
vector = embeddings[0]
|
||||
point_id = str(uuid4())
|
||||
|
||||
payload = {
|
||||
"org_id": str(org_id),
|
||||
"user_id": str(user_id),
|
||||
"thread_id": str(thread_id),
|
||||
"summary_id": str(summary_id),
|
||||
"type": "summary",
|
||||
"text": text
|
||||
}
|
||||
|
||||
if agent_id:
|
||||
payload["agent_id"] = str(agent_id)
|
||||
if workspace_id:
|
||||
payload["workspace_id"] = str(workspace_id)
|
||||
|
||||
self.client.upsert(
|
||||
collection_name=self.memories_collection,
|
||||
points=[
|
||||
qmodels.PointStruct(
|
||||
id=point_id,
|
||||
vector=vector,
|
||||
payload=payload
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
logger.info("summary_indexed", summary_id=str(summary_id), point_id=point_id)
|
||||
return point_id
|
||||
|
||||
async def search_memories(
|
||||
self,
|
||||
query: str,
|
||||
org_id: UUID,
|
||||
user_id: UUID,
|
||||
agent_id: Optional[UUID] = None,
|
||||
workspace_id: Optional[UUID] = None,
|
||||
categories: Optional[List[MemoryCategory]] = None,
|
||||
include_global: bool = True,
|
||||
top_k: int = 10
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Semantic search for memories.
|
||||
|
||||
Args:
|
||||
query: Search query
|
||||
org_id: Organization filter
|
||||
user_id: User filter
|
||||
agent_id: Agent filter (None = include global)
|
||||
workspace_id: Workspace filter
|
||||
categories: Filter by memory categories
|
||||
include_global: Include memories without agent_id
|
||||
top_k: Number of results
|
||||
|
||||
Returns:
|
||||
List of results with scores
|
||||
"""
|
||||
# Get query embedding
|
||||
query_vector = await get_query_embedding(query)
|
||||
if not query_vector:
|
||||
return []
|
||||
|
||||
# Build filter
|
||||
must_conditions = [
|
||||
qmodels.FieldCondition(
|
||||
key="org_id",
|
||||
match=qmodels.MatchValue(value=str(org_id))
|
||||
),
|
||||
qmodels.FieldCondition(
|
||||
key="user_id",
|
||||
match=qmodels.MatchValue(value=str(user_id))
|
||||
)
|
||||
]
|
||||
|
||||
if workspace_id:
|
||||
must_conditions.append(
|
||||
qmodels.FieldCondition(
|
||||
key="workspace_id",
|
||||
match=qmodels.MatchValue(value=str(workspace_id))
|
||||
)
|
||||
)
|
||||
|
||||
if categories:
|
||||
must_conditions.append(
|
||||
qmodels.FieldCondition(
|
||||
key="category",
|
||||
match=qmodels.MatchAny(any=[c.value for c in categories])
|
||||
)
|
||||
)
|
||||
|
||||
# Agent filter with global option
|
||||
if agent_id and not include_global:
|
||||
must_conditions.append(
|
||||
qmodels.FieldCondition(
|
||||
key="agent_id",
|
||||
match=qmodels.MatchValue(value=str(agent_id))
|
||||
)
|
||||
)
|
||||
elif agent_id and include_global:
|
||||
# Include both agent-specific and global (no agent_id)
|
||||
# This requires a should clause
|
||||
pass # Will handle in separate query if needed
|
||||
|
||||
search_filter = qmodels.Filter(must=must_conditions)
|
||||
|
||||
# Search
|
||||
results = self.client.search(
|
||||
collection_name=self.memories_collection,
|
||||
query_vector=query_vector,
|
||||
query_filter=search_filter,
|
||||
limit=top_k,
|
||||
with_payload=True
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"memory_search",
|
||||
query_preview=query[:50],
|
||||
results_count=len(results)
|
||||
)
|
||||
|
||||
return [
|
||||
{
|
||||
"point_id": str(r.id),
|
||||
"score": r.score,
|
||||
**r.payload
|
||||
}
|
||||
for r in results
|
||||
]
|
||||
|
||||
async def delete_memory(self, memory_id: UUID):
|
||||
"""Delete memory from index by memory_id"""
|
||||
self.client.delete(
|
||||
collection_name=self.memories_collection,
|
||||
points_selector=qmodels.FilterSelector(
|
||||
filter=qmodels.Filter(
|
||||
must=[
|
||||
qmodels.FieldCondition(
|
||||
key="memory_id",
|
||||
match=qmodels.MatchValue(value=str(memory_id))
|
||||
)
|
||||
]
|
||||
)
|
||||
)
|
||||
)
|
||||
logger.info("memory_deleted_from_index", memory_id=str(memory_id))
|
||||
|
||||
async def get_collection_stats(self) -> Dict[str, Any]:
|
||||
"""Get collection statistics"""
|
||||
memories_info = self.client.get_collection(self.memories_collection)
|
||||
|
||||
return {
|
||||
"memories": {
|
||||
"points_count": memories_info.points_count,
|
||||
"vectors_count": memories_info.vectors_count,
|
||||
"indexed_vectors_count": memories_info.indexed_vectors_count
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
# Global instance
|
||||
vector_store = VectorStore()
|
||||
@@ -1,6 +1,32 @@
|
||||
fastapi>=0.104.0
|
||||
uvicorn[standard]>=0.24.0
|
||||
sqlalchemy>=2.0.0
|
||||
psycopg2-binary>=2.9.0
|
||||
pydantic>=2.0.0
|
||||
python-dotenv>=1.0.0
|
||||
# DAARION Memory Service
|
||||
# Agent memory management with PostgreSQL + Qdrant + Cohere
|
||||
|
||||
# Web framework
|
||||
fastapi==0.109.0
|
||||
uvicorn[standard]==0.27.0
|
||||
pydantic==2.5.3
|
||||
pydantic-settings==2.1.0
|
||||
|
||||
# Database
|
||||
asyncpg==0.29.0
|
||||
sqlalchemy[asyncio]==2.0.25
|
||||
alembic==1.13.1
|
||||
|
||||
# Vector database
|
||||
qdrant-client==1.7.3
|
||||
|
||||
# Embeddings
|
||||
cohere==4.44
|
||||
|
||||
# Utilities
|
||||
python-dotenv==1.0.0
|
||||
httpx==0.26.0
|
||||
tenacity==8.2.3
|
||||
structlog==24.1.0
|
||||
|
||||
# Token counting
|
||||
tiktoken==0.5.2
|
||||
|
||||
# Testing
|
||||
pytest==7.4.4
|
||||
pytest-asyncio==0.23.3
|
||||
|
||||
Reference in New Issue
Block a user