90758facae
Features: - Three-tier memory architecture (short/mid/long-term) - PostgreSQL schema for conversations, events, memories - Qdrant vector database for semantic search - Cohere embeddings (embed-multilingual-v3.0, 1024 dims) - FastAPI Memory Service with full CRUD - External Secrets integration with Vault - Kubernetes deployment manifests Components: - infrastructure/database/agent-memory-schema.sql - infrastructure/kubernetes/apps/qdrant/ - infrastructure/kubernetes/apps/memory-service/ - services/memory-service/ (FastAPI app) Also includes: - External Secrets Operator - Traefik Ingress Controller - Cert-Manager with Let's Encrypt - ArgoCD for GitOps
459 lines
16 KiB
PL/PgSQL
459 lines
16 KiB
PL/PgSQL
-- ============================================================================
|
|
-- DAARION Agent Memory System - PostgreSQL Schema
|
|
-- Version: 1.0.0
|
|
-- Date: 2026-01-10
|
|
--
|
|
-- Трирівнева пам'ять агентів:
|
|
-- 1. Short-term: conversation_events (робочий буфер)
|
|
-- 2. Mid-term: thread_summaries (сесійна/тематична)
|
|
-- 3. Long-term: long_term_memory_items (персональна/проектна)
|
|
-- ============================================================================
|
|
|
|
-- Extensions
|
|
CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
|
|
CREATE EXTENSION IF NOT EXISTS "pgcrypto";
|
|
|
|
-- ============================================================================
|
|
-- CORE ENTITIES
|
|
-- ============================================================================
|
|
|
|
-- Organizations (top-level tenant)
|
|
CREATE TABLE IF NOT EXISTS organizations (
|
|
org_id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
|
name VARCHAR(255) NOT NULL,
|
|
settings JSONB DEFAULT '{}',
|
|
created_at TIMESTAMPTZ DEFAULT NOW(),
|
|
updated_at TIMESTAMPTZ DEFAULT NOW()
|
|
);
|
|
|
|
-- Workspaces (projects within org)
|
|
CREATE TABLE IF NOT EXISTS workspaces (
|
|
workspace_id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
|
org_id UUID NOT NULL REFERENCES organizations(org_id) ON DELETE CASCADE,
|
|
name VARCHAR(255) NOT NULL,
|
|
description TEXT,
|
|
settings JSONB DEFAULT '{}',
|
|
created_at TIMESTAMPTZ DEFAULT NOW(),
|
|
updated_at TIMESTAMPTZ DEFAULT NOW()
|
|
);
|
|
|
|
CREATE INDEX idx_workspaces_org ON workspaces(org_id);
|
|
|
|
-- Users
|
|
CREATE TABLE IF NOT EXISTS users (
|
|
user_id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
|
org_id UUID NOT NULL REFERENCES organizations(org_id) ON DELETE CASCADE,
|
|
external_id VARCHAR(255), -- for SSO/OAuth mapping
|
|
email VARCHAR(255),
|
|
display_name VARCHAR(255),
|
|
preferences JSONB DEFAULT '{}',
|
|
created_at TIMESTAMPTZ DEFAULT NOW(),
|
|
updated_at TIMESTAMPTZ DEFAULT NOW(),
|
|
UNIQUE(org_id, external_id)
|
|
);
|
|
|
|
CREATE INDEX idx_users_org ON users(org_id);
|
|
CREATE INDEX idx_users_email ON users(email);
|
|
|
|
-- Agents
|
|
CREATE TABLE IF NOT EXISTS agents (
|
|
agent_id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
|
org_id UUID NOT NULL REFERENCES organizations(org_id) ON DELETE CASCADE,
|
|
name VARCHAR(255) NOT NULL,
|
|
type VARCHAR(50) NOT NULL, -- 'assistant', 'specialist', 'coordinator'
|
|
model VARCHAR(100), -- 'claude-3-opus', 'gpt-4', etc.
|
|
system_prompt TEXT,
|
|
capabilities JSONB DEFAULT '[]', -- ['code', 'search', 'memory', 'tools']
|
|
settings JSONB DEFAULT '{}',
|
|
is_active BOOLEAN DEFAULT true,
|
|
created_at TIMESTAMPTZ DEFAULT NOW(),
|
|
updated_at TIMESTAMPTZ DEFAULT NOW()
|
|
);
|
|
|
|
CREATE INDEX idx_agents_org ON agents(org_id);
|
|
CREATE INDEX idx_agents_type ON agents(type);
|
|
|
|
-- ============================================================================
|
|
-- CONVERSATION LAYER (Short-term Memory)
|
|
-- ============================================================================
|
|
|
|
-- Conversation threads
|
|
CREATE TABLE IF NOT EXISTS conversation_threads (
|
|
thread_id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
|
org_id UUID NOT NULL REFERENCES organizations(org_id) ON DELETE CASCADE,
|
|
workspace_id UUID REFERENCES workspaces(workspace_id) ON DELETE SET NULL,
|
|
user_id UUID NOT NULL REFERENCES users(user_id) ON DELETE CASCADE,
|
|
agent_id UUID REFERENCES agents(agent_id) ON DELETE SET NULL,
|
|
|
|
title VARCHAR(500),
|
|
status VARCHAR(50) DEFAULT 'active', -- 'active', 'archived', 'completed'
|
|
|
|
-- Metadata
|
|
tags JSONB DEFAULT '[]',
|
|
metadata JSONB DEFAULT '{}',
|
|
|
|
-- Stats
|
|
message_count INTEGER DEFAULT 0,
|
|
total_tokens INTEGER DEFAULT 0,
|
|
|
|
-- Timestamps
|
|
created_at TIMESTAMPTZ DEFAULT NOW(),
|
|
last_activity_at TIMESTAMPTZ DEFAULT NOW(),
|
|
archived_at TIMESTAMPTZ
|
|
);
|
|
|
|
CREATE INDEX idx_threads_org ON conversation_threads(org_id);
|
|
CREATE INDEX idx_threads_workspace ON conversation_threads(workspace_id);
|
|
CREATE INDEX idx_threads_user ON conversation_threads(user_id);
|
|
CREATE INDEX idx_threads_agent ON conversation_threads(agent_id);
|
|
CREATE INDEX idx_threads_status ON conversation_threads(status);
|
|
CREATE INDEX idx_threads_last_activity ON conversation_threads(last_activity_at DESC);
|
|
|
|
-- Conversation events (Event Log - source of truth)
|
|
CREATE TABLE IF NOT EXISTS conversation_events (
|
|
event_id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
|
thread_id UUID NOT NULL REFERENCES conversation_threads(thread_id) ON DELETE CASCADE,
|
|
|
|
-- Event type
|
|
event_type VARCHAR(50) NOT NULL, -- 'message', 'tool_call', 'tool_result', 'decision', 'summary', 'memory_write', 'memory_retract', 'error'
|
|
|
|
-- For messages
|
|
role VARCHAR(20), -- 'user', 'assistant', 'system', 'tool'
|
|
content TEXT,
|
|
|
|
-- For tool calls
|
|
tool_name VARCHAR(100),
|
|
tool_input JSONB,
|
|
tool_output JSONB,
|
|
|
|
-- Structured payload (flexible)
|
|
payload JSONB DEFAULT '{}',
|
|
|
|
-- Token tracking
|
|
token_count INTEGER,
|
|
|
|
-- Metadata
|
|
model_used VARCHAR(100),
|
|
latency_ms INTEGER,
|
|
metadata JSONB DEFAULT '{}',
|
|
|
|
-- Ordering
|
|
sequence_num SERIAL,
|
|
created_at TIMESTAMPTZ DEFAULT NOW()
|
|
);
|
|
|
|
CREATE INDEX idx_events_thread ON conversation_events(thread_id);
|
|
CREATE INDEX idx_events_type ON conversation_events(event_type);
|
|
CREATE INDEX idx_events_created ON conversation_events(created_at DESC);
|
|
CREATE INDEX idx_events_thread_seq ON conversation_events(thread_id, sequence_num);
|
|
|
|
-- ============================================================================
|
|
-- SUMMARY LAYER (Mid-term Memory)
|
|
-- ============================================================================
|
|
|
|
-- Thread summaries (rolling compression)
|
|
CREATE TABLE IF NOT EXISTS thread_summaries (
|
|
summary_id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
|
thread_id UUID NOT NULL REFERENCES conversation_threads(thread_id) ON DELETE CASCADE,
|
|
|
|
-- Version tracking
|
|
version INTEGER NOT NULL DEFAULT 1,
|
|
|
|
-- Summary content
|
|
summary_text TEXT NOT NULL,
|
|
|
|
-- Structured state
|
|
state JSONB DEFAULT '{}', -- {goals: [], decisions: [], open_questions: [], next_steps: [], key_facts: []}
|
|
|
|
-- Coverage
|
|
events_from_seq INTEGER, -- first event sequence included
|
|
events_to_seq INTEGER, -- last event sequence included
|
|
events_count INTEGER,
|
|
|
|
-- Token info
|
|
original_tokens INTEGER, -- tokens before compression
|
|
summary_tokens INTEGER, -- tokens after compression
|
|
compression_ratio FLOAT,
|
|
|
|
-- Timestamps
|
|
created_at TIMESTAMPTZ DEFAULT NOW(),
|
|
|
|
UNIQUE(thread_id, version)
|
|
);
|
|
|
|
CREATE INDEX idx_summaries_thread ON thread_summaries(thread_id);
|
|
CREATE INDEX idx_summaries_thread_version ON thread_summaries(thread_id, version DESC);
|
|
|
|
-- ============================================================================
|
|
-- LONG-TERM MEMORY LAYER
|
|
-- ============================================================================
|
|
|
|
-- Memory categories enum
|
|
CREATE TYPE memory_category AS ENUM (
|
|
'preference', -- user likes/dislikes
|
|
'identity', -- who the user is
|
|
'constraint', -- limitations, rules
|
|
'project_fact', -- project-specific knowledge
|
|
'relationship', -- connections between entities
|
|
'skill', -- user capabilities
|
|
'goal', -- user objectives
|
|
'context', -- situational info
|
|
'feedback' -- user corrections/confirmations
|
|
);
|
|
|
|
-- Retention policy enum
|
|
CREATE TYPE retention_policy AS ENUM (
|
|
'permanent', -- keep until explicitly deleted
|
|
'session', -- delete after session
|
|
'ttl_days', -- delete after N days
|
|
'until_revoked' -- keep until user revokes
|
|
);
|
|
|
|
-- Long-term memory items
|
|
CREATE TABLE IF NOT EXISTS long_term_memory_items (
|
|
memory_id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
|
|
|
-- Scope (all nullable for flexibility)
|
|
org_id UUID REFERENCES organizations(org_id) ON DELETE CASCADE,
|
|
workspace_id UUID REFERENCES workspaces(workspace_id) ON DELETE SET NULL,
|
|
user_id UUID REFERENCES users(user_id) ON DELETE CASCADE,
|
|
agent_id UUID REFERENCES agents(agent_id) ON DELETE SET NULL, -- null = global for user
|
|
|
|
-- Content
|
|
category memory_category NOT NULL,
|
|
fact_text TEXT NOT NULL, -- atomic statement
|
|
fact_embedding_id VARCHAR(100), -- reference to Qdrant point ID
|
|
|
|
-- Confidence & validation
|
|
confidence FLOAT DEFAULT 0.8 CHECK (confidence >= 0 AND confidence <= 1),
|
|
is_verified BOOLEAN DEFAULT false,
|
|
verification_count INTEGER DEFAULT 0,
|
|
|
|
-- Source tracking
|
|
source_event_id UUID REFERENCES conversation_events(event_id) ON DELETE SET NULL,
|
|
source_thread_id UUID REFERENCES conversation_threads(thread_id) ON DELETE SET NULL,
|
|
extraction_method VARCHAR(50), -- 'explicit', 'inferred', 'confirmed', 'imported'
|
|
|
|
-- Lifecycle
|
|
valid_from TIMESTAMPTZ DEFAULT NOW(),
|
|
valid_to TIMESTAMPTZ, -- null = currently valid
|
|
last_confirmed_at TIMESTAMPTZ,
|
|
last_used_at TIMESTAMPTZ,
|
|
use_count INTEGER DEFAULT 0,
|
|
|
|
-- Privacy & retention
|
|
is_sensitive BOOLEAN DEFAULT false,
|
|
retention retention_policy DEFAULT 'until_revoked',
|
|
ttl_days INTEGER, -- if retention = 'ttl_days'
|
|
|
|
-- Metadata
|
|
tags JSONB DEFAULT '[]',
|
|
metadata JSONB DEFAULT '{}',
|
|
|
|
-- Timestamps
|
|
created_at TIMESTAMPTZ DEFAULT NOW(),
|
|
updated_at TIMESTAMPTZ DEFAULT NOW()
|
|
);
|
|
|
|
-- Indexes for memory retrieval
|
|
CREATE INDEX idx_memory_org ON long_term_memory_items(org_id);
|
|
CREATE INDEX idx_memory_workspace ON long_term_memory_items(workspace_id);
|
|
CREATE INDEX idx_memory_user ON long_term_memory_items(user_id);
|
|
CREATE INDEX idx_memory_agent ON long_term_memory_items(agent_id);
|
|
CREATE INDEX idx_memory_category ON long_term_memory_items(category);
|
|
CREATE INDEX idx_memory_user_agent ON long_term_memory_items(user_id, agent_id);
|
|
CREATE INDEX idx_memory_valid ON long_term_memory_items(valid_from, valid_to);
|
|
CREATE INDEX idx_memory_confidence ON long_term_memory_items(confidence DESC);
|
|
CREATE INDEX idx_memory_created ON long_term_memory_items(created_at DESC);
|
|
|
|
-- GIN index for tags search
|
|
CREATE INDEX idx_memory_tags ON long_term_memory_items USING GIN (tags);
|
|
|
|
-- Memory feedback (user corrections)
|
|
CREATE TABLE IF NOT EXISTS memory_feedback (
|
|
feedback_id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
|
memory_id UUID NOT NULL REFERENCES long_term_memory_items(memory_id) ON DELETE CASCADE,
|
|
user_id UUID NOT NULL REFERENCES users(user_id) ON DELETE CASCADE,
|
|
|
|
action VARCHAR(20) NOT NULL, -- 'confirm', 'reject', 'edit', 'delete'
|
|
old_value TEXT,
|
|
new_value TEXT,
|
|
reason TEXT,
|
|
|
|
created_at TIMESTAMPTZ DEFAULT NOW()
|
|
);
|
|
|
|
CREATE INDEX idx_feedback_memory ON memory_feedback(memory_id);
|
|
CREATE INDEX idx_feedback_user ON memory_feedback(user_id);
|
|
|
|
-- ============================================================================
|
|
-- HELPER VIEWS
|
|
-- ============================================================================
|
|
|
|
-- Active memories for a user (across all agents)
|
|
CREATE OR REPLACE VIEW v_active_user_memories AS
|
|
SELECT
|
|
m.*,
|
|
a.name as agent_name
|
|
FROM long_term_memory_items m
|
|
LEFT JOIN agents a ON m.agent_id = a.agent_id
|
|
WHERE m.valid_to IS NULL
|
|
AND m.confidence >= 0.5
|
|
ORDER BY m.confidence DESC, m.last_used_at DESC NULLS LAST;
|
|
|
|
-- Recent conversations with summaries
|
|
CREATE OR REPLACE VIEW v_recent_conversations AS
|
|
SELECT
|
|
t.thread_id,
|
|
t.title,
|
|
t.user_id,
|
|
t.agent_id,
|
|
t.message_count,
|
|
t.last_activity_at,
|
|
s.summary_text,
|
|
s.state
|
|
FROM conversation_threads t
|
|
LEFT JOIN LATERAL (
|
|
SELECT summary_text, state
|
|
FROM thread_summaries
|
|
WHERE thread_id = t.thread_id
|
|
ORDER BY version DESC
|
|
LIMIT 1
|
|
) s ON true
|
|
WHERE t.status = 'active'
|
|
ORDER BY t.last_activity_at DESC;
|
|
|
|
-- ============================================================================
|
|
-- FUNCTIONS
|
|
-- ============================================================================
|
|
|
|
-- Update thread stats after new event
|
|
CREATE OR REPLACE FUNCTION update_thread_stats()
|
|
RETURNS TRIGGER AS $$
|
|
BEGIN
|
|
UPDATE conversation_threads
|
|
SET
|
|
message_count = message_count + CASE WHEN NEW.event_type = 'message' THEN 1 ELSE 0 END,
|
|
total_tokens = total_tokens + COALESCE(NEW.token_count, 0),
|
|
last_activity_at = NOW()
|
|
WHERE thread_id = NEW.thread_id;
|
|
RETURN NEW;
|
|
END;
|
|
$$ LANGUAGE plpgsql;
|
|
|
|
CREATE TRIGGER trg_update_thread_stats
|
|
AFTER INSERT ON conversation_events
|
|
FOR EACH ROW EXECUTE FUNCTION update_thread_stats();
|
|
|
|
-- Update memory usage stats
|
|
CREATE OR REPLACE FUNCTION update_memory_usage()
|
|
RETURNS TRIGGER AS $$
|
|
BEGIN
|
|
UPDATE long_term_memory_items
|
|
SET
|
|
use_count = use_count + 1,
|
|
last_used_at = NOW()
|
|
WHERE memory_id = NEW.memory_id;
|
|
RETURN NEW;
|
|
END;
|
|
$$ LANGUAGE plpgsql;
|
|
|
|
-- Auto-update updated_at
|
|
CREATE OR REPLACE FUNCTION update_updated_at()
|
|
RETURNS TRIGGER AS $$
|
|
BEGIN
|
|
NEW.updated_at = NOW();
|
|
RETURN NEW;
|
|
END;
|
|
$$ LANGUAGE plpgsql;
|
|
|
|
CREATE TRIGGER trg_organizations_updated
|
|
BEFORE UPDATE ON organizations
|
|
FOR EACH ROW EXECUTE FUNCTION update_updated_at();
|
|
|
|
CREATE TRIGGER trg_workspaces_updated
|
|
BEFORE UPDATE ON workspaces
|
|
FOR EACH ROW EXECUTE FUNCTION update_updated_at();
|
|
|
|
CREATE TRIGGER trg_users_updated
|
|
BEFORE UPDATE ON users
|
|
FOR EACH ROW EXECUTE FUNCTION update_updated_at();
|
|
|
|
CREATE TRIGGER trg_agents_updated
|
|
BEFORE UPDATE ON agents
|
|
FOR EACH ROW EXECUTE FUNCTION update_updated_at();
|
|
|
|
CREATE TRIGGER trg_memory_updated
|
|
BEFORE UPDATE ON long_term_memory_items
|
|
FOR EACH ROW EXECUTE FUNCTION update_updated_at();
|
|
|
|
-- ============================================================================
|
|
-- INITIAL DATA
|
|
-- ============================================================================
|
|
|
|
-- Default organization
|
|
INSERT INTO organizations (org_id, name, settings)
|
|
VALUES (
|
|
'a0000000-0000-0000-0000-000000000001',
|
|
'DAARION',
|
|
'{"tier": "enterprise", "features": ["memory", "multi-agent", "knowledge-base"]}'
|
|
) ON CONFLICT DO NOTHING;
|
|
|
|
-- Default workspace
|
|
INSERT INTO workspaces (workspace_id, org_id, name, description)
|
|
VALUES (
|
|
'b0000000-0000-0000-0000-000000000001',
|
|
'a0000000-0000-0000-0000-000000000001',
|
|
'MicroDAO',
|
|
'Main development workspace for DAARION project'
|
|
) ON CONFLICT DO NOTHING;
|
|
|
|
-- Default user (Ivan)
|
|
INSERT INTO users (user_id, org_id, external_id, display_name, preferences)
|
|
VALUES (
|
|
'c0000000-0000-0000-0000-000000000001',
|
|
'a0000000-0000-0000-0000-000000000001',
|
|
'ivan',
|
|
'Ivan Tytar',
|
|
'{"language": "uk", "timezone": "Europe/Kyiv"}'
|
|
) ON CONFLICT DO NOTHING;
|
|
|
|
-- Default agents
|
|
INSERT INTO agents (agent_id, org_id, name, type, model, capabilities) VALUES
|
|
(
|
|
'd0000000-0000-0000-0000-000000000001',
|
|
'a0000000-0000-0000-0000-000000000001',
|
|
'Claude Assistant',
|
|
'assistant',
|
|
'claude-3-opus',
|
|
'["code", "memory", "tools", "search"]'
|
|
),
|
|
(
|
|
'd0000000-0000-0000-0000-000000000002',
|
|
'a0000000-0000-0000-0000-000000000001',
|
|
'Code Specialist',
|
|
'specialist',
|
|
'claude-3-opus',
|
|
'["code", "review", "refactor"]'
|
|
),
|
|
(
|
|
'd0000000-0000-0000-0000-000000000003',
|
|
'a0000000-0000-0000-0000-000000000001',
|
|
'DevOps Agent',
|
|
'specialist',
|
|
'claude-3-opus',
|
|
'["infrastructure", "deployment", "monitoring"]'
|
|
)
|
|
ON CONFLICT DO NOTHING;
|
|
|
|
-- ============================================================================
|
|
-- COMMENTS
|
|
-- ============================================================================
|
|
|
|
COMMENT ON TABLE conversation_events IS 'Event log for all conversation activities - source of truth';
|
|
COMMENT ON TABLE thread_summaries IS 'Rolling summaries for context compression (mid-term memory)';
|
|
COMMENT ON TABLE long_term_memory_items IS 'Persistent facts about users/projects (long-term memory)';
|
|
COMMENT ON COLUMN long_term_memory_items.fact_text IS 'Atomic statement - one fact per row';
|
|
COMMENT ON COLUMN long_term_memory_items.confidence IS 'Confidence score 0-1, increases with confirmations';
|
|
COMMENT ON COLUMN long_term_memory_items.fact_embedding_id IS 'Reference to Qdrant vector point ID';
|