🧠 Add Agent Memory System with PostgreSQL + Qdrant + Cohere
Features: - Three-tier memory architecture (short/mid/long-term) - PostgreSQL schema for conversations, events, memories - Qdrant vector database for semantic search - Cohere embeddings (embed-multilingual-v3.0, 1024 dims) - FastAPI Memory Service with full CRUD - External Secrets integration with Vault - Kubernetes deployment manifests Components: - infrastructure/database/agent-memory-schema.sql - infrastructure/kubernetes/apps/qdrant/ - infrastructure/kubernetes/apps/memory-service/ - services/memory-service/ (FastAPI app) Also includes: - External Secrets Operator - Traefik Ingress Controller - Cert-Manager with Let's Encrypt - ArgoCD for GitOps
This commit is contained in:
Apple
@@ -1,178 +1,249 @@
|
||||
"""
|
||||
SQLAlchemy моделі для Memory Service
|
||||
Підтримує: user_facts, dialog_summaries, agent_memory_events
|
||||
DAARION Memory Service - Pydantic Models
|
||||
"""
|
||||
|
||||
from sqlalchemy import (
|
||||
Column, String, Text, JSON, TIMESTAMP,
|
||||
CheckConstraint, Index, Boolean, Integer
|
||||
)
|
||||
from sqlalchemy.ext.declarative import declarative_base
|
||||
from sqlalchemy.sql import func
|
||||
import os
|
||||
|
||||
# Перевірка типу бази даних
|
||||
DATABASE_URL = os.getenv("DATABASE_URL", "sqlite:///./memory.db")
|
||||
IS_SQLITE = "sqlite" in DATABASE_URL.lower()
|
||||
|
||||
if IS_SQLITE:
|
||||
# Для SQLite використовуємо стандартні типи
|
||||
from sqlalchemy import JSON as JSONB_TYPE
|
||||
UUID_TYPE = String # SQLite не має UUID, використовуємо String
|
||||
else:
|
||||
# Для PostgreSQL використовуємо специфічні типи
|
||||
from sqlalchemy.dialects.postgresql import UUID, JSONB
|
||||
UUID_TYPE = UUID
|
||||
JSONB_TYPE = JSONB
|
||||
|
||||
try:
|
||||
from pgvector.sqlalchemy import Vector
|
||||
HAS_PGVECTOR = True
|
||||
except ImportError:
|
||||
HAS_PGVECTOR = False
|
||||
# Заглушка для SQLite
|
||||
class Vector:
|
||||
def __init__(self, *args, **kwargs):
|
||||
pass
|
||||
|
||||
Base = declarative_base()
|
||||
from datetime import datetime
|
||||
from typing import Optional, List, Any
|
||||
from uuid import UUID
|
||||
from enum import Enum
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class UserFact(Base):
|
||||
"""
|
||||
Довгострокові факти про користувача
|
||||
Використовується для контрольованої довгострокової пам'яті
|
||||
(мови, вподобання, тип користувача, токен-статуси)
|
||||
"""
|
||||
__tablename__ = "user_facts"
|
||||
# ============================================================================
|
||||
# ENUMS
|
||||
# ============================================================================
|
||||
|
||||
id = Column(UUID_TYPE(as_uuid=False) if not IS_SQLITE else String, primary_key=True, server_default=func.gen_random_uuid() if not IS_SQLITE else None)
|
||||
user_id = Column(String, nullable=False, index=True) # Без FK constraint для тестування
|
||||
team_id = Column(String, nullable=True, index=True) # Без FK constraint, оскільки teams може не існувати
|
||||
|
||||
# Ключ факту (наприклад: "language", "is_donor", "is_validator", "top_contributor")
|
||||
fact_key = Column(String, nullable=False, index=True)
|
||||
|
||||
# Значення факту (може бути текст, число, boolean, JSON)
|
||||
fact_value = Column(Text, nullable=True)
|
||||
fact_value_json = Column(JSONB_TYPE, nullable=True)
|
||||
|
||||
# Метадані: джерело, впевненість, термін дії
|
||||
meta = Column(JSONB_TYPE, nullable=False, server_default="{}")
|
||||
|
||||
# Токен-гейт: чи залежить факт від токенів/активності
|
||||
token_gated = Column(Boolean, nullable=False, server_default="false")
|
||||
token_requirements = Column(JSONB_TYPE, nullable=True) # {"token": "DAAR", "min_balance": 1}
|
||||
|
||||
created_at = Column(TIMESTAMP(timezone=True) if not IS_SQLITE else TIMESTAMP, nullable=False, server_default=func.now())
|
||||
updated_at = Column(TIMESTAMP(timezone=True) if not IS_SQLITE else TIMESTAMP, nullable=True, onupdate=func.now())
|
||||
expires_at = Column(TIMESTAMP(timezone=True) if not IS_SQLITE else TIMESTAMP, nullable=True) # Для тимчасових фактів
|
||||
|
||||
__table_args__ = (
|
||||
Index("idx_user_facts_user_key", "user_id", "fact_key"),
|
||||
Index("idx_user_facts_team", "team_id"),
|
||||
Index("idx_user_facts_token_gated", "token_gated"),
|
||||
)
|
||||
class EventType(str, Enum):
|
||||
MESSAGE = "message"
|
||||
TOOL_CALL = "tool_call"
|
||||
TOOL_RESULT = "tool_result"
|
||||
DECISION = "decision"
|
||||
SUMMARY = "summary"
|
||||
MEMORY_WRITE = "memory_write"
|
||||
MEMORY_RETRACT = "memory_retract"
|
||||
ERROR = "error"
|
||||
|
||||
|
||||
class DialogSummary(Base):
|
||||
"""
|
||||
Підсумки діалогів для масштабування без переповнення контексту
|
||||
Зберігає агреговану інформацію про сесії/діалоги
|
||||
"""
|
||||
__tablename__ = "dialog_summaries"
|
||||
|
||||
id = Column(UUID_TYPE(as_uuid=False) if not IS_SQLITE else String, primary_key=True, server_default=func.gen_random_uuid() if not IS_SQLITE else None)
|
||||
|
||||
# Контекст діалогу (без FK constraints для тестування)
|
||||
team_id = Column(String, nullable=False, index=True)
|
||||
channel_id = Column(String, nullable=True, index=True)
|
||||
agent_id = Column(String, nullable=True, index=True)
|
||||
user_id = Column(String, nullable=True, index=True)
|
||||
|
||||
# Період, який охоплює підсумок
|
||||
period_start = Column(TIMESTAMP(timezone=True) if not IS_SQLITE else TIMESTAMP, nullable=False)
|
||||
period_end = Column(TIMESTAMP(timezone=True) if not IS_SQLITE else TIMESTAMP, nullable=False)
|
||||
|
||||
# Підсумок
|
||||
summary_text = Column(Text, nullable=False)
|
||||
summary_json = Column(JSONB_TYPE, nullable=True) # Структуровані дані
|
||||
|
||||
# Статистика
|
||||
message_count = Column(Integer, nullable=False, server_default="0")
|
||||
participant_count = Column(Integer, nullable=False, server_default="0")
|
||||
|
||||
# Ключові теми/теги
|
||||
topics = Column(JSONB_TYPE, nullable=True) # ["project-planning", "bug-fix", ...]
|
||||
|
||||
# Метадані
|
||||
meta = Column(JSONB_TYPE, nullable=False, server_default="{}")
|
||||
|
||||
created_at = Column(TIMESTAMP(timezone=True) if not IS_SQLITE else TIMESTAMP, nullable=False, server_default=func.now())
|
||||
|
||||
__table_args__ = (
|
||||
Index("idx_dialog_summaries_team_period", "team_id", "period_start", "period_end"),
|
||||
Index("idx_dialog_summaries_channel", "channel_id"),
|
||||
Index("idx_dialog_summaries_agent", "agent_id"),
|
||||
)
|
||||
class MessageRole(str, Enum):
|
||||
USER = "user"
|
||||
ASSISTANT = "assistant"
|
||||
SYSTEM = "system"
|
||||
TOOL = "tool"
|
||||
|
||||
|
||||
class AgentMemoryEvent(Base):
|
||||
"""
|
||||
Події пам'яті агентів (short-term, mid-term, long-term)
|
||||
Базується на документації: docs/cursor/13_agent_memory_system.md
|
||||
"""
|
||||
__tablename__ = "agent_memory_events"
|
||||
|
||||
id = Column(UUID_TYPE(as_uuid=False) if not IS_SQLITE else String, primary_key=True, server_default=func.gen_random_uuid() if not IS_SQLITE else None)
|
||||
|
||||
# Без FK constraints для тестування
|
||||
agent_id = Column(String, nullable=False, index=True)
|
||||
team_id = Column(String, nullable=False, index=True)
|
||||
channel_id = Column(String, nullable=True, index=True)
|
||||
user_id = Column(String, nullable=True, index=True)
|
||||
|
||||
# Scope: short_term, mid_term, long_term
|
||||
scope = Column(String, nullable=False)
|
||||
|
||||
# Kind: message, fact, summary, note
|
||||
kind = Column(String, nullable=False)
|
||||
|
||||
# Тіло події
|
||||
body_text = Column(Text, nullable=True)
|
||||
body_json = Column(JSONB_TYPE, nullable=True)
|
||||
|
||||
created_at = Column(TIMESTAMP(timezone=True) if not IS_SQLITE else TIMESTAMP, nullable=False, server_default=func.now())
|
||||
|
||||
__table_args__ = (
|
||||
CheckConstraint("scope IN ('short_term', 'mid_term', 'long_term')", name="ck_agent_memory_scope"),
|
||||
CheckConstraint("kind IN ('message', 'fact', 'summary', 'note')", name="ck_agent_memory_kind"),
|
||||
Index("idx_agent_memory_events_agent_team_scope", "agent_id", "team_id", "scope"),
|
||||
Index("idx_agent_memory_events_channel", "agent_id", "channel_id"),
|
||||
Index("idx_agent_memory_events_created_at", "created_at"),
|
||||
)
|
||||
class MemoryCategory(str, Enum):
|
||||
PREFERENCE = "preference"
|
||||
IDENTITY = "identity"
|
||||
CONSTRAINT = "constraint"
|
||||
PROJECT_FACT = "project_fact"
|
||||
RELATIONSHIP = "relationship"
|
||||
SKILL = "skill"
|
||||
GOAL = "goal"
|
||||
CONTEXT = "context"
|
||||
FEEDBACK = "feedback"
|
||||
|
||||
|
||||
class AgentMemoryFactsVector(Base):
|
||||
"""
|
||||
Векторні представлення фактів для RAG (Retrieval-Augmented Generation)
|
||||
"""
|
||||
__tablename__ = "agent_memory_facts_vector"
|
||||
class RetentionPolicy(str, Enum):
|
||||
PERMANENT = "permanent"
|
||||
SESSION = "session"
|
||||
TTL_DAYS = "ttl_days"
|
||||
UNTIL_REVOKED = "until_revoked"
|
||||
|
||||
id = Column(UUID_TYPE(as_uuid=False) if not IS_SQLITE else String, primary_key=True, server_default=func.gen_random_uuid() if not IS_SQLITE else None)
|
||||
|
||||
# Без FK constraints для тестування
|
||||
agent_id = Column(String, nullable=False, index=True)
|
||||
team_id = Column(String, nullable=False, index=True)
|
||||
|
||||
fact_text = Column(Text, nullable=False)
|
||||
embedding = Column(Vector(1536), nullable=True) if HAS_PGVECTOR else Column(Text, nullable=True) # OpenAI ada-002 embedding size
|
||||
|
||||
meta = Column(JSONB_TYPE, nullable=False, server_default="{}")
|
||||
|
||||
created_at = Column(TIMESTAMP(timezone=True) if not IS_SQLITE else TIMESTAMP, nullable=False, server_default=func.now())
|
||||
|
||||
__table_args__ = (
|
||||
Index("idx_agent_memory_facts_vector_agent_team", "agent_id", "team_id"),
|
||||
)
|
||||
class FeedbackAction(str, Enum):
|
||||
CONFIRM = "confirm"
|
||||
REJECT = "reject"
|
||||
EDIT = "edit"
|
||||
DELETE = "delete"
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# REQUEST MODELS
|
||||
# ============================================================================
|
||||
|
||||
class CreateThreadRequest(BaseModel):
|
||||
"""Create new conversation thread"""
|
||||
org_id: UUID
|
||||
workspace_id: Optional[UUID] = None
|
||||
user_id: UUID
|
||||
agent_id: Optional[UUID] = None
|
||||
title: Optional[str] = None
|
||||
tags: List[str] = []
|
||||
metadata: dict = {}
|
||||
|
||||
|
||||
class AddEventRequest(BaseModel):
|
||||
"""Add event to conversation"""
|
||||
thread_id: UUID
|
||||
event_type: EventType
|
||||
role: Optional[MessageRole] = None
|
||||
content: Optional[str] = None
|
||||
tool_name: Optional[str] = None
|
||||
tool_input: Optional[dict] = None
|
||||
tool_output: Optional[dict] = None
|
||||
payload: dict = {}
|
||||
token_count: Optional[int] = None
|
||||
model_used: Optional[str] = None
|
||||
latency_ms: Optional[int] = None
|
||||
metadata: dict = {}
|
||||
|
||||
|
||||
class CreateMemoryRequest(BaseModel):
|
||||
"""Create long-term memory item"""
|
||||
org_id: UUID
|
||||
workspace_id: Optional[UUID] = None
|
||||
user_id: UUID
|
||||
agent_id: Optional[UUID] = None # null = global
|
||||
category: MemoryCategory
|
||||
fact_text: str
|
||||
confidence: float = Field(default=0.8, ge=0, le=1)
|
||||
source_event_id: Optional[UUID] = None
|
||||
source_thread_id: Optional[UUID] = None
|
||||
extraction_method: str = "explicit"
|
||||
is_sensitive: bool = False
|
||||
retention: RetentionPolicy = RetentionPolicy.UNTIL_REVOKED
|
||||
ttl_days: Optional[int] = None
|
||||
tags: List[str] = []
|
||||
metadata: dict = {}
|
||||
|
||||
|
||||
class MemoryFeedbackRequest(BaseModel):
|
||||
"""User feedback on memory"""
|
||||
memory_id: UUID
|
||||
user_id: UUID
|
||||
action: FeedbackAction
|
||||
new_value: Optional[str] = None
|
||||
reason: Optional[str] = None
|
||||
|
||||
|
||||
class RetrievalRequest(BaseModel):
|
||||
"""Semantic retrieval request"""
|
||||
org_id: UUID
|
||||
user_id: UUID
|
||||
agent_id: Optional[UUID] = None
|
||||
workspace_id: Optional[UUID] = None
|
||||
queries: List[str]
|
||||
top_k: int = 10
|
||||
min_confidence: float = 0.5
|
||||
include_global: bool = True
|
||||
categories: Optional[List[MemoryCategory]] = None
|
||||
|
||||
|
||||
class SummaryRequest(BaseModel):
|
||||
"""Generate summary for thread"""
|
||||
thread_id: UUID
|
||||
force: bool = False # force even if under token threshold
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# RESPONSE MODELS
|
||||
# ============================================================================
|
||||
|
||||
class ThreadResponse(BaseModel):
|
||||
thread_id: UUID
|
||||
org_id: UUID
|
||||
workspace_id: Optional[UUID]
|
||||
user_id: UUID
|
||||
agent_id: Optional[UUID]
|
||||
title: Optional[str]
|
||||
status: str
|
||||
message_count: int
|
||||
total_tokens: int
|
||||
created_at: datetime
|
||||
last_activity_at: datetime
|
||||
|
||||
|
||||
class EventResponse(BaseModel):
|
||||
event_id: UUID
|
||||
thread_id: UUID
|
||||
event_type: EventType
|
||||
role: Optional[MessageRole]
|
||||
content: Optional[str]
|
||||
tool_name: Optional[str]
|
||||
tool_input: Optional[dict]
|
||||
tool_output: Optional[dict]
|
||||
payload: dict
|
||||
token_count: Optional[int]
|
||||
sequence_num: int
|
||||
created_at: datetime
|
||||
|
||||
|
||||
class MemoryResponse(BaseModel):
|
||||
memory_id: UUID
|
||||
org_id: UUID
|
||||
workspace_id: Optional[UUID]
|
||||
user_id: UUID
|
||||
agent_id: Optional[UUID]
|
||||
category: MemoryCategory
|
||||
fact_text: str
|
||||
confidence: float
|
||||
is_verified: bool
|
||||
is_sensitive: bool
|
||||
retention: RetentionPolicy
|
||||
valid_from: datetime
|
||||
valid_to: Optional[datetime]
|
||||
last_used_at: Optional[datetime]
|
||||
use_count: int
|
||||
created_at: datetime
|
||||
|
||||
|
||||
class SummaryResponse(BaseModel):
|
||||
summary_id: UUID
|
||||
thread_id: UUID
|
||||
version: int
|
||||
summary_text: str
|
||||
state: dict
|
||||
events_count: int
|
||||
compression_ratio: Optional[float]
|
||||
created_at: datetime
|
||||
|
||||
|
||||
class RetrievalResult(BaseModel):
|
||||
"""Single retrieval result"""
|
||||
memory_id: UUID
|
||||
fact_text: str
|
||||
category: MemoryCategory
|
||||
confidence: float
|
||||
relevance_score: float
|
||||
agent_id: Optional[UUID]
|
||||
is_global: bool
|
||||
|
||||
|
||||
class RetrievalResponse(BaseModel):
|
||||
"""Retrieval response with results"""
|
||||
results: List[RetrievalResult]
|
||||
query_count: int
|
||||
total_results: int
|
||||
|
||||
|
||||
class ContextResponse(BaseModel):
|
||||
"""Full context for agent prompt"""
|
||||
thread_id: UUID
|
||||
summary: Optional[SummaryResponse]
|
||||
recent_messages: List[EventResponse]
|
||||
retrieved_memories: List[RetrievalResult]
|
||||
token_estimate: int
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# INTERNAL MODELS
|
||||
# ============================================================================
|
||||
|
||||
class EmbeddingRequest(BaseModel):
|
||||
"""Internal embedding request"""
|
||||
texts: List[str]
|
||||
input_type: str = "search_document" # or "search_query"
|
||||
|
||||
|
||||
class QdrantPayload(BaseModel):
|
||||
"""Qdrant point payload"""
|
||||
org_id: str
|
||||
workspace_id: Optional[str]
|
||||
user_id: str
|
||||
agent_id: Optional[str]
|
||||
thread_id: Optional[str]
|
||||
memory_id: Optional[str]
|
||||
event_id: Optional[str]
|
||||
type: str # "memory", "summary", "message"
|
||||
category: Optional[str]
|
||||
text: str
|
||||
created_at: str
|
||||
|
||||
Reference in New Issue
Block a user