feat(db-hardening): Add database persistence, backups, and MinIO assets storage
Database Hardening: - Add docker-compose.db.yml with persistent PostgreSQL volume - Add automatic DB backups every 12h (7 days, 4 weeks, 6 months retention) - Add MinIO S3-compatible storage for assets Assets Migration: - Add MinIO client (lib/assets_client.py) for upload/delete - Update upload endpoint to use MinIO (with local fallback) - Add migration 043_asset_urls_to_text.sql for full HTTPS URLs - Simplify normalizeAssetUrl for S3 URLs Recovery: - Add seed_full_city_reset.py for emergency city recovery - Add DB_RESTORE.md with backup restore instructions - Add SEED_RECOVERY.md with recovery procedures - Add INFRA_ASSETS_MINIO.md with MinIO setup guide Task: TASK_PHASE_DATABASE_HARDENING_AND_ASSETS_MIGRATION_v1
This commit is contained in:
Apple
162
services/city-service/lib/assets_client.py
Normal file
162
services/city-service/lib/assets_client.py
Normal file
@@ -0,0 +1,162 @@
|
||||
"""
|
||||
MinIO client for uploading assets (logos, banners, avatars) to S3-compatible storage.
|
||||
"""
|
||||
import os
|
||||
from datetime import datetime
|
||||
from uuid import uuid4
|
||||
from typing import BinaryIO, Optional
|
||||
from minio import Minio
|
||||
from minio.error import S3Error
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# MinIO configuration from environment
|
||||
MINIO_ENDPOINT = os.getenv("MINIO_ENDPOINT", "http://minio:9000")
|
||||
MINIO_ACCESS_KEY = os.getenv("MINIO_ROOT_USER", "assets-admin")
|
||||
MINIO_SECRET_KEY = os.getenv("MINIO_ROOT_PASSWORD", "")
|
||||
ASSETS_BUCKET = os.getenv("ASSETS_BUCKET", "daarion-assets")
|
||||
ASSETS_PUBLIC_BASE_URL = os.getenv(
|
||||
"ASSETS_PUBLIC_BASE_URL",
|
||||
"https://assets.daarion.space/daarion-assets"
|
||||
)
|
||||
|
||||
# Initialize MinIO client
|
||||
_minio_client: Optional[Minio] = None
|
||||
|
||||
|
||||
def get_minio_client() -> Minio:
|
||||
"""Get or create MinIO client instance."""
|
||||
global _minio_client
|
||||
if _minio_client is None:
|
||||
# Remove http:// or https:// from endpoint for MinIO client
|
||||
endpoint = MINIO_ENDPOINT.replace("http://", "").replace("https://", "")
|
||||
secure = MINIO_ENDPOINT.startswith("https://")
|
||||
|
||||
_minio_client = Minio(
|
||||
endpoint,
|
||||
access_key=MINIO_ACCESS_KEY,
|
||||
secret_key=MINIO_SECRET_KEY,
|
||||
secure=secure,
|
||||
)
|
||||
return _minio_client
|
||||
|
||||
|
||||
def ensure_bucket():
|
||||
"""Ensure the assets bucket exists, create if it doesn't."""
|
||||
try:
|
||||
client = get_minio_client()
|
||||
if not client.bucket_exists(ASSETS_BUCKET):
|
||||
client.make_bucket(ASSETS_BUCKET)
|
||||
logger.info(f"Created bucket: {ASSETS_BUCKET}")
|
||||
else:
|
||||
logger.debug(f"Bucket {ASSETS_BUCKET} already exists")
|
||||
except S3Error as e:
|
||||
logger.error(f"Failed to ensure bucket {ASSETS_BUCKET}: {e}")
|
||||
raise
|
||||
|
||||
|
||||
def upload_asset(
|
||||
file_obj: BinaryIO,
|
||||
content_type: str,
|
||||
prefix: str,
|
||||
filename: Optional[str] = None
|
||||
) -> str:
|
||||
"""
|
||||
Upload a file to MinIO and return the public URL.
|
||||
|
||||
Args:
|
||||
file_obj: File-like object to upload
|
||||
content_type: MIME type (e.g., 'image/png', 'image/jpeg')
|
||||
prefix: Path prefix (e.g., 'microdao/logo', 'agents/avatar')
|
||||
filename: Optional original filename (for extension detection)
|
||||
|
||||
Returns:
|
||||
Full public HTTPS URL to the uploaded asset
|
||||
"""
|
||||
try:
|
||||
ensure_bucket()
|
||||
client = get_minio_client()
|
||||
|
||||
# Generate object key: prefix/YYYY/MM/DD/uuid.ext
|
||||
date_path = datetime.utcnow().strftime("%Y/%m/%d")
|
||||
unique_id = uuid4().hex
|
||||
|
||||
# Try to extract extension from filename or content_type
|
||||
ext = ""
|
||||
if filename:
|
||||
ext = os.path.splitext(filename)[1] or ""
|
||||
elif content_type:
|
||||
# Map common content types to extensions
|
||||
ext_map = {
|
||||
"image/png": ".png",
|
||||
"image/jpeg": ".jpg",
|
||||
"image/jpg": ".jpg",
|
||||
"image/webp": ".webp",
|
||||
"image/gif": ".gif",
|
||||
}
|
||||
ext = ext_map.get(content_type, "")
|
||||
|
||||
object_key = f"{prefix}/{date_path}/{unique_id}{ext}"
|
||||
|
||||
# Get file size (seek to end, get position, reset)
|
||||
file_obj.seek(0, 2) # Seek to end
|
||||
file_size = file_obj.tell()
|
||||
file_obj.seek(0) # Reset to beginning
|
||||
|
||||
# Upload to MinIO
|
||||
client.put_object(
|
||||
ASSETS_BUCKET,
|
||||
object_key,
|
||||
file_obj,
|
||||
length=file_size,
|
||||
content_type=content_type,
|
||||
)
|
||||
|
||||
# Return public URL
|
||||
public_url = f"{ASSETS_PUBLIC_BASE_URL}/{object_key}"
|
||||
logger.info(f"Uploaded asset to {public_url}")
|
||||
|
||||
return public_url
|
||||
|
||||
except S3Error as e:
|
||||
logger.error(f"Failed to upload asset: {e}")
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"Unexpected error uploading asset: {e}")
|
||||
raise
|
||||
|
||||
|
||||
def delete_asset(object_key: str) -> bool:
|
||||
"""
|
||||
Delete an asset from MinIO.
|
||||
|
||||
Args:
|
||||
object_key: Full object key or URL (will extract key from URL if needed)
|
||||
|
||||
Returns:
|
||||
True if deleted, False otherwise
|
||||
"""
|
||||
try:
|
||||
client = get_minio_client()
|
||||
|
||||
# If it's a full URL, extract the object key
|
||||
if object_key.startswith("http://") or object_key.startswith("https://"):
|
||||
# Extract key from URL like: https://assets.daarion.space/daarion-assets/microdao/logo/...
|
||||
parts = object_key.split(f"{ASSETS_PUBLIC_BASE_URL}/")
|
||||
if len(parts) < 2:
|
||||
logger.warning(f"Could not extract object key from URL: {object_key}")
|
||||
return False
|
||||
object_key = parts[1]
|
||||
|
||||
client.remove_object(ASSETS_BUCKET, object_key)
|
||||
logger.info(f"Deleted asset: {object_key}")
|
||||
return True
|
||||
|
||||
except S3Error as e:
|
||||
logger.error(f"Failed to delete asset {object_key}: {e}")
|
||||
return False
|
||||
except Exception as e:
|
||||
logger.error(f"Unexpected error deleting asset: {e}")
|
||||
return False
|
||||
|
||||
@@ -9,3 +9,4 @@ httpx==0.26.0
|
||||
nats-py==2.6.0
|
||||
Pillow==10.2.0
|
||||
python-multipart==0.0.9
|
||||
minio==7.2.0
|
||||
|
||||
@@ -14,6 +14,14 @@ import uuid
|
||||
from PIL import Image
|
||||
import shutil
|
||||
|
||||
# MinIO assets client
|
||||
try:
|
||||
from lib.assets_client import upload_asset as minio_upload_asset
|
||||
MINIO_AVAILABLE = True
|
||||
except ImportError:
|
||||
MINIO_AVAILABLE = False
|
||||
logger.warning("MinIO client not available, falling back to local storage")
|
||||
|
||||
from models_city import (
|
||||
CityRoomRead,
|
||||
CityRoomCreate,
|
||||
@@ -339,24 +347,69 @@ async def upload_asset(
|
||||
|
||||
processed_bytes, thumb_bytes = process_image(content, max_size=max_size, force_square=force_square)
|
||||
|
||||
# Save to disk
|
||||
# Map type to prefix
|
||||
type_to_prefix = {
|
||||
'microdao_logo': 'microdao/logo',
|
||||
'microdao_banner': 'microdao/banner',
|
||||
'room_logo': 'rooms/logo',
|
||||
'room_banner': 'rooms/banner',
|
||||
'agent_avatar': 'agents/avatar',
|
||||
}
|
||||
prefix = type_to_prefix.get(type, 'uploads')
|
||||
|
||||
# Upload to MinIO if available, otherwise fallback to local storage
|
||||
if MINIO_AVAILABLE:
|
||||
try:
|
||||
# Upload processed image
|
||||
processed_file = io.BytesIO(processed_bytes)
|
||||
processed_url = minio_upload_asset(
|
||||
processed_file,
|
||||
content_type="image/png",
|
||||
prefix=prefix,
|
||||
filename=file.filename
|
||||
)
|
||||
|
||||
# Upload thumbnail if exists
|
||||
thumb_url = None
|
||||
if thumb_bytes:
|
||||
thumb_file = io.BytesIO(thumb_bytes)
|
||||
thumb_url = minio_upload_asset(
|
||||
thumb_file,
|
||||
content_type="image/png",
|
||||
prefix=f"{prefix}/thumb",
|
||||
filename=file.filename
|
||||
)
|
||||
|
||||
return {
|
||||
"original_url": processed_url,
|
||||
"processed_url": processed_url,
|
||||
"thumb_url": thumb_url or processed_url
|
||||
}
|
||||
except Exception as e:
|
||||
logger.warning(f"MinIO upload failed, falling back to local: {e}")
|
||||
# Fall through to local storage
|
||||
|
||||
# Fallback: Save to local disk
|
||||
filename = f"{uuid.uuid4()}.png"
|
||||
filepath = f"static/uploads/{filename}"
|
||||
thumb_filepath = f"static/uploads/thumb_{filename}"
|
||||
|
||||
os.makedirs("static/uploads", exist_ok=True)
|
||||
|
||||
with open(filepath, "wb") as f:
|
||||
f.write(processed_bytes)
|
||||
|
||||
with open(thumb_filepath, "wb") as f:
|
||||
f.write(thumb_bytes)
|
||||
|
||||
if thumb_bytes:
|
||||
with open(thumb_filepath, "wb") as f:
|
||||
f.write(thumb_bytes)
|
||||
|
||||
# Construct URLs
|
||||
base_url = "/static/uploads"
|
||||
base_url = "/api/static/uploads"
|
||||
|
||||
return {
|
||||
"original_url": f"{base_url}/{filename}",
|
||||
"processed_url": f"{base_url}/{filename}",
|
||||
"thumb_url": f"{base_url}/thumb_{filename}"
|
||||
"thumb_url": f"{base_url}/thumb_{filename}" if thumb_bytes else f"{base_url}/{filename}"
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
|
||||
Reference in New Issue
Block a user