""" DAARION Memory Service - Integrations Obsidian та Google Drive інтеграції """ import os import re import json import logging import hashlib import shutil import io from pathlib import Path from typing import List, Dict, Set, Optional, Any, Tuple from datetime import datetime logger = logging.getLogger(__name__) class ObsidianIntegrator: """Obsidian інтегратор для Memory Service""" HOST_VAULT_MAPPINGS = { '/vault/rd': '/Users/apple/Desktop/R&D', '/vault/obsidian': '/Users/apple/Documents/Obsidian Vault', } def __init__(self, vault_path: str = None): self.vault_path = self._resolve_vault_path(vault_path) if vault_path else None self.notes_cache = {} self.links_graph = {} self.tags_index = {} def _resolve_vault_path(self, path: str) -> Path: """Resolve vault path, handling Docker-to-host mappings""" path_obj = Path(path) if path_obj.exists(): return path_obj if path in self.HOST_VAULT_MAPPINGS: resolved = Path(self.HOST_VAULT_MAPPINGS[path]) if resolved.exists(): return resolved return path_obj def find_vault(self) -> Optional[Path]: """Автоматично знайти Obsidian vault""" possible_paths = [ Path.home() / "Documents" / "Obsidian Vault", Path.home() / "Documents" / "Notes", Path.home() / "Desktop" / "Obsidian Vault", Path.home() / "Obsidian", Path.home() / "Notes", ] documents_path = Path.home() / "Documents" if documents_path.exists(): for item in documents_path.iterdir(): if item.is_dir() and (item / ".obsidian").exists(): possible_paths.append(item) for path in possible_paths: if path.exists() and (path / ".obsidian").exists(): logger.info(f"Found Obsidian vault at: {path}") return path return None def set_vault_path(self, vault_path: str) -> bool: resolved = self._resolve_vault_path(vault_path) if not resolved.exists(): logger.error(f"Vault path does not exist: {vault_path} (resolved: {resolved})") return False if not (resolved / ".obsidian").exists(): logger.error(f"Not a valid Obsidian vault: {vault_path} (resolved: {resolved})") return False self.vault_path = resolved logger.info(f"Vault path set to: {resolved}") return True def scan_vault(self) -> Dict[str, Any]: if not self.vault_path: return {} stats = { 'total_notes': 0, 'total_attachments': 0, 'total_links': 0, 'total_tags': 0, 'folders': set(), 'file_types': {}, 'notes': [] } for file_path in self.vault_path.rglob('*'): if file_path.is_file() and not file_path.name.startswith('.'): suffix = file_path.suffix.lower() stats['file_types'][suffix] = stats['file_types'].get(suffix, 0) + 1 relative_folder = file_path.parent.relative_to(self.vault_path) if relative_folder != Path('.'): stats['folders'].add(str(relative_folder)) if suffix == '.md': note_data = self._parse_note(file_path) if note_data: self.notes_cache[file_path.stem] = note_data stats['notes'].append(note_data) stats['total_notes'] += 1 stats['total_links'] += len(note_data['links']) stats['total_tags'] += len(note_data['tags']) else: stats['total_attachments'] += 1 self._build_links_graph() self._build_tags_index() stats['folders'] = list(stats['folders']) return stats def _parse_note(self, file_path: Path) -> Optional[Dict[str, Any]]: try: with open(file_path, 'r', encoding='utf-8') as f: content = f.read() note_data = { 'title': file_path.stem, 'path': str(file_path.relative_to(self.vault_path)), 'full_path': str(file_path), 'size': len(content), 'created': datetime.fromtimestamp(file_path.stat().st_ctime), 'modified': datetime.fromtimestamp(file_path.stat().st_mtime), 'content': content, 'content_hash': hashlib.md5(content.encode()).hexdigest(), 'frontmatter': {}, 'headings': [], 'links': [], 'tags': [], 'backlinks': [], 'blocks': [] } frontmatter_match = re.match(r'^---\s*\n(.*?)\n---\s*\n', content, re.DOTALL) if frontmatter_match: try: import yaml note_data['frontmatter'] = yaml.safe_load(frontmatter_match.group(1)) except Exception: pass headings = re.findall(r'^(#{1,6})\s+(.+)$', content, re.MULTILINE) note_data['headings'] = [(len(h[0]), h[1].strip()) for h in headings] internal_links = re.findall(r'\[\[([^\]]+)\]\]', content) note_data['links'] = [link.split('|')[0].strip() for link in internal_links] tags = re.findall(r'(?:^|\s)#([\w\-\/]+)', content) note_data['tags'] = list(set(tags)) blocks = re.findall(r'\^([\w\-]+)', content) note_data['blocks'] = blocks return note_data except Exception as e: logger.error(f"Error parsing note {file_path}: {e}") return None def _build_links_graph(self): self.links_graph = {} for note_title, note_data in self.notes_cache.items(): self.links_graph[note_title] = { 'outbound': note_data['links'], 'inbound': [] } for note_title, note_data in self.notes_cache.items(): for linked_note in note_data['links']: if linked_note in self.links_graph: self.links_graph[linked_note]['inbound'].append(note_title) if linked_note in self.notes_cache: self.notes_cache[linked_note]['backlinks'].append(note_title) def _build_tags_index(self): self.tags_index = {} for note_title, note_data in self.notes_cache.items(): for tag in note_data['tags']: if tag not in self.tags_index: self.tags_index[tag] = [] self.tags_index[tag].append(note_title) def search_notes(self, query: str, search_content: bool = True) -> List[Dict[str, Any]]: results = [] query_lower = query.lower() for note_title, note_data in self.notes_cache.items(): match_score = 0 if query_lower in note_title.lower(): match_score += 10 for tag in note_data['tags']: if query_lower in tag.lower(): match_score += 5 if search_content and query_lower in note_data['content'].lower(): match_score += 1 if match_score > 0: result = note_data.copy() result['match_score'] = match_score results.append(result) results.sort(key=lambda x: x['match_score'], reverse=True) return results def get_status(self) -> Dict[str, Any]: return { 'available': True, 'vault_configured': self.vault_path is not None, 'vault_path': str(self.vault_path) if self.vault_path else None, 'notes_count': len(self.notes_cache), 'tags_count': len(self.tags_index) } class GoogleDriveIntegrator: """Google Drive інтегратор для Memory Service""" SCOPES = ['https://www.googleapis.com/auth/drive.readonly'] CREDENTIALS_DIR = Path.home() / '.daarion' CREDENTIALS_FILE = CREDENTIALS_DIR / 'google_credentials.json' TOKEN_FILE = CREDENTIALS_DIR / 'google_token.json' SUPPORTED_MIMETYPES = { 'application/vnd.google-apps.document': 'text/plain', 'application/vnd.openxmlformats-officedocument.wordprocessingml.document': None, 'application/pdf': None, 'text/plain': None, 'text/markdown': None, 'application/vnd.google-apps.spreadsheet': 'text/csv', } def __init__(self): self.service = None self.CREDENTIALS_DIR.mkdir(exist_ok=True) def authenticate(self) -> bool: try: from googleapiclient.discovery import build from google_auth_oauthlib.flow import InstalledAppFlow from google.auth.transport.requests import Request from google.oauth2.credentials import Credentials except ImportError: logger.warning("Google API libraries not installed") return False creds = None if self.TOKEN_FILE.exists(): creds = Credentials.from_authorized_user_file(str(self.TOKEN_FILE), self.SCOPES) if not creds or not creds.valid: if creds and creds.expired and creds.refresh_token: creds.refresh(Request()) else: client_secrets = self.CREDENTIALS_DIR / 'client_secrets.json' if not client_secrets.exists(): logger.warning("Google client_secrets.json not found") return False flow = InstalledAppFlow.from_client_secrets_file( str(client_secrets), self.SCOPES) creds = flow.run_local_server(port=0) with open(self.TOKEN_FILE, 'w') as token: token.write(creds.to_json()) self.service = build('drive', 'v3', credentials=creds) logger.info("Google Drive API authenticated") return True def list_files(self, folder_id: Optional[str] = None, max_results: int = 100) -> List[Dict]: if not self.service: if not self.authenticate(): return [] search_query = [] if folder_id: search_query.append(f"'{folder_id}' in parents") mime_conditions = [] for mime_type in self.SUPPORTED_MIMETYPES.keys(): mime_conditions.append(f"mimeType='{mime_type}'") if mime_conditions: search_query.append(f"({' or '.join(mime_conditions)})") search_query.extend([ "trashed=false", "mimeType!='application/vnd.google-apps.folder'" ]) final_query = ' and '.join(search_query) try: results = self.service.files().list( q=final_query, pageSize=max_results, fields="nextPageToken, files(id, name, mimeType, size, createdTime, modifiedTime, parents, webViewLink)" ).execute() return results.get('files', []) except Exception as e: logger.error(f"Error listing Google Drive files: {e}") return [] def download_file(self, file_id: str, mime_type: str) -> Optional[str]: if not self.service: return None try: from googleapiclient.http import MediaIoBaseDownload export_mime_type = self.SUPPORTED_MIMETYPES.get(mime_type) if export_mime_type: request = self.service.files().export_media( fileId=file_id, mimeType=export_mime_type ) else: request = self.service.files().get_media(fileId=file_id) file_io = io.BytesIO() downloader = MediaIoBaseDownload(file_io, request) done = False while done is False: status, done = downloader.next_chunk() content = file_io.getvalue() for encoding in ['utf-8', 'utf-16', 'latin-1']: try: return content.decode(encoding) except UnicodeDecodeError: continue return None except Exception as e: logger.error(f"Error downloading file {file_id}: {e}") return None def get_status(self) -> Dict[str, Any]: available = False authenticated = False try: from googleapiclient.discovery import build from google.oauth2.credentials import Credentials available = True if self.TOKEN_FILE.exists(): authenticated = True except ImportError: pass return { 'available': available, 'authenticated': authenticated, 'credentials_configured': self.CREDENTIALS_DIR.exists() } def sync_to_daarion(self, output_dir: Path, folder_ids: List[str] = None, file_extensions: List[str] = None) -> Dict[str, Any]: """Sync files from Google Drive to DAARION""" stats = { 'total_files': 0, 'downloaded': 0, 'errors': 0, 'skipped': 0, 'files': [] } all_files = [] if folder_ids: for folder_id in folder_ids: files = self.list_files(folder_id=folder_id) all_files.extend(files) else: all_files = self.list_files() stats['total_files'] = len(all_files) for file_data in all_files: file_id = file_data['id'] file_name = file_data['name'] mime_type = file_data['mimeType'] if file_extensions: file_ext = Path(file_name).suffix.lower() if file_ext not in file_extensions: stats['skipped'] += 1 continue content = self.download_file(file_id, mime_type) if content: safe_filename = "".join(c for c in file_name if c.isalnum() or c in (' ', '-', '_', '.')).rstrip() file_path = output_dir / f"gdrive_{file_id}_{safe_filename}.txt" try: with open(file_path, 'w', encoding='utf-8') as f: f.write(f"# Google Drive: {file_name}\n") f.write(f"Source: {file_data.get('webViewLink', 'N/A')}\n") f.write(f"Modified: {file_data.get('modifiedTime', 'N/A')}\n") f.write(f"MIME Type: {mime_type}\n\n") f.write("---\n\n") f.write(content) stats['downloaded'] += 1 stats['files'].append({ 'original_name': file_name, 'saved_path': str(file_path), 'file_id': file_id, 'size': len(content), 'url': file_data.get('webViewLink') }) except Exception as e: logger.error(f"Error saving {file_name}: {e}") stats['errors'] += 1 else: stats['errors'] += 1 return stats def get_folder_structure(self, folder_id: str = None, level: int = 0) -> Dict: """Get Google Drive folder structure""" if not self.service: if not self.authenticate(): return {} try: query = "mimeType='application/vnd.google-apps.folder' and trashed=false" if folder_id: query += f" and '{folder_id}' in parents" results = self.service.files().list( q=query, fields="files(id, name, parents)" ).execute() folders = results.get('files', []) structure = {} for folder in folders: folder_name = folder['name'] fid = folder['id'] structure[folder_name] = { 'id': fid, 'subfolders': self.get_folder_structure(fid, level + 1) if level < 3 else {} } return structure except Exception as e: logger.error(f"Error getting folder structure: {e}") return {} obsidian_integrator = ObsidianIntegrator() gdrive_integrator = GoogleDriveIntegrator()