"""
Category-based File Management Tools for MCP Server

Provides write/edit/read/list tools for categorized Python scripts with automatic
directory structure, metadata management, and validation harnesses.

Categories:
- strategy: Trading strategies with specific data feed requirements
- indicator: Technical indicators with configurable parameters
- research: Research scripts with pyplot visualization support

File Structure:
{DATA_DIR}/{category}/{sanitized_name}/
  - implementation.py  # Python implementation
  - metadata.json      # Category-specific metadata (includes name and description)

After write/edit operations, a category-specific test harness runs to validate
the code and capture errors/output for agent feedback.
"""

import base64
import json
import logging
import re
import subprocess
import sys
from dataclasses import dataclass, asdict
from enum import Enum
from pathlib import Path
from typing import Any, Optional

from dexorder.tools.subprocess_runner import run_subprocess_argv, run_in_thread

log = logging.getLogger(__name__)

# Paths to harness scripts run as subprocesses
_RESEARCH_HARNESS = Path(__file__).parent / "research_harness.py"
_STRATEGY_HARNESS = Path(__file__).parent / "strategy_harness.py"


# Import conda manager for package installation and tracking
try:
    from dexorder.conda_manager import install_packages_async, cleanup_extra_packages_async
except ImportError:
    log.warning("conda_manager not available - package installation disabled")
    install_packages_async = None
    cleanup_extra_packages_async = None


# =============================================================================
# Categories and Metadata
# =============================================================================

class Category(str, Enum):
    """Supported file categories."""
    STRATEGY = "strategy"
    INDICATOR = "indicator"
    RESEARCH = "research"


@dataclass
class BaseMetadata:
    """Base metadata for all categories."""
    name: str  # Display name (can have special chars)
    description: str  # LLM-generated description
    conda_packages: list[str] = None  # Additional conda packages required

    def __post_init__(self):
        if self.conda_packages is None:
            self.conda_packages = []


@dataclass
class StrategyMetadata(BaseMetadata):
    """Metadata for trading strategies."""
    data_feeds: list[dict] = None  # Required data feeds: [{"symbol": "BTC/USDT.BINANCE", "period_seconds": 3600, "description": "..."}]
    parameters: dict = None  # Strategy parameters: {"param_name": {"default": value, "description": "..."}}

    def __post_init__(self):
        super().__post_init__()
        if self.data_feeds is None:
            self.data_feeds = []
        if self.parameters is None:
            self.parameters = {}


@dataclass
class IndicatorMetadata(BaseMetadata):
    """Metadata for technical indicators."""
    # Canonical pandas-ta name, e.g. "custom_trendflex". Set automatically
    # by CategoryFileManager.write() — do not pass manually.
    pandas_ta_name: str = ""

    # Fields for TradingView custom study auto-construction:
    parameters: dict = None
    # Parameter schema: {param_name: {type: "int"|"float"|"bool"|"string",
    #   default: value, description?: str, min?: num, max?: num}}
    # Example: {"length": {"type": "int", "default": 14, "min": 1, "max": 500}}

    input_series: list = None
    # OHLCV columns the indicator function receives as positional args.
    # Valid values: "open", "high", "low", "close", "volume"
    # Example: ["close"] or ["high", "low", "close", "volume"]

    output_columns: list = None
    # Output series produced by the function. Each entry:
    #   {
    #     name: str,           # column name (or "value" for plain Series)
    #     display_name?: str,  # label shown in TV legend
    #     description?: str,
    #     plot?: {
    #       style: int,        # LineStudyPlotStyle: 0=Line, 1=Histogram, 3=Dots/Cross,
    #                          #   4=Area, 5=Columns, 6=Circles, 9=StepLine
    #       color?: str,       # CSS hex e.g. "#2196F3" (auto-assigned if omitted)
    #       linewidth?: int,   # 1–4 (default 2)
    #       visible?: bool     # default true
    #     }
    #   }
    # Example (single line):   [{"name": "value", "display_name": "My Indicator"}]
    # Example (multi-line):    [{"name": "upper", "plot": {"style": 0}}, {"name": "lower", "plot": {"style": 0}}]
    # Example (histogram):     [{"name": "value", "plot": {"style": 1}}]
    # Example (MACD-style):    [{"name": "macd", "plot": {"style": 0}}, {"name": "signal", "plot": {"style": 0}}, {"name": "hist", "plot": {"style": 1}}]

    pane: str = "separate"
    # Where to render: "price" (overlaid on candles) or "separate" (sub-pane)

    filled_areas: list = None
    # Optional shaded regions between two plots or two bands. Each entry:
    #   {
    #     id: str,           # unique id e.g. "fill_upper_lower"
    #     type: str,         # "plot_plot" (between two series) or "hline_hline" (between two bands)
    #     series1: str,      # output_column name (for plot_plot) or band id (for hline_hline)
    #     series2: str,
    #     color?: str,       # CSS hex fill color (default semi-transparent blue)
    #     opacity?: float    # 0.0–1.0 (default 0.1)
    #   }
    # Example (Bollinger fill): [{"id": "fill", "type": "plot_plot", "series1": "upper", "series2": "lower", "color": "#2196F3", "opacity": 0.1}]

    bands: list = None
    # Optional horizontal reference lines (e.g. RSI overbought/oversold). Each entry:
    #   {
    #     id: str,          # unique id e.g. "ob"
    #     value: float,     # fixed y-level
    #     color?: str,      # CSS hex (default "#787B86")
    #     linewidth?: int,  # default 1
    #     linestyle?: int,  # 0=solid, 1=dotted, 2=dashed (default 2)
    #     visible?: bool    # default true
    #   }
    # Example (RSI levels): [{"id": "ob", "value": 70}, {"id": "os", "value": 30}]

    def __post_init__(self):
        super().__post_init__()
        if self.input_series is None:
            self.input_series = ["close"]
        if self.output_columns is None:
            self.output_columns = [{"name": "value"}]
        if self.parameters is None:
            self.parameters = {}
        if self.filled_areas is None:
            self.filled_areas = []
        if self.bands is None:
            self.bands = []


@dataclass
class ResearchMetadata(BaseMetadata):
    """Metadata for research scripts."""
    output: dict = None  # Output files: {"analysis": "analysis.md", "images": ["img1.png", ...]}

    def __post_init__(self):
        super().__post_init__()
        if self.output is None:
            self.output = {}


# Metadata class registry
METADATA_CLASSES = {
    Category.STRATEGY: StrategyMetadata,
    Category.INDICATOR: IndicatorMetadata,
    Category.RESEARCH: ResearchMetadata,
}


# =============================================================================
# Utilities
# =============================================================================

def sanitize_name(name: str) -> str:
    """
    Sanitize a name for use as a directory name.

    Converts special characters to underscores and preserves alphanumerics.
    Examples:
        "Tim's 5/13 EMA+" -> "Tims_5_13_EMA"
        "My Strategy (v2)" -> "My_Strategy_v2"
    """
    # Replace non-alphanumeric chars (except spaces/hyphens) with underscores
    sanitized = re.sub(r'[^\w\s\-]', '_', name)
    # Replace spaces and hyphens with underscores
    sanitized = re.sub(r'[\s\-]+', '_', sanitized)
    # Remove duplicate underscores
    sanitized = re.sub(r'_+', '_', sanitized)
    # Strip leading/trailing underscores
    sanitized = sanitized.strip('_')
    return sanitized


def get_category_path(data_dir: Path, category: Category, name: str) -> Path:
    """
    Get the directory path for a category item.

    Args:
        data_dir: Base data directory
        category: Category enum
        name: Display name (will be sanitized)

    Returns:
        Path to the category item directory
    """
    safe_name = sanitize_name(name)
    return data_dir / category.value / safe_name


# =============================================================================
# Git Manager
# =============================================================================

class GitManager:
    """
    Thin wrapper around git subprocess calls for category revision tracking.
    All operations are non-fatal: errors are logged as warnings.
    """

    def __init__(self, repo_dir: Path):
        self.repo_dir = repo_dir

    def _run(self, *args, check: bool = True) -> subprocess.CompletedProcess:
        return subprocess.run(
            ["git"] + list(args),
            cwd=self.repo_dir,
            capture_output=True,
            text=True,
            check=check,
        )

    def ensure_init(self):
        """Init git repo if not exists; initial commit if files already present."""
        if (self.repo_dir / ".git").exists():
            return
        self._run("init", "-b", "main")
        self._run("config", "user.email", "sandbox@dexorder.ai")
        self._run("config", "user.name", "Dexorder Sandbox")
        # Commit any pre-existing files (migrated from old layout)
        status = self._run("status", "--porcelain")
        if status.stdout.strip():
            self._run("add", "-A")
            self._run("commit", "-m", "init: migrate existing category files")
        log.info(f"Git repo initialized at {self.repo_dir}")

    def commit(self, message: str) -> Optional[str]:
        """Stage all changes and commit. Returns short hash or None if nothing to commit / on error."""
        try:
            self._run("add", "-A")
            status = self._run("status", "--porcelain")
            if not status.stdout.strip():
                return None  # nothing changed
            self._run("commit", "-m", message)
            result = self._run("rev-parse", "--short", "HEAD")
            return result.stdout.strip()
        except Exception as e:
            log.warning(f"Git commit failed (non-fatal): {e}")
            return None

    def log(self, path: Optional[Path] = None, n: int = 20) -> list[dict]:
        """Return recent commits, optionally filtered to a path."""
        cmd = ["log", f"-{n}", "--pretty=format:%H|%h|%s|%ai"]
        if path:
            cmd += ["--", str(path.relative_to(self.repo_dir))]
        result = self._run(*cmd, check=False)
        entries = []
        for line in result.stdout.strip().splitlines():
            if line:
                parts = line.split("|", 3)
                if len(parts) == 4:
                    entries.append({
                        "hash": parts[0],
                        "short_hash": parts[1],
                        "message": parts[2],
                        "date": parts[3],
                    })
        return entries

    def restore(self, revision: str, path: Optional[Path] = None) -> Optional[str]:
        """Restore path (or entire tree) to revision state. Returns new commit hash."""
        try:
            rel = str(path.relative_to(self.repo_dir)) if path else "."
            self._run("checkout", revision, "--", rel)
            return self.commit(f"revert: restore to {revision[:8]}")
        except subprocess.CalledProcessError as e:
            raise RuntimeError(e.stderr.strip()) from e

    def head_short_hash(self) -> str:
        """Return the short hash of HEAD, or 'unknown' on error."""
        try:
            result = self._run("rev-parse", "--short", "HEAD")
            return result.stdout.strip()
        except Exception:
            return "unknown"

    def create_worktree(self, worktree_path: Path, revision: str = "HEAD") -> str:
        """
        Create a git worktree at worktree_path pinned to revision.

        Returns the short hash of the checked-out commit.
        """
        worktree_path.parent.mkdir(parents=True, exist_ok=True)
        try:
            self._run("worktree", "add", "--detach", str(worktree_path), revision)
            # Get short hash of the worktree's HEAD
            result = subprocess.run(
                ["git", "rev-parse", "--short", "HEAD"],
                cwd=str(worktree_path),
                capture_output=True,
                text=True,
                check=True,
            )
            return result.stdout.strip()
        except subprocess.CalledProcessError as e:
            raise RuntimeError(f"git worktree add failed: {e.stderr.strip()}") from e

    def remove_worktree(self, worktree_path: Path) -> None:
        """Remove a git worktree, silently ignoring errors if it no longer exists."""
        try:
            self._run("worktree", "remove", "--force", str(worktree_path), check=False)
        except Exception as e:
            log.warning("git worktree remove failed (non-fatal): %s", e)

    def prune_worktrees(self) -> None:
        """Prune stale worktree references."""
        try:
            self._run("worktree", "prune", check=False)
        except Exception:
            pass

    # ------------------------------------------------------------------
    # Async variants — delegates to sync methods via asyncio.to_thread
    # so the event loop stays responsive during git operations.
    # ------------------------------------------------------------------

    async def commit_async(self, message: str) -> Optional[str]:
        import asyncio
        return await asyncio.to_thread(self.commit, message)

    async def log_async(self, path: Optional[Path] = None, n: int = 20) -> list[dict]:
        import asyncio
        return await asyncio.to_thread(self.log, path, n)

    async def restore_async(self, revision: str, path: Optional[Path] = None) -> Optional[str]:
        import asyncio
        return await asyncio.to_thread(self.restore, revision, path)

    async def head_short_hash_async(self) -> str:
        import asyncio
        return await asyncio.to_thread(self.head_short_hash)

    async def create_worktree_async(self, worktree_path: Path, revision: str = "HEAD") -> str:
        import asyncio
        return await asyncio.to_thread(self.create_worktree, worktree_path, revision)

    async def remove_worktree_async(self, worktree_path: Path) -> None:
        import asyncio
        return await asyncio.to_thread(self.remove_worktree, worktree_path)

    async def prune_worktrees_async(self) -> None:
        import asyncio
        return await asyncio.to_thread(self.prune_worktrees)


# =============================================================================
# Custom Indicator Setup
# =============================================================================

def setup_custom_indicators(data_dir: Path) -> None:
    """
    Register user's custom indicators with pandas-ta.

    Loads each indicator's implementation.py directly via importlib and binds
    the function as ``ta.custom_{sanitized_name}`` so that evaluate_indicator
    can call it as ``getattr(ta, "custom_trendflex", None)``.

    The binding is idempotent — indicators already registered are skipped.

    Note: pandas-ta's ta.import_dir() requires a category-based directory
    structure (e.g. tmpdir/momentum/trendflex.py) plus a companion
    ``{name}_method`` function.  Our indicators don't follow that convention,
    so we bind directly instead.
    """
    try:
        import pandas_ta as ta
    except ImportError:
        log.warning("pandas-ta not available — custom indicators will not be registered")
        return

    src_dir = data_dir / "src"
    indicator_root = src_dir / "indicator"
    if not indicator_root.exists():
        return

    import importlib.util
    import types

    # Track which sanitized names we've seen to handle duplicate directories
    # (e.g. "TrendFlex" and "trendflex" both sanitise to "custom_trendflex").
    seen: set[str] = set()
    registered = 0

    # Sort so that exact-lowercase names (e.g. "trendflex") come before mixed-case
    # variants (e.g. "TrendFlex") — when duplicates exist the lowercase one wins.
    for ind_dir in sorted(indicator_root.iterdir(), key=lambda p: (p.name != p.name.lower(), p.name.lower())):
        if not ind_dir.is_dir():
            continue
        impl = ind_dir / "implementation.py"
        if not impl.exists():
            continue

        sanitized = ind_dir.name.lower().replace("-", "_").replace(" ", "_")
        ta_name = f"custom_{sanitized}"

        if ta_name in seen:
            log.warning(
                "Duplicate custom indicator name '%s' from directory '%s' — skipping",
                ta_name, ind_dir.name,
            )
            continue
        seen.add(ta_name)

        # Skip if already bound (e.g. called multiple times in a process)
        if getattr(ta, ta_name, None) is not None:
            continue

        try:
            spec = importlib.util.spec_from_file_location(ta_name, impl)
            module = importlib.util.module_from_spec(spec)
            spec.loader.exec_module(module)  # type: ignore[union-attr]

            # Find the callable: prefer the function whose name matches the
            # sanitized directory name, fall back to any top-level function.
            fn = getattr(module, sanitized, None)
            if fn is None:
                candidates = [
                    v for k, v in vars(module).items()
                    if isinstance(v, types.FunctionType) and not k.startswith("_")
                ]
                fn = candidates[0] if candidates else None

            if fn is None:
                log.warning("No callable found in %s — skipping", impl)
                continue

            setattr(ta, ta_name, fn)
            registered += 1
            log.debug("Registered custom indicator '%s' from %s", ta_name, impl)

        except Exception:
            log.warning("Could not register indicator '%s':", ind_dir.name, exc_info=True)

    if registered > 0:
        log.info("Registered %d custom indicator(s) with pandas-ta", registered)


# =============================================================================
# Category File Manager
# =============================================================================

class CategoryFileManager:
    """
    Manages category-based file operations with validation and git revision tracking.
    Category files live under {data_dir}/src/ which is the git repo root.
    Workspace and other ephemeral data remain under {data_dir}/ but outside the repo.
    """

    def __init__(self, data_dir: Path):
        self.data_dir = data_dir

        src = self.src_dir
        src.mkdir(parents=True, exist_ok=True)

        # Migrate: move existing top-level category dirs into src/ (one-time)
        for cat in Category:
            old = data_dir / cat.value
            new = src / cat.value
            if old.exists() and not new.exists():
                old.rename(new)
                log.info(f"Migrated {old} → {new}")
            else:
                new.mkdir(exist_ok=True)

        # Init git repo in src/
        self.git = GitManager(src)
        self.git.ensure_init()

    @property
    def src_dir(self) -> Path:
        """Root of the versioned category code (git repo root)."""
        return self.data_dir / "src"

    async def write(
        self,
        category: str,
        name: str,
        description: str,
        details: str,
        code: str,
        metadata: Optional[dict] = None
    ) -> dict[str, Any]:
        """
        Write a new category file with metadata.

        Args:
            category: Category name (strategy, indicator, research)
            name: Display name for the item
            description: LLM-generated description (required)
            details: Full markdown description with enough detail to reproduce the code (required)
            code: Python implementation code
            metadata: Additional category-specific metadata fields

        Returns:
            dict with:
            - success: bool
            - path: str - path to the implementation file
            - validation: dict - results from test harness
            - error: str (if any)
        """
        try:
            cat = Category(category)
        except ValueError:
            return {
                "success": False,
                "error": f"Invalid category '{category}'. Must be one of: {', '.join(c.value for c in Category)}"
            }

        # Get item directory
        item_dir = get_category_path(self.src_dir, cat, name)
        item_dir.mkdir(parents=True, exist_ok=True)

        # Write implementation
        impl_path = item_dir / "implementation.py"
        try:
            impl_path.write_text(code)
            log.info(f"Wrote {cat.value} implementation: {impl_path}")
        except Exception as e:
            return {"success": False, "error": f"Failed to write implementation: {e}"}

        # Write details.md (stored separately from metadata)
        details_path = item_dir / "details.md"
        try:
            details_path.write_text(details or "")
            log.info(f"Wrote details: {details_path}")
        except Exception as e:
            return {"success": False, "error": f"Failed to write details: {e}"}

        # Build metadata (details stored separately in details.md)
        meta_dict = metadata or {}
        meta_dict["name"] = name
        meta_dict["description"] = description
        meta_dict.pop("details", None)  # ensure details not stored in metadata

        # For indicators, store the canonical pandas_ta_name so the reverse
        # mapping (ta_name → directory) is reliable regardless of name casing.
        if cat == Category.INDICATOR:
            sanitized = sanitize_name(name).lower()
            meta_dict["pandas_ta_name"] = f"custom_{sanitized}"

        # Validate and write metadata
        try:
            metadata_class = METADATA_CLASSES[cat]
            meta_obj = metadata_class(**meta_dict)
            validated_meta = asdict(meta_obj)

            meta_path = item_dir / "metadata.json"
            meta_path.write_text(json.dumps(validated_meta, indent=2))
            log.info(f"Wrote metadata: {meta_path}")
        except Exception as e:
            return {"success": False, "error": f"Failed to write metadata: {e}"}

        # Run validation harness
        validation = await self._validate(cat, item_dir)

        result = {
            "success": validation["success"],
            "path": str(impl_path),
            "validation": validation,
        }

        # Auto-execute after successful write to give the agent immediate runtime feedback
        if validation["success"]:
            if cat == Category.RESEARCH:
                log.info(f"Auto-executing research script: {name}")
                result["execution"] = await self.execute_research(name, commit=False)
            elif cat == Category.INDICATOR:
                log.info(f"Auto-executing indicator test: {name}")
                result["execution"] = await self._execute_indicator(item_dir)

        # Commit to git
        commit_hash = await self.git.commit_async(f"create({category}): {name}")
        if commit_hash:
            result["revision"] = commit_hash

        return result

    async def edit(
        self,
        category: str,
        name: str,
        code: Optional[str] = None,
        patches: Optional[list[dict]] = None,
        description: Optional[str] = None,
        details: Optional[str] = None,
        detail_patches: Optional[list[dict]] = None,
        metadata: Optional[dict] = None
    ) -> dict[str, Any]:
        """
        Edit an existing category file.

        Args:
            category: Category name
            name: Display name for the item
            code: Full Python implementation code to replace existing (optional)
            patches: List of {old_string, new_string} replacements (optional, preferred for small changes)
            description: Updated description (optional, omit to keep existing)
            details: Full replacement for the details field (optional, mutually exclusive with detail_patches)
            detail_patches: List of {old_string, new_string} replacements applied to the details field (optional)
            metadata: Additional metadata updates (optional)

        Returns:
            dict with:
            - success: bool
            - path: str - path to the implementation file
            - validation: dict - results from test harness (if code updated)
            - error: str (if any)
        """
        if code is not None and patches is not None:
            return {"success": False, "error": "Provide either 'code' or 'patches', not both"}
        if details is not None and detail_patches is not None:
            return {"success": False, "error": "Provide either 'details' or 'detail_patches', not both"}

        try:
            cat = Category(category)
        except ValueError:
            return {"success": False, "error": f"Invalid category '{category}'"}

        item_dir = get_category_path(self.src_dir, cat, name)

        if not item_dir.exists():
            return {"success": False, "error": f"Item '{name}' does not exist in category '{category}'"}

        impl_path = item_dir / "implementation.py"
        meta_path = item_dir / "metadata.json"

        # Load existing metadata
        try:
            existing_meta = {}
            if meta_path.exists():
                existing_meta = json.loads(meta_path.read_text())
        except Exception as e:
            return {"success": False, "error": f"Failed to read existing metadata: {e}"}

        # Load existing details from details.md; migrate from metadata.json if needed
        details_path = item_dir / "details.md"
        existing_details = ""
        if details_path.exists():
            existing_details = details_path.read_text()
        elif existing_meta.get("details"):
            existing_details = existing_meta.pop("details")
            try:
                details_path.write_text(existing_details)
            except Exception:
                pass  # migration failure is non-fatal

        # Apply string-replacement patches if provided
        if patches is not None:
            if not impl_path.exists():
                return {"success": False, "error": "Cannot patch: implementation file does not exist"}
            try:
                current_code = impl_path.read_text()
                for i, patch in enumerate(patches):
                    old = patch.get("old_string", "")
                    new = patch.get("new_string", "")
                    if old not in current_code:
                        return {"success": False, "error": f"Patch {i}: old_string not found in file"}
                    if current_code.count(old) > 1:
                        return {"success": False, "error": f"Patch {i}: old_string is not unique — add more surrounding context"}
                    current_code = current_code.replace(old, new, 1)
                impl_path.write_text(current_code)
                log.info(f"Applied {len(patches)} patch(es) to {impl_path}")
                code = current_code  # trigger validation below
            except Exception as e:
                return {"success": False, "error": f"Failed to apply patches: {e}"}

        # Update code if provided (full replace)
        if code is not None and patches is None:
            try:
                impl_path.write_text(code)
                log.info(f"Updated {cat.value} implementation: {impl_path}")
            except Exception as e:
                return {"success": False, "error": f"Failed to write implementation: {e}"}

        # Apply text-replacement patches to details field if provided
        if detail_patches is not None:
            current_details = existing_details
            for i, patch in enumerate(detail_patches):
                old = patch.get("old_string", "")
                new = patch.get("new_string", "")
                if old not in current_details:
                    return {"success": False, "error": f"Detail patch {i}: old_string not found in details"}
                if current_details.count(old) > 1:
                    return {"success": False, "error": f"Detail patch {i}: old_string is not unique — add more surrounding context"}
                current_details = current_details.replace(old, new, 1)
            details = current_details

        # Write details.md if details was updated
        if details is not None:
            try:
                details_path.write_text(details)
                log.info(f"Updated details.md: {details_path}")
            except Exception as e:
                return {"success": False, "error": f"Failed to write details: {e}"}

        # Update metadata (details always stored in details.md, never in metadata)
        updated_meta = existing_meta.copy()
        updated_meta.pop("details", None)
        if description is not None:
            updated_meta["description"] = description
        if metadata:
            meta_updates = {k: v for k, v in metadata.items() if k != "details"}
            updated_meta.update(meta_updates)

        # Validate and write metadata
        try:
            metadata_class = METADATA_CLASSES[cat]
            meta_obj = metadata_class(**updated_meta)
            validated_meta = asdict(meta_obj)

            meta_path.write_text(json.dumps(validated_meta, indent=2))
            log.info(f"Updated metadata: {meta_path}")
        except Exception as e:
            return {"success": False, "error": f"Failed to write metadata: {e}"}

        # Run validation harness if code was updated
        validation = None
        if code is not None:
            validation = await self._validate(cat, item_dir)

        result = {
            "success": True,
            "path": str(impl_path),
        }
        if validation:
            result["validation"] = validation
            result["success"] = validation["success"]

        # Auto-execute after successful edit to give the agent immediate runtime feedback
        if code is not None and result["success"]:
            if cat == Category.RESEARCH:
                log.info(f"Auto-executing research script after edit: {name}")
                result["execution"] = await self.execute_research(name, commit=False)
            elif cat == Category.INDICATOR:
                log.info(f"Auto-executing indicator test after edit: {name}")
                result["execution"] = await self._execute_indicator(item_dir)

        # Commit to git if code changed
        if code is not None and result["success"]:
            action = "patch" if patches is not None else "edit"
            commit_hash = await self.git.commit_async(f"{action}({category}): {name}")
            if commit_hash:
                result["revision"] = commit_hash

        return result

    def read(
        self,
        category: str,
        name: str
    ) -> dict[str, Any]:
        """
        Read a category file and its metadata.

        Returns:
            dict with:
            - exists: bool
            - code: str - implementation code
            - metadata: dict - metadata
            - error: str (if any)
        """
        try:
            cat = Category(category)
        except ValueError:
            return {"exists": False, "error": f"Invalid category '{category}'"}

        item_dir = get_category_path(self.src_dir, cat, name)

        if not item_dir.exists():
            return {"exists": False}

        impl_path = item_dir / "implementation.py"
        meta_path = item_dir / "metadata.json"

        try:
            code = impl_path.read_text() if impl_path.exists() else ""
            metadata = {}
            if meta_path.exists():
                metadata = json.loads(meta_path.read_text())

            # Read details from details.md; migrate from metadata.json if needed
            details_path = item_dir / "details.md"
            details = ""
            if details_path.exists():
                details = details_path.read_text()
            elif metadata.get("details"):
                details = metadata.pop("details")
                try:
                    details_path.write_text(details)
                    meta_path.write_text(json.dumps(metadata, indent=2))
                    log.info(f"Migrated details to details.md for {item_dir.name}")
                except Exception:
                    pass  # migration failure is non-fatal

            return {
                "exists": True,
                "code": code,
                "details": details,
                "metadata": metadata,
            }
        except Exception as e:
            return {"exists": False, "error": str(e)}

    def list_items(self, category: str) -> dict[str, Any]:
        """
        List all items in a category with names and descriptions.

        Returns:
            dict with:
            - items: list of dicts with name, description, and full metadata
            - error: str (if any)
        """
        try:
            cat = Category(category)
        except ValueError:
            return {"error": f"Invalid category '{category}'"}

        cat_dir = self.src_dir / cat.value
        items = []

        for item_dir in cat_dir.iterdir():
            if not item_dir.is_dir():
                continue

            meta_path = item_dir / "metadata.json"
            if meta_path.exists():
                try:
                    metadata = json.loads(meta_path.read_text())
                    items.append({
                        "name": metadata.get("name", item_dir.name),
                        "description": metadata.get("description", ""),
                        "metadata": metadata,
                    })
                except Exception as e:
                    log.error(f"Failed to read metadata for {item_dir}: {e}")

        return {"items": items}

    async def _validate(self, category: Category, item_dir: Path) -> dict[str, Any]:
        """
        Run category-specific validation harness.

        Returns:
            dict with:
            - success: bool
            - output: str - stdout/stderr from validation
            - images: list[dict] - base64-encoded images (for research)
            - packages_installed: list[str] - packages that were installed
            - error: str (if any)
        """
        impl_path = item_dir / "implementation.py"
        meta_path = item_dir / "metadata.json"

        # Install required packages before validation
        packages_installed = []
        if install_packages_async and meta_path.exists():
            try:
                metadata = json.loads(meta_path.read_text())
                conda_packages = metadata.get("conda_packages", [])
                if conda_packages:
                    log.info(f"Installing packages for validation: {conda_packages}")
                    install_result = await install_packages_async(conda_packages, data_dir=self.data_dir)
                    if install_result.get("success"):
                        packages_installed = install_result.get("installed", [])
                        if packages_installed:
                            log.info(f"Installed packages: {packages_installed}")
                    else:
                        log.warning(f"Failed to install packages: {install_result.get('error')}")
            except Exception as e:
                log.error(f"Error installing packages: {e}")

        # Run validation
        if category == Category.STRATEGY:
            result = await self._validate_strategy(impl_path)
        elif category == Category.INDICATOR:
            result = await self._validate_indicator(impl_path)
        elif category == Category.RESEARCH:
            result = await self._validate_research(impl_path, item_dir)
        else:
            result = {"success": False, "error": f"No validator for category {category}"}

        # Add package installation info
        if packages_installed:
            result["packages_installed"] = packages_installed

        return result

    async def _validate_strategy(self, impl_path: Path) -> dict[str, Any]:
        """
        Validate a strategy by running it against synthetic OHLC data.

        Runs strategy_harness.py as a subprocess. Catches import errors,
        runtime errors in evaluate(), and wrong class hierarchy — not just syntax.
        """
        return await self._execute_strategy(impl_path.parent, timeout=45)

    async def _execute_strategy(self, item_dir: Path, timeout: int = 45) -> dict[str, Any]:
        """
        Run a strategy against synthetic OHLC data via strategy_harness.py subprocess.

        Returns:
            dict with success, output (human-readable summary), trade_count, error
        """
        impl_path = item_dir / "implementation.py"
        meta_path = item_dir / "metadata.json"

        if not impl_path.exists():
            return {"success": False, "error": "implementation.py not found"}
        if not meta_path.exists():
            return {"success": False, "error": "metadata.json not found"}

        data = await run_subprocess_argv(
            sys.executable, str(_STRATEGY_HARNESS), str(impl_path), str(meta_path),
            timeout=timeout,
        )
        if data.get("_timeout"):
            return {"success": False, "error": f"Strategy test timed out after {timeout}s"}
        if data.get("error") and not data.get("success"):
            return {"success": False, "error": data.get("stderr") or "Harness failed"}
        return data

    async def _validate_indicator(self, impl_path: Path) -> dict[str, Any]:
        """
        Validate an indicator by running it against synthetic OHLC data.

        Runs indicator_harness.py in-process via a thread (main proc). Catches
        import errors, runtime errors, and wrong return types — not just syntax.
        """
        return await self._execute_indicator(impl_path.parent, timeout=30)

    async def _execute_indicator(self, item_dir: Path, timeout: int = 30) -> dict[str, Any]:
        """
        Run an indicator against synthetic OHLC data in-process via a thread.

        Returns:
            dict with success, output (human-readable summary), error
        """
        impl_path = item_dir / "implementation.py"
        meta_path = item_dir / "metadata.json"

        if not impl_path.exists():
            return {"success": False, "error": "implementation.py not found"}
        if not meta_path.exists():
            return {"success": False, "error": "metadata.json not found"}

        from dexorder.tools.indicator_harness import run as _indicator_run
        result = await run_in_thread(_indicator_run, impl_path, meta_path, timeout=timeout)

        if result.get("_timeout"):
            return {"success": False, "error": f"Indicator test timed out after {timeout}s"}
        return result

    async def _run_research_harness(self, impl_path: Path, item_dir: Path, timeout: int = 300) -> dict[str, Any]:
        """
        Run a research script via research_harness.py subprocess and return captured results.

        Returns:
            dict with stdout, stderr, images, error fields.
        """
        return await run_subprocess_argv(
            sys.executable, str(_RESEARCH_HARNESS), str(impl_path),
            timeout=timeout,
            cwd=item_dir,
        )

    async def _validate_research(self, impl_path: Path, item_dir: Path) -> dict[str, Any]:
        """
        Validate a research script.

        Runs the script via the harness and captures output + pyplot images.
        """
        data = await self._run_research_harness(impl_path, item_dir, timeout=300)

        if data.get("_timeout"):
            return {"success": False, "error": "Research script timeout"}

        if data["error"]:
            return {
                "success": False,
                "output": data["stderr"],
                "error": "Research script execution failed",
            }

        return {
            "success": True,
            "output": data["stdout"],
            "images": data["images"],
        }

    async def execute_research(self, name: str, commit: bool = True) -> dict[str, Any]:
        """
        Execute a research script and return structured content with images.

        Args:
            name: Display name of the research script
            commit: Whether to commit output files to git (default True; set False
                    when called from write()/edit() which commit everything together)

        Returns:
            dict with:
            - content: list of TextContent and ImageContent objects (MCP format)
            - error: str (if any)
        """
        item_dir = get_category_path(self.src_dir, Category.RESEARCH, name)

        if not item_dir.exists():
            return {"error": f"Research script '{name}' does not exist"}

        impl_path = item_dir / "implementation.py"
        if not impl_path.exists():
            return {"error": f"Implementation file not found for '{name}'"}

        data = await self._run_research_harness(impl_path, item_dir, timeout=300)

        if data.get("_timeout"):
            log.error(f"execute_research '{name}': timeout")
            return {"error": "Research script timeout (5 minutes exceeded)"}

        log.info(
            f"execute_research '{name}': script_error={data.get('error')}, "
            f"stdout_len={len(data.get('stdout', ''))}, "
            f"stderr_len={len(data.get('stderr', ''))}, "
            f"image_count={len(data.get('images', []))}"
        )
        if data.get("stderr"):
            log.warning(f"execute_research '{name}' stderr: {data['stderr'][:500]}")

        # Build MCP structured content
        from mcp.types import TextContent, ImageContent

        content = []

        # Add text output (stdout/stderr combined)
        text_parts = []
        if data["stdout"]:
            text_parts.append(f"stdout:\n{data['stdout']}")
        if data["stderr"]:
            text_parts.append(f"stderr:\n{data['stderr']}")

        if text_parts:
            content.append(
                TextContent(type="text", text="\n\n".join(text_parts))
            )

        # Add images
        for img in data["images"]:
            content.append(
                ImageContent(
                    type="image",
                    data=img["data"],
                    mimeType="image/png"
                )
            )

        # If there was an error but we still got output, include error flag
        if data.get("error") and not content:
            log.error(f"execute_research '{name}': script failed with no output")
            return {"error": "Research script execution failed"}

        # Persist output to output/ subdir
        if content:
            output_dir = item_dir / "output"
            output_dir.mkdir(exist_ok=True)
            output_meta: dict[str, Any] = {}

            if data.get("stdout"):
                analysis_path = output_dir / "analysis.md"
                try:
                    analysis_path.write_text(data["stdout"])
                    output_meta["analysis"] = "analysis.md"
                except Exception as e:
                    log.warning(f"execute_research '{name}': failed to write analysis.md: {e}")

            image_files = []
            for i, img in enumerate(data.get("images", []), 1):
                img_filename = f"img{i}.png"
                img_path = output_dir / img_filename
                try:
                    img_path.write_bytes(base64.b64decode(img["data"]))
                    image_files.append(img_filename)
                except Exception as e:
                    log.warning(f"execute_research '{name}': failed to write {img_filename}: {e}")

            if image_files:
                output_meta["images"] = image_files

            # Update metadata.json with output section
            meta_path = item_dir / "metadata.json"
            if meta_path.exists():
                try:
                    meta = json.loads(meta_path.read_text())
                    meta["output"] = output_meta
                    meta_path.write_text(json.dumps(meta, indent=2))
                except Exception as e:
                    log.warning(f"execute_research '{name}': failed to update metadata output: {e}")

            # Commit output files
            if commit:
                try:
                    await self.git.commit_async(f"output(research): {name}")
                except Exception as e:
                    log.warning(f"execute_research '{name}': git commit failed: {e}")

        log.info(f"execute_research '{name}': returning {len(content)} content items")
        return {"content": content}


    def read_output(
        self,
        category: str,
        name: str,
        files: Optional[list[str]] = None
    ) -> dict[str, Any]:
        """
        Read output files for a category item.

        Args:
            category: Category name
            name: Display name of the item
            files: Specific filenames under output/ to return (e.g. ["analysis.md", "img1.png"]).
                   If omitted, returns all output files listed in metadata.

        Returns:
            dict with:
            - content: list of TextContent and ImageContent objects (MCP format)
            - files_returned: list of filenames returned
            - output_dir: str path to output directory
            - error: str (if any)
        """
        try:
            cat = Category(category)
        except ValueError:
            return {"error": f"Invalid category '{category}'"}

        item_dir = get_category_path(self.src_dir, cat, name)
        if not item_dir.exists():
            return {"error": f"Item '{name}' not found in '{category}'"}

        output_dir = item_dir / "output"
        if not output_dir.exists():
            return {"error": f"No output directory for '{name}' — run the script first"}

        # Determine which files to return
        if files is None:
            meta_path = item_dir / "metadata.json"
            if meta_path.exists():
                try:
                    meta = json.loads(meta_path.read_text())
                    output_meta = meta.get("output") or {}
                    files = []
                    if output_meta.get("analysis"):
                        files.append(output_meta["analysis"])
                    files.extend(output_meta.get("images") or [])
                except Exception:
                    files = []
            if not files:
                # Fallback: return all files in output dir
                files = [p.name for p in sorted(output_dir.iterdir()) if p.is_file()]

        if not files:
            return {"error": f"No output files found for '{name}'"}

        from mcp.types import TextContent, ImageContent

        content = []
        files_returned = []

        for filename in files:
            file_path = output_dir / filename
            if not file_path.exists():
                log.warning(f"Output file not found: {file_path}")
                continue

            suffix = file_path.suffix.lower()
            if suffix in ('.md', '.txt'):
                text = file_path.read_text()
                content.append(TextContent(type="text", text=text))
                files_returned.append(filename)
            elif suffix in ('.png', '.jpg', '.jpeg'):
                data = base64.b64encode(file_path.read_bytes()).decode()
                mime = "image/png" if suffix == '.png' else "image/jpeg"
                content.append(ImageContent(type="image", data=data, mimeType=mime))
                files_returned.append(filename)
            else:
                log.warning(f"Unsupported output file type: {filename}")

        return {
            "content": content,
            "files_returned": files_returned,
            "output_dir": str(output_dir),
        }

    async def delete(self, category: str, name: str) -> dict[str, Any]:
        """
        Delete a category script directory and commit the removal to git.

        Args:
            category: Category name (strategy, indicator, research)
            name: Display name of the item to delete

        Returns:
            dict with:
            - success: bool
            - category: str
            - name: str
            - revision: str - git commit hash of the deletion commit
            - error: str (if any)
        """
        import shutil

        try:
            cat = Category(category)
        except ValueError:
            return {
                "success": False,
                "error": f"Invalid category '{category}'. Must be one of: {', '.join(c.value for c in Category)}"
            }

        item_dir = get_category_path(self.src_dir, cat, name)
        if not item_dir.exists():
            return {"success": False, "error": f"{category} '{name}' not found"}

        try:
            shutil.rmtree(item_dir)
            log.info(f"Deleted {cat.value}: {item_dir}")
        except Exception as e:
            return {"success": False, "error": f"Failed to delete: {e}"}

        commit_hash = await self.git.commit_async(f"delete({category}): {name}")
        result: dict[str, Any] = {"success": True, "category": category, "name": name}
        if commit_hash:
            result["revision"] = commit_hash
        return result

    async def git_log(
        self,
        category: Optional[str] = None,
        name: Optional[str] = None,
        limit: int = 20
    ) -> dict[str, Any]:
        """
        List recent git commits, optionally filtered to a category or item.

        Returns:
            dict with:
            - success: bool
            - commits: list of {hash, short_hash, message, date}
        """
        path = None
        if category:
            try:
                cat = Category(category)
            except ValueError:
                return {"success": False, "error": f"Invalid category '{category}'"}
            if name:
                path = get_category_path(self.src_dir, cat, name)
            else:
                path = self.src_dir / cat.value
        entries = await self.git.log_async(path=path, n=limit)
        return {"success": True, "commits": entries}

    async def git_revert(self, revision: str, category: str, name: str) -> dict[str, Any]:
        """
        Restore a category item to a previous git revision (creates a new commit).

        Returns:
            dict with:
            - success: bool
            - revision: str - new commit hash
            - validation: dict
            - error: str (if any)
        """
        try:
            cat = Category(category)
        except ValueError:
            return {"success": False, "error": f"Invalid category '{category}'"}

        item_dir = get_category_path(self.src_dir, cat, name)
        if not item_dir.exists():
            return {"success": False, "error": f"Item '{name}' not found in '{category}'"}

        try:
            commit_hash = await self.git.restore_async(revision, path=item_dir)
        except RuntimeError as e:
            return {"success": False, "error": str(e)}

        validation = await self._validate(cat, item_dir)
        return {
            "success": validation["success"],
            "revision": commit_hash,
            "validation": validation,
        }


# =============================================================================
# Global Manager Instance
# =============================================================================

_category_manager: Optional[CategoryFileManager] = None


def get_category_manager(data_dir: Optional[Path] = None) -> CategoryFileManager:
    """Get or create the global category file manager."""
    global _category_manager
    if _category_manager is None:
        if data_dir is None:
            raise ValueError("data_dir required for first initialization")
        _category_manager = CategoryFileManager(data_dir)
    return _category_manager
