kamidso-back/kamidso/engine/app/openapi/api_v1/native_dbgpt.py

"""Native DB-GPT memory/prompt wiring for the Kamidso ReAct path.

Matches F:\\DB-GPT ``agentic_data_api``: persistent GptsMemory plus the default
in-process short-term buffer. That path does not embed step text. HybridMemory
stays on the multi-agent controller only. Feature flags allow rollback.
"""

from __future__ import annotations

import logging
import os
import re
from typing import Any, Dict, Optional

from kamidso.engine.core.agent import AgentMemory, ProfileConfig
from kamidso.engine.core.agent.core.memory.gpts import GptsMemory
from kamidso.engine.core.agent.expand.react_agent import (
    _REACT_DEFAULT_GOAL,
    _REACT_SYSTEM_TEMPLATE,
    _REACT_USER_TEMPLATE,
    _REACT_WRITE_MEMORY_TEMPLATE,
)
from kamidso.engine.serve.agent.agents.db_gpts_memory import (
    MetaDbGptsMessageMemory,
    MetaDbGptsPlansMemory,
)

logger = logging.getLogger(__name__)

_MEMORY_FLAG = "KAMIDSO_USE_NATIVE_DBGPT_MEMORY"
_PROMPT_FLAG = "KAMIDSO_USE_NATIVE_DBGPT_PROMPT"

_KAMIDSO_NATIVE_IDENTITY = (
    "You are Kamidso (کامیدسو), the intelligent organizational data assistant. "
    "Always identify yourself as Kamidso, never as DB-GPT. "
    "Respond in the user's language."
)

_INDEX_PREFIX = "agent_memory_long_term_"
_CHROMA_NAME_MAX = 63


def memory_data_root() -> str:
    """Canonical writable ``data/`` root for native memory files."""
    from kamidso.engine.core.configs.model_config import (
        ROOT_PATH,
        get_kamidso_data_root,
        writable_data_root,
    )

    root = get_kamidso_data_root() or writable_data_root(ROOT_PATH)
    abs_root = os.path.abspath(root)
    os.makedirs(abs_root, exist_ok=True)
    return abs_root


def long_term_persist_dir() -> str:
    """Chroma persist dir for long-term agent memory: ``data/vectors/chromadb``."""
    path = os.path.join(memory_data_root(), "vectors", "chromadb")
    os.makedirs(path, exist_ok=True)
    return path


def op_snapshot_dir() -> str:
    """Operation snapshot dir: ``data/runtime/op_snapshots``."""
    path = os.path.join(memory_data_root(), "runtime", "op_snapshots")
    os.makedirs(path, exist_ok=True)
    return path


def engine_meta_db_path() -> str:
    """Engine SQLite file for GPTs tables, if a file DB is used."""
    path = os.path.join(memory_data_root(), "meta")
    os.makedirs(path, exist_ok=True)
    return os.path.join(path, "kamidso-engine.db")


def _env_bool(name: str, default: bool) -> bool:
    raw = os.environ.get(name)
    if raw is None:
        return default
    return raw.strip().lower() in {"1", "true", "yes", "on"}


def use_native_dbgpt_memory() -> bool:
    """Return True when ReAct memory uses persistent MetaDb plans."""
    return _env_bool(_MEMORY_FLAG, True)


def use_native_dbgpt_prompt() -> bool:
    """Return True when the native ReAct system template should be used."""
    return _env_bool(_PROMPT_FLAG, True)


def _is_development() -> bool:
    env = (os.environ.get("KAMIDSO_ENV") or "").strip().lower()
    if env in {"development", "dev", "local"}:
        return True
    return _env_bool("KAMIDSO_DEVELOPER_MODE", False)


def _sanitize_index_part(raw: str, max_len: int) -> str:
    cleaned = re.sub(r"[^a-zA-Z0-9._-]", "_", (raw or "").strip())
    cleaned = cleaned.strip("._-") or "default"
    return cleaned[:max_len]


def long_term_index_name(
    organization_id: Optional[str] = None, conv_id: Optional[str] = None
) -> str:
    """Tenant- or conversation-scoped vector collection name.

    Never returns a global unscoped index.
    """
    suffix_budget = max(8, _CHROMA_NAME_MAX - len(_INDEX_PREFIX))
    if organization_id:
        suffix = _sanitize_index_part(str(organization_id), suffix_budget)
        return f"{_INDEX_PREFIX}{suffix}"
    conv_part = _sanitize_index_part(str(conv_id or "default"), suffix_budget - 5)
    return f"{_INDEX_PREFIX}conv_{conv_part}"


def prepare_react_gpts_memory(cache: Dict[str, GptsMemory], conv_id: str) -> GptsMemory:
    """Return the process-local GptsMemory for ``conv_id``, initializing once.

    Native mode uses persistent MetaDb backends for both messages and plans.
    Legacy mode keeps in-process DefaultGptsPlansMemory + MetaDb messages.
    ``init`` runs only on cache miss. A finished Kamidso turn drops this entry
    with ``dispose_runtime``; the next turn builds a new scratch and reloads
    the needed history from MetaDB and ``kamidso_history``.
    """
    existing = cache.get(conv_id)
    if existing is not None:
        return existing
    from kamidso.engine.core.agent.core.memory.gpts import DefaultGptsPlansMemory

    if use_native_dbgpt_memory():
        gpt_memory = GptsMemory(
            plans_memory=MetaDbGptsPlansMemory(),
            message_memory=MetaDbGptsMessageMemory(),
        )
        logger.info("[memory] native DB-GPT memory enabled")
        logger.info("[memory] conversation_id=%s", conv_id)
        logger.info("[memory] gpts message memory=MetaDbGptsMessageMemory")
        logger.info("[memory] plans memory=MetaDbGptsPlansMemory")
    else:
        gpt_memory = GptsMemory(
            plans_memory=DefaultGptsPlansMemory(),
            message_memory=MetaDbGptsMessageMemory(),
        )
        logger.info("[memory] legacy Kamidso memory path enabled")
        logger.info("[memory] conversation_id=%s", conv_id)
        logger.info("[memory] gpts message memory=MetaDbGptsMessageMemory")
        logger.info("[memory] plans memory=DefaultGptsPlansMemory")

    gpt_memory.init(conv_id, enable_vis_message=False)
    cache[conv_id] = gpt_memory
    return gpt_memory


def build_native_dbgpt_agent_memory(
    gpt_memory: GptsMemory,
    system_app: Any,
    conv_id: str,
    organization_id: Optional[str] = None,
) -> AgentMemory:
    """Build the ReAct AgentMemory used by DB-GPT's data chat.

    ``agentic_data_api`` constructs ``AgentMemory(gpts_memory=gpt_memory)``.
    The default inner store is an in-process short-term buffer. It keeps the
    recent step text and does not call the embeddings API. Vector HybridMemory
    is not part of this path.
    """
    del system_app, organization_id
    logger.info("[memory] react memory=AgentMemory short-term buffer")
    logger.info("[memory] conversation_id=%s", conv_id)
    logger.info("[memory] step embeddings disabled")
    return AgentMemory(gpts_memory=gpt_memory)


def build_agent_memory(
    conv_id: str,
    cache: Dict[str, GptsMemory],
    system_app: Any = None,
    organization_id: Optional[str] = None,
) -> AgentMemory:
    """Construct AgentMemory for one conversation / request."""
    gpt_memory = prepare_react_gpts_memory(cache, conv_id)
    if not use_native_dbgpt_memory():
        return AgentMemory(gpts_memory=gpt_memory)
    return build_native_dbgpt_agent_memory(
        gpt_memory,
        system_app,
        conv_id,
        organization_id=organization_id,
    )


def kamidso_native_profile(*, extra_goal: str = "") -> ProfileConfig:
    """Compact identity profile that keeps the native ReAct system template."""
    goal_parts = [_KAMIDSO_NATIVE_IDENTITY, _REACT_DEFAULT_GOAL]
    extra = (extra_goal or "").strip()
    if extra:
        goal_parts.append(extra)
    return ProfileConfig(
        name="Kamidso",
        role="intelligent organizational data assistant",
        goal="\n\n".join(goal_parts),
        system_prompt_template=_REACT_SYSTEM_TEMPLATE,
        user_prompt_template=_REACT_USER_TEMPLATE,
        write_memory_template=_REACT_WRITE_MEMORY_TEMPLATE,
    )


def bind_agent_prompt(
    agent: Any,
    workflow_prompt_template: Any = None,
    *,
    extra_goal: str = "",
) -> Any:
    """Bind native ProfileConfig or the legacy Kamidso PromptTemplate."""
    if use_native_dbgpt_prompt():
        return agent.bind(kamidso_native_profile(extra_goal=extra_goal))
    if workflow_prompt_template is not None:
        return agent.bind(workflow_prompt_template)
    return agent


def log_prompt_metadata(
    *,
    conversation_id: str,
    question: Optional[str] = None,
    task_progress: Optional[str] = None,
    recent_memories_count: Optional[int] = None,
    observation_present: bool = False,
    round_num: Optional[int] = None,
) -> None:
    """Log prompt presence flags in development; never log org payload."""
    if not _is_development():
        return
    memories = (
        recent_memories_count if recent_memories_count is not None else 0
    )
    logger.info(
        "[prompt] native=%s question_present=%s task_progress_present=%s "
        "recent_memories_count=%s observation_present=%s conversation_id=%s "
        "round=%s",
        use_native_dbgpt_prompt(),
        bool(question),
        bool(task_progress),
        memories,
        bool(observation_present),
        conversation_id,
        round_num,
    )