kamidso-back/kamidso/native_connectors/registry.py

"""Load and cache predefined native connector catalogs."""

from __future__ import annotations

import copy
import json
import logging
from pathlib import Path
from typing import Dict, List, Optional
from urllib.parse import urlparse

from kamidso.native_connectors.errors import CatalogError, ProviderNotFound
from kamidso.native_connectors.flags import (
    didar_connector_enabled,
    is_production,
    native_connectors_v1_enabled,
)
from kamidso.native_connectors.models import ConnectorFunction, NativeProvider
from kamidso.native_connectors.validator import (
    check_input_schema,
    clear_validator_cache,
)

logger = logging.getLogger(__name__)

_ALLOWED_METHODS = {"GET", "POST", "PUT", "PATCH", "DELETE", "HEAD"}
_ALLOWED_EFFECTS = {"read", "write", "delete"}
_GUIDANCE_LIMIT = 8000
_REGISTRY: Optional["NativeConnectorRegistry"] = None


def _read_json(path: Path) -> object:
    try:
        return json.loads(path.read_text(encoding="utf-8"))
    except (OSError, json.JSONDecodeError) as exc:
        raise CatalogError(f"Cannot read {path.name}: {exc}") from exc


def function_from_raw(raw: Dict[str, object]) -> ConnectorFunction:
    transport = raw.get("transport") if isinstance(raw.get("transport"), dict) else {}
    method = str(transport.get("method") or raw.get("method") or "").upper()
    path = str(transport.get("path") or raw.get("path") or "")
    response = raw.get("response") if isinstance(raw.get("response"), dict) else {}
    root = str(response.get("root") or raw.get("response_root") or "Response")
    records_path = response.get("records_path", raw.get("records_path"))
    if records_path is not None:
        records_path = str(records_path) or None
    summary = response.get("summary_fields") or raw.get("summary_fields") or []
    pagination = raw.get("pagination") if isinstance(raw.get("pagination"), dict) else None
    if records_path is None and pagination and pagination.get("type") == "offset_limit":
        records_path = f"{root}.List"
    hints = raw.get("semantic_hints") or []
    schema = raw.get("input_schema") if isinstance(raw.get("input_schema"), dict) else {}
    return ConnectorFunction(
        name=str(raw.get("name") or ""),
        title=str(raw.get("title") or raw.get("name") or ""),
        description=str(raw.get("description") or ""),
        domain=str(raw.get("domain") or raw.get("entity") or ""),
        effect=str(raw.get("effect") or ""),
        enabled=bool(raw.get("enabled")),
        method=method,
        path=path,
        input_schema=schema,
        response_root=root,
        records_path=records_path,
        summary_fields=[str(item) for item in summary] if isinstance(summary, list) else [],
        semantic_hints=[str(item) for item in hints] if isinstance(hints, list) else [],
        pagination=pagination,
        content_type=str(raw.get("content_type") or "application/json"),
        timeout_seconds=int(raw.get("timeout_seconds") or 30),
    )


def validate_provider(provider: NativeProvider) -> None:
    parsed = urlparse(provider.base_url)
    if parsed.scheme != "https" or not parsed.hostname:
        raise CatalogError(f"Provider '{provider.id}' base_url must be https with a host.")
    if provider.mode not in {"read_only", "read_write"}:
        raise CatalogError(f"Provider '{provider.id}' has unknown mode '{provider.mode}'.")
    if not provider.credential_field or not provider.auth_parameter:
        raise CatalogError(f"Provider '{provider.id}' is missing authentication fields.")
    seen = set()
    for function in provider.functions:
        if not function.name or function.name in seen:
            raise CatalogError(
                f"Provider '{provider.id}' has a duplicate or empty function name "
                f"'{function.name}'."
            )
        seen.add(function.name)
        if function.effect not in _ALLOWED_EFFECTS:
            raise CatalogError(
                f"Function '{function.name}' has unknown effect '{function.effect}'."
            )
        if function.method not in _ALLOWED_METHODS:
            raise CatalogError(
                f"Function '{function.name}' has invalid method '{function.method}'."
            )
        if (
            not function.path.startswith("/")
            or "://" in function.path
            or function.path.startswith("//")
        ):
            raise CatalogError(f"Function '{function.name}' path must be relative.")
        check_input_schema(function.name, function.input_schema)
        if provider.mode == "read_only" and function.enabled and function.effect != "read":
            raise CatalogError(
                f"Function '{function.name}' is enabled but is not a read."
            )
    if provider.health_function not in seen:
        raise CatalogError(
            f"Provider '{provider.id}' health check function "
            f"'{provider.health_function}' is missing."
        )
    for alias, targets in provider.aliases.items():
        for target in targets:
            if target not in seen:
                raise CatalogError(
                    f"Alias '{alias}' points at unknown function '{target}'."
                )


def _merge_object_schema(base: Dict[str, object], incoming: Dict[str, object]) -> None:
    """Add accuracy-patch fields without replacing richer existing definitions."""
    properties = base.get("properties")
    incoming_properties = incoming.get("properties")
    if not isinstance(properties, dict):
        properties = {}
        base["properties"] = properties
    if not isinstance(incoming_properties, dict):
        return
    for key, spec in incoming_properties.items():
        if not isinstance(spec, dict):
            continue
        current = properties.get(key)
        if not isinstance(current, dict):
            properties[key] = copy.deepcopy(spec)
            continue
        if current.get("type") == "object" and spec.get("type") == "object":
            _merge_object_schema(current, spec)
            continue
        if "enum" in spec and "enum" not in current:
            current["enum"] = copy.deepcopy(spec["enum"])
        if "minimum" in spec and "minimum" not in current:
            current["minimum"] = spec["minimum"]
        if "maximum" in spec and "maximum" not in current:
            current["maximum"] = spec["maximum"]


def _apply_accuracy_patch(directory: Path, functions_raw: List[object]) -> None:
    path = directory / "function_accuracy_patch.json"
    if not path.is_file():
        return
    document = _read_json(path)
    if not isinstance(document, dict):
        return
    patches = document.get("patches")
    if not isinstance(patches, list):
        return
    by_name = {
        str(item.get("name")): item
        for item in functions_raw
        if isinstance(item, dict) and item.get("name")
    }
    for patch in patches:
        if not isinstance(patch, dict):
            continue
        current = by_name.get(str(patch.get("name") or ""))
        if not isinstance(current, dict):
            continue
        schema = patch.get("input_schema")
        existing_schema = current.get("input_schema")
        if isinstance(schema, dict) and isinstance(existing_schema, dict):
            _merge_object_schema(existing_schema, schema)
        hints = patch.get("semantic_hints")
        if isinstance(hints, list):
            existing_hints = current.get("semantic_hints")
            if not isinstance(existing_hints, list):
                existing_hints = []
                current["semantic_hints"] = existing_hints
            for hint in hints:
                if hint not in existing_hints:
                    existing_hints.append(hint)
        contract = patch.get("response_contract")
        if not isinstance(contract, dict):
            continue
        if contract.get("records_path") and not current.get("records_path"):
            current["records_path"] = str(contract["records_path"])
        aggregates = contract.get("aggregates")
        if isinstance(aggregates, list):
            fields = current.get("summary_fields")
            if not isinstance(fields, list):
                fields = []
                current["summary_fields"] = fields
            for field in aggregates:
                if field not in fields:
                    fields.append(field)


def load_provider(directory: Path) -> NativeProvider:
    manifest_path = directory / "manifest.json"
    functions_path = directory / "functions.json"
    manifest = _read_json(manifest_path)
    functions_raw = _read_json(functions_path)
    if not isinstance(manifest, dict):
        raise CatalogError(f"{directory.name} manifest must be an object.")
    if not isinstance(functions_raw, list):
        raise CatalogError(f"{directory.name} functions.json must be an array.")
    _apply_accuracy_patch(directory, functions_raw)
    auth = manifest.get("authentication") if isinstance(manifest.get("authentication"), dict) else {}
    health = manifest.get("health_check") if isinstance(manifest.get("health_check"), dict) else {}
    headers = manifest.get("default_headers") if isinstance(manifest.get("default_headers"), dict) else {}
    aliases = manifest.get("aliases") if isinstance(manifest.get("aliases"), dict) else {}
    guidance_path = directory / "guidance.md"
    guidance = ""
    if guidance_path.is_file():
        guidance = guidance_path.read_text(encoding="utf-8").strip()
    if len(guidance) > _GUIDANCE_LIMIT:
        guidance = guidance[:_GUIDANCE_LIMIT]
    functions = [
        function_from_raw(item)
        for item in functions_raw
        if isinstance(item, dict)
    ]
    if len(functions) != len(functions_raw):
        raise CatalogError(f"{directory.name} functions.json contains a non-object entry.")
    normalized_aliases: Dict[str, List[str]] = {}
    for key, value in aliases.items():
        if isinstance(value, list):
            normalized_aliases[str(key)] = [str(item) for item in value]
    return NativeProvider(
        id=str(manifest.get("id") or directory.name),
        version=str(manifest.get("version") or ""),
        display_name=str(manifest.get("display_name") or manifest.get("id") or directory.name),
        category=str(manifest.get("category") or ""),
        base_url=str(manifest.get("base_url") or "").rstrip("/"),
        auth_type=str(auth.get("type") or ""),
        auth_parameter=str(auth.get("parameter_name") or ""),
        credential_field=str(auth.get("credential_field") or ""),
        default_headers={str(key): str(value) for key, value in headers.items()},
        mode=str(manifest.get("mode") or "read_only"),
        health_function=str(health.get("function") or ""),
        health_body=health.get("body") if isinstance(health.get("body"), dict) else {},
        aliases=normalized_aliases,
        guidance=guidance,
        functions=functions,
        description=str(manifest.get("description") or ""),
    )


class NativeConnectorRegistry:
    def __init__(self, providers: Dict[str, NativeProvider]) -> None:
        self._providers = dict(providers)

    def contains(self, connector_type: Optional[str]) -> bool:
        return bool(connector_type) and connector_type in self._providers

    def get(self, connector_type: str) -> NativeProvider:
        provider = self._providers.get(connector_type)
        if provider is None:
            raise ProviderNotFound(f"Native provider '{connector_type}' is not available.")
        return provider

    def provider_ids(self) -> List[str]:
        return sorted(self._providers)

    @classmethod
    def load(cls, root: Optional[Path] = None) -> "NativeConnectorRegistry":
        clear_validator_cache()
        if not native_connectors_v1_enabled():
            return cls({})
        directory = root or Path(__file__).resolve().parent / "providers"
        providers: Dict[str, NativeProvider] = {}
        if not directory.is_dir():
            return cls(providers)
        for child in sorted(path for path in directory.iterdir() if path.is_dir()):
            if not (child / "manifest.json").is_file():
                continue
            if child.name == "didar" and not didar_connector_enabled():
                continue
            try:
                provider = load_provider(child)
                if provider.id != child.name:
                    raise CatalogError(
                        f"Provider id '{provider.id}' does not match directory '{child.name}'."
                    )
                validate_provider(provider)
            except CatalogError as exc:
                if is_production():
                    logger.critical(
                        "Disabling native provider '%s': %s", child.name, exc
                    )
                    continue
                raise
            providers[provider.id] = provider
        return cls(providers)


def get_registry() -> NativeConnectorRegistry:
    global _REGISTRY
    if _REGISTRY is None:
        _REGISTRY = NativeConnectorRegistry.load()
    return _REGISTRY


def reset_registry() -> None:
    global _REGISTRY
    _REGISTRY = None
    clear_validator_cache()
    from kamidso.native_connectors.semantic.catalog import reset_semantic_cache

    reset_semantic_cache()