From edbaa2acecf9acbeec77481961a31bd8fd5c86a9 Mon Sep 17 00:00:00 2001 From: javad Date: Mon, 12 Oct 2026 19:26:47 +0330 Subject: [PATCH] Add native Didar connectors so chats can query CRM data reliably. Connector calls, semantic rules, and DuckDB memory keep long runs stable, and stream retry stops chats from stalling. Co-authored-by: Cursor --- kamidso/api/routes/__init__.py | 2 + kamidso/api/routes/connectors.py | 224 + kamidso/api/routes/health.py | 17 +- kamidso/api/sse.py | 3 +- kamidso/assistant/context_builder.py | 35 +- kamidso/contracts/resource.py | 2 +- kamidso/data_graph/api/routes.py | 1 - .../app/openapi/api_v1/agentic_data_api.py | 209 +- kamidso/engine/app/openapi/api_v1/api_v1.py | 2 +- .../engine/app/openapi/api_v1/native_dbgpt.py | 4 +- .../app/openapi/api_v1/react_runtime.py | 91 + .../app/openapi/api_v1/subagent/dispatcher.py | 113 +- .../app/openapi/api_v1/subagent/history.py | 2 + .../openapi/api_v1/subagent/react_tools.py | 1 + .../subagent/tests/test_dispatch_tool.py | 6 +- .../api_v1/subagent/tests/test_dispatcher.py | 73 + .../app/openapi/api_v1/tools/__init__.py | 6 +- .../openapi/api_v1/tools/code_interpreter.py | 8 + .../openapi/api_v1/tools/connector_call.py | 202 + kamidso/engine/app/openapi/api_v2.py | 2 +- .../chat_data/chat_excel/excel_reader.py | 25 +- .../agent/core/memory/gpts/gpts_memory.py | 176 +- .../agent/resource/connector/credential.py | 9 + .../engine/core/datasource/duckdb_memory.py | 223 + .../engine/core/model/proxy/llms/chatgpt.py | 122 +- .../proxy/tests/test_chatgpt_stream_retry.py | 47 + .../engine/core/model/utils/chatgpt_utils.py | 19 +- .../model/utils/tests/test_chatgpt_httpx.py | 23 +- .../ext/datasource/rdbms/conn_duckdb.py | 6 + .../engine/serve/agent/agents/controller.py | 38 +- .../engine/serve/connector/api/endpoints.py | 43 + .../engine/serve/connector/service/service.py | 240 +- .../serve/connector/tests/test_service.py | 157 + kamidso/integrations/engine/engine.py | 44 +- kamidso/integrations/engine/events.py | 114 +- kamidso/native_connectors/__init__.py | 5 + kamidso/native_connectors/errors.py | 144 + kamidso/native_connectors/executor.py | 482 ++ kamidso/native_connectors/flags.py | 21 + kamidso/native_connectors/models.py | 59 + kamidso/native_connectors/prompt.py | 110 + .../providers/didar/calibration_schema.json | 94 + .../providers/didar/date_rules.json | 93 + .../providers/didar/enums.json | 67 + .../didar/function_accuracy_patch.json | 398 ++ .../providers/didar/functions.json | 5102 +++++++++++++++++ .../didar/golden_reconciliation_template.json | 43 + .../providers/didar/guidance.md | 43 + .../providers/didar/manifest.json | 26 + .../providers/didar/money_rules.json | 48 + .../providers/didar/reports.json | 260 + .../didar/semantic_contract_tests.json | 94 + .../providers/didar/semantics.json | 112 + .../providers/didar/sources.json | 57 + kamidso/native_connectors/redact.py | 42 + kamidso/native_connectors/registry.py | 311 + kamidso/native_connectors/resolve.py | 71 + kamidso/native_connectors/result_store.py | 180 + kamidso/native_connectors/runtime/__init__.py | 0 kamidso/native_connectors/runtime/attached.py | 12 + .../native_connectors/semantic/__init__.py | 1 + .../native_connectors/semantic/aggregates.py | 35 + kamidso/native_connectors/semantic/catalog.py | 174 + kamidso/native_connectors/semantic/dates.py | 236 + .../native_connectors/semantic/executor.py | 782 +++ kamidso/native_connectors/semantic/golden.py | 29 + kamidso/native_connectors/semantic/money.py | 122 + kamidso/native_connectors/semantic/status.py | 20 + kamidso/native_connectors/semantic/text.py | 12 + kamidso/native_connectors/ux.py | 68 + kamidso/native_connectors/validator.py | 127 + kamidso/resources/registry.py | 24 + kamidso/storage/layout.py | 7 +- pyproject.toml | 1 + requirements.txt | 1 + tests/kamidso/test_connector_product.py | 74 + tests/kamidso/test_context_builder.py | 26 + tests/kamidso/test_data_layout.py | 7 +- tests/kamidso/test_didar_semantics.py | 585 ++ tests/kamidso/test_memory_lifecycle.py | 647 +++ tests/kamidso/test_native_connector.py | 482 ++ tests/kamidso/test_sse_keepalive.py | 7 +- 82 files changed, 13427 insertions(+), 203 deletions(-) create mode 100644 kamidso/api/routes/connectors.py create mode 100644 kamidso/engine/app/openapi/api_v1/react_runtime.py create mode 100644 kamidso/engine/app/openapi/api_v1/tools/connector_call.py create mode 100644 kamidso/engine/core/datasource/duckdb_memory.py create mode 100644 kamidso/native_connectors/__init__.py create mode 100644 kamidso/native_connectors/errors.py create mode 100644 kamidso/native_connectors/executor.py create mode 100644 kamidso/native_connectors/flags.py create mode 100644 kamidso/native_connectors/models.py create mode 100644 kamidso/native_connectors/prompt.py create mode 100644 kamidso/native_connectors/providers/didar/calibration_schema.json create mode 100644 kamidso/native_connectors/providers/didar/date_rules.json create mode 100644 kamidso/native_connectors/providers/didar/enums.json create mode 100644 kamidso/native_connectors/providers/didar/function_accuracy_patch.json create mode 100644 kamidso/native_connectors/providers/didar/functions.json create mode 100644 kamidso/native_connectors/providers/didar/golden_reconciliation_template.json create mode 100644 kamidso/native_connectors/providers/didar/guidance.md create mode 100644 kamidso/native_connectors/providers/didar/manifest.json create mode 100644 kamidso/native_connectors/providers/didar/money_rules.json create mode 100644 kamidso/native_connectors/providers/didar/reports.json create mode 100644 kamidso/native_connectors/providers/didar/semantic_contract_tests.json create mode 100644 kamidso/native_connectors/providers/didar/semantics.json create mode 100644 kamidso/native_connectors/providers/didar/sources.json create mode 100644 kamidso/native_connectors/redact.py create mode 100644 kamidso/native_connectors/registry.py create mode 100644 kamidso/native_connectors/resolve.py create mode 100644 kamidso/native_connectors/result_store.py create mode 100644 kamidso/native_connectors/runtime/__init__.py create mode 100644 kamidso/native_connectors/runtime/attached.py create mode 100644 kamidso/native_connectors/semantic/__init__.py create mode 100644 kamidso/native_connectors/semantic/aggregates.py create mode 100644 kamidso/native_connectors/semantic/catalog.py create mode 100644 kamidso/native_connectors/semantic/dates.py create mode 100644 kamidso/native_connectors/semantic/executor.py create mode 100644 kamidso/native_connectors/semantic/golden.py create mode 100644 kamidso/native_connectors/semantic/money.py create mode 100644 kamidso/native_connectors/semantic/status.py create mode 100644 kamidso/native_connectors/semantic/text.py create mode 100644 kamidso/native_connectors/ux.py create mode 100644 kamidso/native_connectors/validator.py create mode 100644 tests/kamidso/test_connector_product.py create mode 100644 tests/kamidso/test_didar_semantics.py create mode 100644 tests/kamidso/test_memory_lifecycle.py create mode 100644 tests/kamidso/test_native_connector.py diff --git a/kamidso/api/routes/__init__.py b/kamidso/api/routes/__init__.py index 282fc80..0280255 100644 --- a/kamidso/api/routes/__init__.py +++ b/kamidso/api/routes/__init__.py @@ -5,6 +5,7 @@ from fastapi import FastAPI from kamidso.api.routes import ( auth, chat, + connectors, conversations, datasources, files, @@ -26,6 +27,7 @@ def register_routes(app: FastAPI, *, prefix: str) -> None: app.include_router(runs.router, prefix=prefix, tags=["runs"]) app.include_router(files.router, prefix=prefix, tags=["files"]) app.include_router(datasources.router, prefix=prefix, tags=["datasources"]) + app.include_router(connectors.router, prefix=prefix, tags=["connectors"]) app.include_router(knowledge.router, prefix=prefix, tags=["knowledge"]) app.include_router(resources.router, prefix=prefix, tags=["resources"]) app.include_router(organization.router, prefix=prefix, tags=["organization"]) diff --git a/kamidso/api/routes/connectors.py b/kamidso/api/routes/connectors.py new file mode 100644 index 0000000..45dd431 --- /dev/null +++ b/kamidso/api/routes/connectors.py @@ -0,0 +1,224 @@ +"""Product API for predefined native connectors. Users cannot define providers.""" + +from __future__ import annotations + +import logging +from typing import Any, Dict, List, Optional + +from fastapi import APIRouter, HTTPException, Request +from pydantic import BaseModel, Field + +from kamidso.api.deps import require_user +from kamidso.engine.serve.core import blocking_func_to_async +from kamidso.native_connectors.flags import ( + didar_connector_enabled, + native_connectors_v1_enabled, +) +from kamidso.native_connectors.ux import ( + connector_test_message, + public_connector, + public_provider, +) + +logger = logging.getLogger(__name__) + +router = APIRouter() + + +def _ok(data: Any) -> Dict[str, Any]: + return {"success": True, "data": data, "err_code": None, "err_msg": None} + + +def _native_enabled() -> bool: + return native_connectors_v1_enabled() and didar_connector_enabled() + + +def _service(request: Request): + from kamidso.engine.serve.connector.config import SERVE_SERVICE_COMPONENT_NAME + from kamidso.engine.serve.connector.service.service import ConnectorService + + runtime = getattr(request.app.state.kamidso, "engine_runtime", None) + system_app = getattr(runtime, "system_app", None) if runtime is not None else None + if system_app is None: + raise HTTPException(status_code=503, detail="موتور اتصال‌ها آماده نیست.") + service = system_app.get_component( + SERVE_SERVICE_COMPONENT_NAME, + ConnectorService, + default_component=None, + ) + if service is None: + raise HTTPException(status_code=503, detail="موتور اتصال‌ها آماده نیست.") + return service, system_app + + +def _owned(service, connector_id: str, user): + row = service.get_connector(connector_id) + if row is None or row.status == "deleted": + raise HTTPException(status_code=404, detail="اتصال پیدا نشد.") + if row.user_name != user.id: + raise HTTPException(status_code=404, detail="اتصال پیدا نشد.") + if user.workspace_id and row.sys_code and row.sys_code != user.workspace_id: + raise HTTPException(status_code=404, detail="اتصال پیدا نشد.") + if not service._is_native_type(row.connector_type): + raise HTTPException(status_code=404, detail="اتصال پیدا نشد.") + return row + + +class ConnectorCreateBody(BaseModel): + connector_type: str + display_name: str + credentials: Dict[str, str] = Field(default_factory=dict) + + +class ConnectorTestBody(BaseModel): + connector_type: str + credentials: Dict[str, str] = Field(default_factory=dict) + + +class ConnectorUpdateBody(BaseModel): + display_name: Optional[str] = None + credentials: Optional[Dict[str, str]] = None + + +def _probe_http(exc: Exception) -> HTTPException: + from kamidso.engine.serve.connector.service.service import ConnectorProbeFailed + + if isinstance(exc, ConnectorProbeFailed): + public = connector_test_message( + { + "success": False, + "status_code": exc.status_code, + "error_type": exc.error_type, + } + ) + return HTTPException(status_code=400, detail=public["message"]) + text = str(exc) + if text.startswith("Missing credential"): + return HTTPException(status_code=400, detail="کلید API را وارد کنید.") + if "ENCRYPT_KEY" in text: + return HTTPException( + status_code=400, + detail="ذخیرهٔ امن کلید API در این محیط پیکربندی نشده است.", + ) + if text == "Unknown native provider": + return HTTPException(status_code=404, detail="این اتصال در دسترس نیست.") + return HTTPException(status_code=400, detail="ارتباط با دیدار برقرار نشد.") + + +@router.get("/connectors/providers") +async def list_providers(request: Request): + require_user(request) + if not _native_enabled(): + return _ok([]) + from kamidso.native_connectors.registry import get_registry + + registry = get_registry() + return _ok([public_provider(registry.get(item)) for item in registry.provider_ids()]) + + +@router.get("/connectors") +async def list_user_connectors(request: Request): + user = require_user(request) + if not _native_enabled(): + return _ok([]) + service, _system = _service(request) + rows = service.list_connectors( + user_name=user.id, + sys_code=user.workspace_id or None, + ) + visible: List[Dict[str, Any]] = [] + for row in rows: + if row.status == "deleted" or not service._is_native_type(row.connector_type): + continue + visible.append(public_connector(row)) + return _ok(visible) + + +@router.post("/connectors/test") +async def test_new_connector(body: ConnectorTestBody, request: Request): + require_user(request) + if not _native_enabled(): + raise HTTPException(status_code=404, detail="این اتصال در دسترس نیست.") + service, _system = _service(request) + try: + raw = await service.test_native_credentials(body.connector_type, body.credentials) + except Exception as exc: + logger.exception("Native connector probe failed") + raise _probe_http(exc) from exc + return _ok(connector_test_message(raw)) + + +@router.post("/connectors") +async def create_user_connector(body: ConnectorCreateBody, request: Request): + user = require_user(request) + if not _native_enabled(): + raise HTTPException(status_code=404, detail="این اتصال در دسترس نیست.") + from kamidso.engine.serve.connector.service.service import ConnectorCreateRequest + + service, system_app = _service(request) + if not service._is_native_type(body.connector_type): + raise HTTPException(status_code=404, detail="این اتصال در دسترس نیست.") + request_model = ConnectorCreateRequest( + connector_type=body.connector_type, + display_name=body.display_name.strip() or "دیدار CRM", + credentials=dict(body.credentials), + user_name=user.id, + sys_code=user.workspace_id or None, + ) + try: + created = await blocking_func_to_async( + system_app, service.create_connector, request_model + ) + except Exception as exc: + logger.exception("Create native connector failed") + raise _probe_http(exc) from exc + return _ok(public_connector(created)) + + +@router.patch("/connectors/{connector_id}") +async def update_user_connector( + connector_id: str, body: ConnectorUpdateBody, request: Request +): + user = require_user(request) + from kamidso.engine.serve.connector.service.service import ConnectorUpdateRequest + + service, system_app = _service(request) + _owned(service, connector_id, user) + credentials = None + if body.credentials: + filled = { + key: value + for key, value in body.credentials.items() + if str(value).strip() + } + credentials = filled or None + update = ConnectorUpdateRequest( + display_name=body.display_name.strip() if body.display_name else None, + credentials=credentials, + ) + try: + updated = await blocking_func_to_async( + system_app, service.update_connector, connector_id, update + ) + except Exception as exc: + logger.exception("Update native connector failed") + raise _probe_http(exc) from exc + return _ok(public_connector(updated)) + + +@router.delete("/connectors/{connector_id}") +async def delete_user_connector(connector_id: str, request: Request): + user = require_user(request) + service, system_app = _service(request) + _owned(service, connector_id, user) + await blocking_func_to_async(system_app, service.retire_connector, connector_id) + return _ok({"ok": True}) + + +@router.post("/connectors/{connector_id}/test") +async def test_saved_connector(connector_id: str, request: Request): + user = require_user(request) + service, _system = _service(request) + _owned(service, connector_id, user) + raw = await service.test_connection(connector_id) + return _ok(connector_test_message(raw)) diff --git a/kamidso/api/routes/health.py b/kamidso/api/routes/health.py index 77924d9..29b946f 100644 --- a/kamidso/api/routes/health.py +++ b/kamidso/api/routes/health.py @@ -5,6 +5,19 @@ from fastapi import APIRouter, Request router = APIRouter() +def _features_payload(settings) -> dict: + flags = settings.features.to_dict() + from kamidso.native_connectors.flags import ( + didar_connector_enabled, + native_connectors_v1_enabled, + ) + + flags["connectors"] = bool( + native_connectors_v1_enabled() and didar_connector_enabled() + ) + return flags + + @router.get("/health") async def health(request: Request): app = request.app.state.kamidso @@ -25,7 +38,7 @@ async def product(request: Request): "product": settings.product.name, "locale": settings.product.locale, "direction": settings.product.direction, - "features": settings.features.to_dict(), + "features": _features_payload(settings), "developer_mode": settings.developer_mode, } @@ -34,7 +47,7 @@ async def product(request: Request): async def features(request: Request): settings = request.app.state.kamidso.settings user = getattr(request.state, "kamidso_user", None) - flags = settings.features.to_dict() + flags = _features_payload(settings) role = user.role if user else "user" if role == "user" and not settings.developer_mode: flags["apps"] = False diff --git a/kamidso/api/sse.py b/kamidso/api/sse.py index 9ebef5c..522f839 100644 --- a/kamidso/api/sse.py +++ b/kamidso/api/sse.py @@ -5,9 +5,10 @@ from __future__ import annotations import asyncio from typing import AsyncIterator +# Do not send Connection. It is hop-by-hop; proxies that honor it reuse the +# socket while the body is still streaming and the next chat drops both streams. SSE_HEADERS = { "Cache-Control": "no-cache, no-transform", - "Connection": "keep-alive", "X-Accel-Buffering": "no", } diff --git a/kamidso/assistant/context_builder.py b/kamidso/assistant/context_builder.py index 756d907..cf40aff 100644 --- a/kamidso/assistant/context_builder.py +++ b/kamidso/assistant/context_builder.py @@ -74,6 +74,12 @@ class KamidsoContextBuilder: skills.append(skill) behavior = DEFAULT_BEHAVIOR + connector_ids = ext_info.get("connector_ids") + attached_connectors = ( + [str(item).strip() for item in connector_ids if str(item).strip()] + if isinstance(connector_ids, list) + else [] + ) kb_label = str(ext_info.get("knowledge_space_label") or (knowledge[0] if knowledge else "")).strip() if knowledge: behavior += ( @@ -135,13 +141,40 @@ class KamidsoContextBuilder: behavior += ( f" Optional picker hint: `{skill}` — load it only if it fits." ) - else: + elif not attached_connectors: behavior += ( "\nNo file or database is attached this turn. " "Do not reopen Excel/CSV files from earlier chats " "(data/runtime, data/uploads, previous run folders). " "If the question needs data, call the question tool and ask which file or database to use." ) + if attached_connectors: + behavior += ( + "\nA native business connector is attached. " + "Use connector_call to fetch business data. " + "Do not use code_interpreter to call the external API. " + "Use code_interpreter only after connector_call when computation, " + "dataframe operations, merging, charts, statistics or export are required. " + "Never invent connector function names. " + "On validation_error, correct the request using the returned schema " + "and retry the same function. " + "Prefer server-provided totals and aggregates. " + "Do not paginate through all records merely to count them." + ) + if datasources or files: + behavior += ( + " Didar and other native connectors use connector_call. " + "Databases use database(). Excel and CSV files use code_interpreter. " + "Join the results in code_interpreter after each source has been read." + ) + labels = ext_info.get("connector_labels") + if isinstance(labels, list) and any(str(item).strip() for item in labels): + shown = ", ".join(str(item).strip() for item in labels if str(item).strip()) + else: + shown = ", ".join(attached_connectors) + behavior += ( + f" Attached connector{'s' if len(attached_connectors) > 1 else ''}: {shown}." + ) if skill: behavior += ( f"\nThe user highlighted skill `{skill}` in the picker. " diff --git a/kamidso/contracts/resource.py b/kamidso/contracts/resource.py index 0676601..9c34ee5 100644 --- a/kamidso/contracts/resource.py +++ b/kamidso/contracts/resource.py @@ -3,7 +3,7 @@ from __future__ import annotations from dataclasses import asdict, dataclass from typing import Any, Dict, Literal, Optional -ResourceType = Literal["file", "datasource", "knowledge"] +ResourceType = Literal["file", "datasource", "knowledge", "connector"] @dataclass(frozen=True) diff --git a/kamidso/data_graph/api/routes.py b/kamidso/data_graph/api/routes.py index 22bc552..ad50067 100644 --- a/kamidso/data_graph/api/routes.py +++ b/kamidso/data_graph/api/routes.py @@ -87,7 +87,6 @@ async def list_edges(request: Request): _SSE_HEADERS = { "Cache-Control": "no-cache, no-transform", - "Connection": "keep-alive", "X-Accel-Buffering": "no", } diff --git a/kamidso/engine/app/openapi/api_v1/agentic_data_api.py b/kamidso/engine/app/openapi/api_v1/agentic_data_api.py index 644abee..06fd299 100644 --- a/kamidso/engine/app/openapi/api_v1/agentic_data_api.py +++ b/kamidso/engine/app/openapi/api_v1/agentic_data_api.py @@ -34,6 +34,13 @@ from kamidso.engine.app.openapi.api_v1.native_dbgpt import ( prepare_react_gpts_memory, # noqa: F401 — re-export for tests use_native_dbgpt_prompt, ) +from kamidso.engine.app.openapi.api_v1.react_runtime import ( + REACT_AGENT_MEMORY_CACHE, + REACT_CONVERSATION_AGENT_TASKS, + REACT_CONVERSATION_LOCKS, + acquire_conversation_run, + end_react_turn, +) from kamidso.engine.app.openapi.api_view_model import ( ConversationVo, Result, @@ -84,12 +91,6 @@ if TYPE_CHECKING: from kamidso.engine.core.agent.resource.connector.manager import ConnectorManager from kamidso.engine.core.agent.resource.tool.base import BaseTool -REACT_AGENT_MEMORY_CACHE: Dict[str, "GptsMemory"] = {} -# Serialize ReAct runs per conversation. A second message is rejected while a -# run is live; explicit cancel uses REACT_CONVERSATION_AGENT_TASKS. -REACT_CONVERSATION_LOCKS: Dict[str, asyncio.Lock] = {} -REACT_CONVERSATION_AGENT_TASKS: Dict[str, "asyncio.Task[Any]"] = {} - _FATAL_LLM_ERROR_MARKERS = ("LLMServer Generate Error", "proxy_llm_client") _REACT_SCRATCHPAD_MARKERS = ("Thought:", "Action:", "Action Input:", "Observation:") @@ -2133,64 +2134,64 @@ async def _react_agent_stream_inner( ) -> AsyncGenerator[str, None]: """Stream ReAct events while owning the lifetime of its background task.""" conv_id = getattr(dialogue, "conv_uid", None) or "" - conv_lock = _conversation_agent_lock(conv_id) if conv_id else None - if conv_lock is not None: - await conv_lock.acquire() - # Same conversation is serialized by the lock above. Do not cancel a live - # agent for another turn — the orchestrator rejects a second message while - # a run is still active. Explicit cancel uses _cancel_conversation_agent. - - agent_task_holder: List["asyncio.Task[Any]"] = [] - final_emitted = False - done_emitted = False - from kamidso.engine.app.openapi.api_v1.tools.skill_runtime import ( - bind_shared_execution, - empty_skill_execution, - reset_shared_execution, - ) - from kamidso.engine.core.reasoning_policy import bind_turn, reset_turn - - ext_info_raw = getattr(dialogue, "ext_info", None) - ext_info = ext_info_raw if isinstance(ext_info_raw, dict) else {} - reasoning_token = bind_turn(ext_info.get("reasoning_mode") or "fast") - execution_token = bind_shared_execution(empty_skill_execution()) + # Same conversation is serialized by this lock. A second message waits on + # the same lock object; the lock is removed only after every waiter exits. + # Explicit cancel uses _cancel_conversation_agent. + conv_lock = await acquire_conversation_run(conv_id) if conv_id else None try: - async for event in _react_agent_stream_impl( - dialogue, - tool_mode=tool_mode, - attachment_ctx=attachment_ctx, - agent_task_holder=agent_task_holder, - ): - event_type = _sse_event_type(event) - final_emitted = final_emitted or event_type == "final" - done_emitted = done_emitted or event_type == "done" - yield event - except Exception: - logger.exception("ReAct agent stream failed before normal completion") - if not final_emitted and not done_emitted: - yield _sse_event( - AgentFinalAnswer( - content=_stream_failure_user_message(ext_info) - ).to_sse_payload() - ) - if not done_emitted: - yield _sse_event({"type": "done"}) + agent_task_holder: List["asyncio.Task[Any]"] = [] + final_emitted = False + done_emitted = False + from kamidso.engine.app.openapi.api_v1.tools.skill_runtime import ( + bind_shared_execution, + empty_skill_execution, + reset_shared_execution, + ) + from kamidso.engine.core.reasoning_policy import bind_turn, reset_turn + + ext_info_raw = getattr(dialogue, "ext_info", None) + ext_info = ext_info_raw if isinstance(ext_info_raw, dict) else {} + reasoning_token = bind_turn(ext_info.get("reasoning_mode") or "fast") + execution_token = bind_shared_execution(empty_skill_execution()) + try: + async for event in _react_agent_stream_impl( + dialogue, + tool_mode=tool_mode, + attachment_ctx=attachment_ctx, + agent_task_holder=agent_task_holder, + ): + event_type = _sse_event_type(event) + final_emitted = final_emitted or event_type == "final" + done_emitted = done_emitted or event_type == "done" + yield event + except Exception: + logger.exception("ReAct agent stream failed before normal completion") + if not final_emitted and not done_emitted: + yield _sse_event( + AgentFinalAnswer( + content=_stream_failure_user_message(ext_info) + ).to_sse_payload() + ) + if not done_emitted: + yield _sse_event({"type": "done"}) + finally: + try: + reset_turn(reasoning_token) + except (ValueError, LookupError): + logger.debug("reasoning turn token already reset", exc_info=True) + try: + reset_shared_execution(execution_token) + except (ValueError, LookupError): + logger.debug("skill execution token already reset", exc_info=True) + if agent_task_holder: + task = agent_task_holder[0] + if conv_id and REACT_CONVERSATION_AGENT_TASKS.get(conv_id) is task: + REACT_CONVERSATION_AGENT_TASKS.pop(conv_id, None) + await _cancel_and_await_agent_task(task) finally: - try: - reset_turn(reasoning_token) - except (ValueError, LookupError): - logger.debug("reasoning turn token already reset", exc_info=True) - try: - reset_shared_execution(execution_token) - except (ValueError, LookupError): - logger.debug("skill execution token already reset", exc_info=True) - if agent_task_holder: - task = agent_task_holder[0] - if conv_id and REACT_CONVERSATION_AGENT_TASKS.get(conv_id) is task: - REACT_CONVERSATION_AGENT_TASKS.pop(conv_id, None) - await _cancel_and_await_agent_task(task) - if conv_lock is not None: - conv_lock.release() + # History persistence runs inside the stream body, before this + # finally. Scratch memory is dropped while the lock is still held. + await end_react_turn(conv_id, conv_lock) class _AgentStreamingResponse(StreamingResponse): @@ -2264,6 +2265,20 @@ async def _react_agent_stream_impl( # Connector selection (Task C): only inject user-selected connectors. connector_ids: List[str] = _parse_connector_ids(dialogue.ext_info) + mcp_connector_ids = list(connector_ids) + native_connector_instances: List[Dict[str, Any]] = [] + try: + from kamidso.native_connectors.flags import native_connectors_v1_enabled + from kamidso.native_connectors.resolve import partition_connector_ids + + if native_connectors_v1_enabled() and connector_ids: + mcp_connector_ids, native_connector_instances = partition_connector_ids( + connector_ids + ) + except Exception: + logger.exception("native connector partition failed") + mcp_connector_ids = list(connector_ids) + native_connector_instances = [] code_graph_available = False if knowledge_space: @@ -2652,6 +2667,10 @@ async def _react_agent_stream_impl( "database_connector_labels": database_connector_labels, "db_connection_failures": int(db_failure_state.get("db_connection_failures") or 0), } + if native_connector_instances: + from kamidso.native_connectors.resolve import install_native_connectors + + install_native_connectors(react_state, native_connector_instances) if db_failure_state.get("db_connection_give_up"): react_state["db_connection_give_up"] = db_failure_state["db_connection_give_up"] from kamidso.assistant.data_sketch import compose_turn_data_sketch, paths_from_ext_info @@ -3126,7 +3145,18 @@ print(json.dumps(summary, ensure_ascii=False)) react_state, sql_enabled=_this_turn_allows_sql_query(attached_dbs), database_runtime=_this_turn_uses_db_runtime(attached_dbs), + native_connector=bool(native_connector_instances), ) + connector_call_tool = None + if native_connector_instances and tool_mode != "knowledge": + from kamidso.engine.app.openapi.api_v1.tools.connector_call import ( + make_connector_call, + ) + + connector_call_tool = make_connector_call( + react_state, + user_name=dialogue.user_name, + ) shell_interpreter_tool = make_shell_interpreter(react_state) html_interpreter_tool = make_html_interpreter(react_state, DEFAULT_SKILLS_DIR) todowrite_tool = make_todowrite(_todo_list, stream_callback) @@ -3639,9 +3669,9 @@ print(json.dumps(summary, ensure_ascii=False)) _connector_manager = CFG.SYSTEM_APP.get_component( "connector_manager", _ConnectorManager, default_component=None ) - if _connector_manager is not None and connector_ids: + if _connector_manager is not None and mcp_connector_ids: connector_tool_extras, _missing = _select_connector_tools( - connector_ids, _connector_manager + mcp_connector_ids, _connector_manager ) for _mid in _missing: logger.warning( @@ -3653,7 +3683,7 @@ print(json.dumps(summary, ensure_ascii=False)) "_react_agent_stream: injected %d connector tool pack(s) " "(selected: %d)", len(connector_tool_extras), - len(connector_ids), + len(mcp_connector_ids), ) except Exception: pass # graceful degradation — connector tools are optional @@ -3686,9 +3716,19 @@ print(json.dumps(summary, ensure_ascii=False)) emit_event=_emit_subagent_event, max_parallel=_max_parallel_subagents, report_mode=report_mode, + native_connectors=native_connector_instances, + user_name=dialogue.user_name, ) _native_prompt = use_native_dbgpt_prompt() + native_section = "" + if native_connector_instances: + try: + from kamidso.native_connectors.prompt import render_native_prompt + + native_section = render_native_prompt(native_connector_instances) + except Exception: + logger.exception("native connector prompt failed") if is_skill_mode: if attached_dbs and not has_attached_files: skill_compute_rule = ( @@ -3842,8 +3882,10 @@ Never use Chinese in any user-visible text. execute_skill_script, get_skill_resource, execute_skill_script_file_tool, - code_interpreter_tool, ] + if connector_call_tool is not None: + skill_mode_tools.append(connector_call_tool) + skill_mode_tools.append(code_interpreter_tool) if has_attached_files: skill_mode_tools.append(load_file_tool) skill_mode_tools.append(shell_interpreter_tool) @@ -4044,11 +4086,17 @@ Never use Chinese in any user-visible text. execute_skill_script, get_skill_resource, execute_skill_script_file_tool, - code_interpreter_tool, - load_file_tool, - execute_analysis_tool, - shell_interpreter_tool, ] + if connector_call_tool is not None: + analysis_tools.append(connector_call_tool) + analysis_tools.extend( + [ + code_interpreter_tool, + load_file_tool, + execute_analysis_tool, + shell_interpreter_tool, + ] + ) if report_mode: analysis_tools.append(html_interpreter_tool) if expose_sql_query: @@ -4095,14 +4143,13 @@ Never use Chinese in any user-visible text. _cm = CFG.SYSTEM_APP.get_component( "connector_manager", _ConnectorManager, default_component=None ) - if _cm is not None and connector_ids: + if _cm is not None and mcp_connector_ids: _active = _cm.list_active() # Only describe connectors the user explicitly selected. - # Iterate connector_ids (not _active) so prompt order matches - # user selection order and stays consistent with _select_connector_tools. + # Iterate mcp ids so native connectors stay out of this block. _active_map = {c["connector_id"]: c for c in _active if isinstance(c, dict)} _selected = [ - _active_map[cid] for cid in connector_ids if cid in _active_map + _active_map[cid] for cid in mcp_connector_ids if cid in _active_map ] if _selected: _connector_lines = [] @@ -4211,6 +4258,9 @@ Never use Chinese in any user-visible text. Never invent the meaning of an unresolved field. """ + if native_section: + workflow_prompt += "\n\n" + native_section.replace("{", "{{").replace("}", "}}") + workflow_prompt = ( f"{workflow_prompt.rstrip()}\n\n" f"{_MULTI_TURN_PROMPT_SECTION}\n\n" @@ -4235,7 +4285,11 @@ Never use Chinese in any user-visible text. .bind(llm_config) .bind(tool_pack) ) - agent_builder = bind_agent_prompt(agent_builder, workflow_prompt_template) + agent_builder = bind_agent_prompt( + agent_builder, + workflow_prompt_template, + extra_goal=native_section, + ) agent = await agent_builder.build() @@ -4348,6 +4402,7 @@ Never use Chinese in any user-visible text. "agent.start", "agent.done", "agent.step", + "agent.thinking", "subagent.artifacts", ): # Sub-agent progress uses a separate event channel so parallel @@ -5216,7 +5271,7 @@ async def chat_react_agent( headers = { "Content-Type": "text/event-stream", "Cache-Control": "no-cache", - "Connection": "keep-alive", + "X-Accel-Buffering": "no", "Transfer-Encoding": "chunked", } try: @@ -5264,7 +5319,7 @@ async def chat_knowledge_agent( headers = { "Content-Type": "text/event-stream", "Cache-Control": "no-cache", - "Connection": "keep-alive", + "X-Accel-Buffering": "no", "Transfer-Encoding": "chunked", } try: diff --git a/kamidso/engine/app/openapi/api_v1/api_v1.py b/kamidso/engine/app/openapi/api_v1/api_v1.py index ddb89f3..89fb7da 100644 --- a/kamidso/engine/app/openapi/api_v1/api_v1.py +++ b/kamidso/engine/app/openapi/api_v1/api_v1.py @@ -543,7 +543,7 @@ async def chat_completions( headers = { "Content-Type": "text/event-stream", "Cache-Control": "no-cache", - "Connection": "keep-alive", + "X-Accel-Buffering": "no", "Transfer-Encoding": "chunked", } try: diff --git a/kamidso/engine/app/openapi/api_v1/native_dbgpt.py b/kamidso/engine/app/openapi/api_v1/native_dbgpt.py index efa151c..6093b7e 100644 --- a/kamidso/engine/app/openapi/api_v1/native_dbgpt.py +++ b/kamidso/engine/app/openapi/api_v1/native_dbgpt.py @@ -125,7 +125,9 @@ def prepare_react_gpts_memory(cache: Dict[str, GptsMemory], conv_id: str) -> Gpt Native mode uses persistent MetaDb backends for both messages and plans. Legacy mode keeps in-process DefaultGptsPlansMemory + MetaDb messages. - ``init`` runs only on cache miss so later user turns keep the scratchpad. + ``init`` runs only on cache miss. A finished Kamidso turn drops this entry + with ``dispose_runtime``; the next turn builds a new scratch and reloads + the needed history from MetaDB and ``kamidso_history``. """ existing = cache.get(conv_id) if existing is not None: diff --git a/kamidso/engine/app/openapi/api_v1/react_runtime.py b/kamidso/engine/app/openapi/api_v1/react_runtime.py new file mode 100644 index 0000000..8e2ac30 --- /dev/null +++ b/kamidso/engine/app/openapi/api_v1/react_runtime.py @@ -0,0 +1,91 @@ +"""In-process lifetime of one ReAct turn. + +The dictionaries below used to live for the whole backend process. A finished +turn drops its scratch memory and its conversation lock only after every +waiter for that conversation has left. +""" + +from __future__ import annotations + +import asyncio +from typing import Dict, Optional + +REACT_AGENT_MEMORY_CACHE: Dict[str, object] = {} +# Serialize ReAct runs per conversation. A second message waits on the same +# lock; explicit cancel uses REACT_CONVERSATION_AGENT_TASKS. +REACT_CONVERSATION_LOCKS: Dict[str, asyncio.Lock] = {} +REACT_CONVERSATION_AGENT_TASKS: Dict[str, "asyncio.Task[object]"] = {} + +_LOCK_GUARD = asyncio.Lock() +_LOCK_REFS: Dict[str, int] = {} + + +async def acquire_conversation_run(conv_id: str) -> asyncio.Lock: + """Take the per-conversation lock and count this waiter. + + The refcount is incremented before waiting, so a finishing turn cannot + delete the lock out from under someone already queued on it. + """ + async with _LOCK_GUARD: + lock = REACT_CONVERSATION_LOCKS.get(conv_id) + if lock is None: + lock = asyncio.Lock() + REACT_CONVERSATION_LOCKS[conv_id] = lock + _LOCK_REFS[conv_id] = _LOCK_REFS.get(conv_id, 0) + 1 + try: + await lock.acquire() + except BaseException: + # Cancelled while queued. The lock was not taken; drop the reservation + # so a finished turn can still remove it once every other waiter is gone. + await _drop_reservation(conv_id, lock) + raise + return lock + + +async def release_conversation_run(conv_id: str, lock: asyncio.Lock) -> None: + """Release one holder. Drop the lock only when nobody else is waiting.""" + lock.release() + await _drop_reservation(conv_id, lock) + + +async def _drop_reservation(conv_id: str, lock: asyncio.Lock) -> None: + """Forget one reserved waiter. Keep the lock while anyone else still needs it.""" + async with _LOCK_GUARD: + refs = _LOCK_REFS.get(conv_id, 1) - 1 + current = REACT_CONVERSATION_LOCKS.get(conv_id) + if refs <= 0 and current is lock and not lock.locked(): + REACT_CONVERSATION_LOCKS.pop(conv_id, None) + _LOCK_REFS.pop(conv_id, None) + else: + _LOCK_REFS[conv_id] = max(refs, 0) + + +def release_cached_gpts_memory(conv_id: str) -> None: + """Drop the turn's GptsMemory scratch. Persisted history is not deleted.""" + memory = REACT_AGENT_MEMORY_CACHE.pop(conv_id, None) + if memory is None: + return + dispose = getattr(memory, "dispose_runtime", None) + if callable(dispose): + dispose(conv_id) + + +async def end_react_turn(conv_id: str, lock: Optional[asyncio.Lock]) -> None: + """Free scratch memory, then release the conversation lock. + + Call this from the stream ``finally`` after the agent task has finished + and after ``_react_terminal_events`` has already persisted the round. + The lock is still held while the cache is removed, so the next waiter + cannot enter until this turn's scratch is gone. + """ + try: + if conv_id: + release_cached_gpts_memory(conv_id) + finally: + if lock is not None and conv_id: + await release_conversation_run(conv_id, lock) + + +def conversation_lock_refcount(conv_id: str) -> int: + """Test helper: holders plus waiters that have already reserved the lock.""" + return _LOCK_REFS.get(conv_id, 0) diff --git a/kamidso/engine/app/openapi/api_v1/subagent/dispatcher.py b/kamidso/engine/app/openapi/api_v1/subagent/dispatcher.py index 05a3b43..ec00fe4 100644 --- a/kamidso/engine/app/openapi/api_v1/subagent/dispatcher.py +++ b/kamidso/engine/app/openapi/api_v1/subagent/dispatcher.py @@ -34,6 +34,7 @@ from kamidso.engine.app.openapi.api_v1.native_dbgpt import ( build_agent_memory, op_snapshot_dir, ) +from kamidso.engine.app.openapi.api_v1.thinking_stream import live_thinking_display from kamidso.engine.core._private.config import Config from kamidso.engine.core.agent.expand.actions.react_action import Terminate from kamidso.engine.core.agent.resource import ToolPack, tool @@ -211,6 +212,11 @@ Use the database / knowledge base / MCP connector named in your goal. - You CANNOT delegate further (no parallel sub-tasks of your own). - Keep your context tight; produce a concise, self-contained conclusion. - When done, call ``terminate`` with your final result. +- If ``connector_call`` is in the action space, CRM and other business-app data + MUST be read with ``connector_call`` (semantic operation or the function named + in the goal). Never call ``http://127.0.0.1``, ``http://localhost``, or invent + paths such as ``/api/crm``, ``/api/didar``, or ``/search_deals``. Those hit + this application and return 404; they are not the business API. ## Output formatting (IMPORTANT — keep results readable) - One query/result per step. Prefer separate ``database().query`` calls inside @@ -324,6 +330,8 @@ async def build_sub_react_agent( extra_context: Optional[str] = None, batch_id: int = 0, report_mode: bool = False, + native_connectors: Optional[List[Any]] = None, + user_name: Optional[str] = None, ): """Construct an isolated child ReActAgent for one sub-task. @@ -359,6 +367,10 @@ async def build_sub_react_agent( # (2) Independent react_state — artifact bookkeeping isolation. A fresh # dict means generated_images / image_url_map etc. never cross-write. sub_state = {"conv_id": sub_conv_id, "file_path": None} + if native_connectors: + from kamidso.native_connectors.resolve import install_native_connectors + + install_native_connectors(sub_state, native_connectors) # (3) Independent AgentMemory — not cached in REACT_AGENT_MEMORY_CACHE, # so the sub-agent's history never pollutes the parent's memory. @@ -377,9 +389,17 @@ async def build_sub_react_agent( database_connector=database_connector, knowledge_resources=knowledge_resources, ) + native_tools: List[Any] = [] + if sub_state.get("native_connectors"): + from kamidso.engine.app.openapi.api_v1.tools.connector_call import ( + make_connector_call, + ) + + native_tools.append(make_connector_call(sub_state, user_name=user_name)) sub_tool_pack = ToolPack( [sub_tools[name] for name in _subagent_tool_names(report_mode=report_mode)] + (readonly_connector_tools or []) + + native_tools + [Terminate()] ) @@ -500,9 +520,10 @@ an execution mechanism after decomposition, not a decomposition tool. [Usage rules] - Every subtask goal must be self-contained. A sub-agent cannot see the current conversation history, so put all required information in `goal` or `context`. -- Sub-agents inherit database, knowledge-base, and read-only MCP tools, but do - not know which resource to use by default. Name the required database and - relevant tables, knowledge base, or MCP connector in every goal/context. +- Sub-agents inherit database, knowledge-base, read-only MCP tools, and, when a + native business connector is attached, ``connector_call``. Name the required + database, knowledge base, or connector operation in every goal/context. + Do not tell a sub-agent to call localhost or invent ``/api/crm`` paths. Sub-agents are read-only; the lead agent must perform write operations. - Dispatch at most N subtasks per call. Excess tasks are truncated and reported; dispatch additional batches when necessary. @@ -510,6 +531,50 @@ an execution mechanism after decomposition, not a decomposition tool. """ +def subagent_thought_event( + agent_id: str, agent_name: str, raw_text: str +) -> Optional[Dict[str, Any]]: + """Live thought text for one sub-agent, ready for the chat stream.""" + display = live_thinking_display(raw_text) + if not display: + return None + return { + "type": "agent.thinking", + "agent_id": agent_id, + "agent_name": agent_name, + "content": display, + } + + +def subagent_running_step_event( + agent_id: str, + agent_name: str, + round_num: int, + llm_reply: str, +) -> Optional[Dict[str, Any]]: + """Announce the tool a sub-agent is about to run, before it returns.""" + if not llm_reply: + return None + try: + steps = ReActOutputParser().parse(llm_reply) + except Exception: + return None + if not steps or not steps[0].action: + return None + action = str(steps[0].action) + if action.lower() == "terminate": + return None + return { + "type": "agent.step", + "agent_id": agent_id, + "agent_name": agent_name, + "round": round_num, + "action": action, + "intention": steps[0].action_intention or "", + "status": "running", + } + + def make_dispatch_tool( *, parent_conv_id: str, @@ -522,6 +587,8 @@ def make_dispatch_tool( emit_event: Any, max_parallel: int = 3, report_mode: bool = False, + native_connectors: Optional[List[Any]] = None, + user_name: Optional[str] = None, ): """Build the ``dispatch_parallel_tasks`` tool bound to the lead context. @@ -630,26 +697,50 @@ def make_dispatch_tool( extra_context=extra, batch_id=batch_id, report_mode=report_mode, + native_connectors=native_connectors, + user_name=user_name, ) from kamidso.engine.core.agent import AgentMessage - # Forward the sub-agent's CONFIRMED tool actions (not raw token - # thinking) as agent.step events tagged with this agent_id, so - # the frontend can show a live "current action" line and a - # drill-down step list. These go through emit_event on their own - # channel — they never enter the main loop's round_step_map - # (which keys steps by round_num and would collide across - # parallel sub-agents). + # Live thoughts and tool steps go out on this agent_id channel. + # They never enter the lead loop's round_step_map, which would + # collide when several sub-agents run at once. pending_sql_by_round: Dict[int, str] = {} + raw_thoughts: Dict[str, List[str]] = {} + last_thought: Dict[str, str] = {} async def _sub_stream_callback( event_type: str, payload: dict, _aid: str = agent_id ) -> None: round_num = int(payload.get("round") or 0) + if event_type == "thinking_chunk": + piece = payload.get("delta_thinking") or payload.get( + "delta_text" + ) + if not piece: + return + parts = raw_thoughts.setdefault(_aid, []) + parts.append(str(piece)) + thought = subagent_thought_event( + _aid, title, "".join(parts) + ) + if thought and thought["content"] != last_thought.get(_aid): + last_thought[_aid] = thought["content"] + await emit_event(thought) + return if event_type == "thinking": sql = _extract_sql_from_llm_reply(payload.get("llm_reply")) if sql: pending_sql_by_round[round_num] = sql + running = subagent_running_step_event( + _aid, + title, + round_num, + str(payload.get("llm_reply") or ""), + ) + if running: + running["batch_id"] = batch_id + await emit_event(running) return if event_type != "act": return @@ -690,11 +781,13 @@ def make_dispatch_tool( step_event = { "type": "agent.step", "agent_id": _aid, + "agent_name": title, "batch_id": batch_id, "action": action, "intention": ao.get("action_intention"), "chunks": chunks, "round": payload.get("round"), + "status": "completed", } if action.lower() == "sql_query": sql = _extract_sql_action_input( diff --git a/kamidso/engine/app/openapi/api_v1/subagent/history.py b/kamidso/engine/app/openapi/api_v1/subagent/history.py index b67bfce..38d1f30 100644 --- a/kamidso/engine/app/openapi/api_v1/subagent/history.py +++ b/kamidso/engine/app/openapi/api_v1/subagent/history.py @@ -216,6 +216,8 @@ def update_subagent_history( return if event_type == "agent.step": + if event.get("status") == "running": + return action = _bounded_text(event.get("action"), _MAX_NAME_CHARS) if not action: return diff --git a/kamidso/engine/app/openapi/api_v1/subagent/react_tools.py b/kamidso/engine/app/openapi/api_v1/subagent/react_tools.py index ecde1e5..6f8c145 100644 --- a/kamidso/engine/app/openapi/api_v1/subagent/react_tools.py +++ b/kamidso/engine/app/openapi/api_v1/subagent/react_tools.py @@ -353,6 +353,7 @@ def make_react_tools( or bool(react_state.get("database_connectors")) or bool(react_state.get("db_runtime")) ), + native_connector=bool(react_state.get("native_connectors")), ) @tool( diff --git a/kamidso/engine/app/openapi/api_v1/subagent/tests/test_dispatch_tool.py b/kamidso/engine/app/openapi/api_v1/subagent/tests/test_dispatch_tool.py index e97f7f8..4c4f32b 100644 --- a/kamidso/engine/app/openapi/api_v1/subagent/tests/test_dispatch_tool.py +++ b/kamidso/engine/app/openapi/api_v1/subagent/tests/test_dispatch_tool.py @@ -339,7 +339,11 @@ async def test_dispatch_sql_step_recovers_sql_from_confirmed_thinking_event( ) await tool(tasks=[{"goal": "g0", "title": "A"}]) - step = next(e for e in events if e["type"] == "agent.step") + step = next( + e + for e in events + if e["type"] == "agent.step" and e.get("status") != "running" + ) assert step["sql"] == "SELECT * FROM users" assert "action_input" not in step diff --git a/kamidso/engine/app/openapi/api_v1/subagent/tests/test_dispatcher.py b/kamidso/engine/app/openapi/api_v1/subagent/tests/test_dispatcher.py index 46d09ed..1c9bb96 100644 --- a/kamidso/engine/app/openapi/api_v1/subagent/tests/test_dispatcher.py +++ b/kamidso/engine/app/openapi/api_v1/subagent/tests/test_dispatcher.py @@ -17,6 +17,8 @@ from kamidso.engine.app.openapi.api_v1.subagent.dispatcher import ( _filter_readonly_connector_tools, _subagent_tool_names, build_sub_react_agent, + subagent_running_step_event, + subagent_thought_event, ) @@ -107,6 +109,8 @@ def test_build_sub_prompt_includes_goal(): assert "analyze sales" in pt.template # No context block when extra_context is None. assert "Shared context" not in pt.template + assert "connector_call" in pt.template + assert "localhost" in pt.template def test_build_sub_prompt_includes_context(): @@ -229,3 +233,72 @@ async def test_build_sub_react_agent_batch_id_isolates_conv_id(monkeypatch): assert cid_batch1_sub0 == "p__d1_sub_0" assert cid_batch2_sub0 == "p__d2_sub_0" assert cid_batch1_sub0 != cid_batch2_sub0 + + +def test_subagent_thought_event_keeps_the_live_thought(): + event = subagent_thought_event( + "sub-1", + "صفحه ۲", + "Thought: اتصال دیدار را با connector_call می‌خوانم\nAction: connector_call", + ) + assert event is not None + assert event["type"] == "agent.thinking" + assert "connector_call" in event["content"] + assert event["agent_name"] == "صفحه ۲" + + +def test_subagent_running_step_event_names_the_tool_before_it_returns(): + reply = ( + "Thought: صفحه بعد معاملات موفق\n" + "Action Intention: دریافت صفحه ۲\n" + "Action: connector_call\n" + 'Action Input: {"name": "search_deals"}\n' + ) + event = subagent_running_step_event("sub-1", "صفحه ۲", 2, reply) + assert event is not None + assert event["status"] == "running" + assert event["action"] == "connector_call" + assert event["round"] == 2 + assert subagent_running_step_event("sub-1", "صفحه ۲", 3, "Action: terminate\nAction Input: {}") is None + + +@pytest.mark.asyncio +async def test_sub_agent_receives_native_connector_call(monkeypatch): + import kamidso.engine.core.agent.expand.react_agent as react_mod + + class _FakeBuilt: + def __init__(self, tool_pack): + self.tool_pack = tool_pack + + class _FakeReActAgent: + def __init__(self, *a, **k): + self._tool_pack = None + + def bind(self, obj): + from kamidso.engine.core.agent.resource import ToolPack + + if isinstance(obj, ToolPack): + self._tool_pack = obj + return self + + async def build(self): + return _FakeBuilt(self._tool_pack) + + monkeypatch.setattr(react_mod, "ReActAgent", _FakeReActAgent) + agent, _, state = await build_sub_react_agent( + "goal", + 0, + parent_conv_id="parent", + llm_client=_make_fake_llm_client(), + native_connectors=[ + { + "connector_id": "didar-1", + "connector_type": "didar", + "display_name": "دیدار", + } + ], + user_name="user-1", + ) + tool_names = set(agent.tool_pack._resources.keys()) + assert "connector_call" in tool_names + assert "didar-1" in state["native_connectors"] diff --git a/kamidso/engine/app/openapi/api_v1/tools/__init__.py b/kamidso/engine/app/openapi/api_v1/tools/__init__.py index b5461e2..59e233e 100644 --- a/kamidso/engine/app/openapi/api_v1/tools/__init__.py +++ b/kamidso/engine/app/openapi/api_v1/tools/__init__.py @@ -1,6 +1,7 @@ """Built-in tools for the ReAct agent in agentic_data_api.""" from .code_interpreter import make_code_interpreter +from .connector_call import make_connector_call from .execute_analysis import make_execute_analysis from .execute_tool import make_execute_tool from .html_interpreter import make_html_interpreter @@ -10,7 +11,9 @@ from .load_file import make_load_file from .load_tools import make_load_tools from .question import make_question from .read_file import make_read_file -from .select_skill import make_select_skill # deprecated; not registered on the live agent +from .select_skill import ( + make_select_skill, # deprecated; not registered on the live agent +) from .shell_interpreter import make_shell_interpreter from .skill_tools import make_execute_skill_script_file, make_load_skill from .sql_query import make_sql_query @@ -32,5 +35,6 @@ __all__ = [ "make_execute_skill_script_file", "make_load_skill", "make_sql_query", + "make_connector_call", "make_todowrite", ] diff --git a/kamidso/engine/app/openapi/api_v1/tools/code_interpreter.py b/kamidso/engine/app/openapi/api_v1/tools/code_interpreter.py index c253eba..4dacfa5 100644 --- a/kamidso/engine/app/openapi/api_v1/tools/code_interpreter.py +++ b/kamidso/engine/app/openapi/api_v1/tools/code_interpreter.py @@ -119,6 +119,7 @@ def make_code_interpreter( *, sql_enabled: bool = True, database_runtime: bool = False, + native_connector: bool = False, ): if database_runtime: description = ( @@ -169,6 +170,13 @@ def make_code_interpreter( "unless you wrote that file yourself. " 'Parameters: {"code": "python code string"}' ) + if native_connector: + description += ( + " Business-app data is fetched with connector_call, not with Python HTTP clients. " + "When connector_call saves a result under PLOT_DIR/connector_results/*.json, " + "load that JSON with Python/pandas for calculations, joins, charts or exports. " + "Do not use requests/httpx to call attached business apps." + ) @tool(description=description) async def code_interpreter(code: str) -> str: diff --git a/kamidso/engine/app/openapi/api_v1/tools/connector_call.py b/kamidso/engine/app/openapi/api_v1/tools/connector_call.py new file mode 100644 index 0000000..64fd98a --- /dev/null +++ b/kamidso/engine/app/openapi/api_v1/tools/connector_call.py @@ -0,0 +1,202 @@ +"""Single native connector tool. One call, predefined functions only.""" + +from __future__ import annotations + +import json +from typing import Any, Dict, List, Optional + +from kamidso.engine.core.agent.resource.tool.base import tool +from kamidso.native_connectors.errors import ( + ConnectorAmbiguous, + ConnectorNotAttached, + NativeConnectorError, +) +from kamidso.native_connectors.executor import NativeConnectorExecutor +from kamidso.native_connectors.redact import redact_text +from kamidso.native_connectors.semantic.catalog import find_operation +from kamidso.native_connectors.semantic.executor import SemanticExecutor + +_TOOL_DESCRIPTION = """Call one predefined READ function or semantic operation on a native connector attached to this conversation. + +Semantic operations for management questions: sales_summary, deals_created, won_deals, lost_deals, pending_pipeline_snapshot, sales_by_owner, sales_over_time. +For those, pass a business body such as {"period": "مهر ۱۴۰۵"}. Do not build Didar Criteria yourself. + +Raw function names must be listed in the attached connector catalog. + +Do not invent API paths, HTTP methods, credentials or function names. + +If the call returns validation_error, correct the body using the returned schema and retry the SAME function. + +Do not switch to another function merely because validation failed. + +The tool validates the request before HTTP, injects credentials securely, calls the external service, and saves structured results for later analysis in code_interpreter. + +Parameters: +- name: predefined connector function name +- body: JSON body for that function +- connection: attached connector name or id when multiple connectors are attached +""" + +_REPEATED_INVALID = ( + "This exact invalid request has failed repeatedly. " + "Use the schema from the previous observation and correct the body before retrying." +) + + +def _chunks(content: str, output_type: str = "text") -> str: + return json.dumps( + {"chunks": [{"output_type": output_type, "content": content}]}, + ensure_ascii=False, + ) + + +def _canonical_body(body: Any) -> str: + if body is None: + body = {} + return json.dumps(body, sort_keys=True, ensure_ascii=False, separators=(",", ":")) + + +def _attached(react_state: Dict[str, Any]) -> List[Dict[str, Any]]: + raw = react_state.get("native_connectors") or {} + if isinstance(raw, dict): + return [item for item in raw.values() if isinstance(item, dict)] + return [] + + +def resolve_connection( + instances: List[Dict[str, Any]], + connection: Optional[str], +) -> Dict[str, Any]: + if not instances: + raise ConnectorNotAttached("No native connector is attached to this conversation.") + needle = (connection or "").strip() + if not needle: + if len(instances) == 1: + return instances[0] + raise ConnectorAmbiguous( + [str(item.get("display_name") or item.get("connector_id")) for item in instances] + ) + matches = [ + item + for item in instances + if needle == str(item.get("connector_id") or "") + or needle == str(item.get("display_name") or "") + ] + if len(matches) == 1: + return matches[0] + if not matches: + raise ConnectorNotAttached( + f"Connector '{needle}' is not attached to this conversation." + ) + raise ConnectorAmbiguous( + [str(item.get("display_name") or item.get("connector_id")) for item in matches] + ) + + +def _cache_key(connector_id: str, name: str, body: Any) -> str: + return connector_id + "\n" + name + "\n" + _canonical_body(body) + + +def _default_executor() -> NativeConnectorExecutor: + system_app = None + try: + from kamidso.engine.app.openapi.api_v1.agentic_data_api import CFG + + system_app = getattr(CFG, "SYSTEM_APP", None) + except Exception: + system_app = None + from kamidso.engine.core.agent.resource.connector.credential import CredentialStore + + return NativeConnectorExecutor(credential_store=CredentialStore(system_app=system_app)) + + +def make_connector_call( + react_state: Dict[str, Any], + *, + executor: Optional[NativeConnectorExecutor] = None, + user_name: Optional[str] = None, +): + runner = executor or _default_executor() + if user_name: + react_state["user_name"] = user_name + + @tool("connector_call", description=_TOOL_DESCRIPTION) + async def connector_call( + name: str, + body: Optional[dict] = None, + connection: Optional[str] = None, + ) -> str: + """Call one predefined read function on an attached native connector.""" + try: + instance = resolve_connection(_attached(react_state), connection) + except NativeConnectorError as exc: + return _chunks(json.dumps(exc.to_payload(), ensure_ascii=False)) + + key = _cache_key(str(instance.get("connector_id") or ""), name, body) + cache = react_state.setdefault("connector_call_cache", {}) + if not isinstance(cache, dict): + cache = {} + react_state["connector_call_cache"] = cache + cached = cache.get(key) + if isinstance(cached, dict) and cached.get("observation"): + cached["hits"] = int(cached.get("hits") or 0) + 1 + if cached["hits"] >= 2: + saved = cached.get("relative_path") or "" + where = f" Saved result: PLOT_DIR/{saved}." if saved else "" + return _chunks( + "This exact connector request was already executed. " + "Use the existing result or change the request." + + where + ) + relative = cached.get("relative_path") or "" + prefix = "Reuse previous result" + if relative: + prefix += f":\nPLOT_DIR/{relative}" + prefix += "\n\nDo not repeat this exact connector_call.\n\n" + return _chunks(prefix + str(cached.get("observation"))) + + try: + operation = find_operation(str(instance.get("connector_type") or ""), name) + if operation is not None: + result = await SemanticExecutor(runner).execute( + operation=operation, + connector_id=str(instance["connector_id"]), + body=body, + react_state=react_state, + display_name=str(instance.get("display_name") or ""), + ) + else: + result = await runner.execute( + connector_id=str(instance["connector_id"]), + function_name=name, + body=body if body is not None else {}, + react_state=react_state, + display_name=str(instance.get("display_name") or ""), + ) + except NativeConnectorError as exc: + payload = exc.to_payload() + text = json.dumps(payload, ensure_ascii=False) + if payload.get("http_sent"): + cache[key] = {"observation": text, "relative_path": "", "hits": 0} + return _chunks(redact_text(text)) + + if not result.get("ok"): + if result.get("error_type") == "validation_error": + failures = react_state.setdefault("connector_failures", {}) + if not isinstance(failures, dict): + failures = {} + react_state["connector_failures"] = failures + failures[key] = int(failures.get(key) or 0) + 1 + if failures[key] >= 3: + return _chunks(_REPEATED_INVALID) + return _chunks(json.dumps(result, ensure_ascii=False)) + + observation = redact_text(str(result.get("observation") or "")) + cache[key] = { + "observation": observation, + "relative_path": result.get("relative_path") or "", + "hits": 0, + } + return _chunks(observation, "markdown") + + return connector_call diff --git a/kamidso/engine/app/openapi/api_v2.py b/kamidso/engine/app/openapi/api_v2.py index 9952448..ba3a44d 100644 --- a/kamidso/engine/app/openapi/api_v2.py +++ b/kamidso/engine/app/openapi/api_v2.py @@ -89,7 +89,7 @@ async def chat_completions( headers = { "Content-Type": "text/event-stream", "Cache-Control": "no-cache", - "Connection": "keep-alive", + "X-Accel-Buffering": "no", "Transfer-Encoding": "chunked", } # check chat request diff --git a/kamidso/engine/app/scene/chat_data/chat_excel/excel_reader.py b/kamidso/engine/app/scene/chat_data/chat_excel/excel_reader.py index 970dd40..aa21ddb 100644 --- a/kamidso/engine/app/scene/chat_data/chat_excel/excel_reader.py +++ b/kamidso/engine/app/scene/chat_data/chat_excel/excel_reader.py @@ -242,7 +242,13 @@ class ExcelReader: db_exists = os.path.exists(database_name) - self.db = duckdb.connect(database=database_name, read_only=False) + from kamidso.engine.core.datasource.duckdb_memory import ( + open_tool_duckdb, + release_tool_duckdb, + ) + + self._release_duckdb = release_tool_duckdb + self.db = open_tool_duckdb(database_name, read_only=False) self.temp_table_name = "temp_table" self.table_name = table_name @@ -269,9 +275,20 @@ class ExcelReader: print(column) def close(self): - if self.db: - self.db.close() - self.db = None + db = self.db + self.db = None + if not db: + return + release = getattr(self, "_release_duckdb", None) + if callable(release): + try: + release(db) + except Exception: + logger.debug("Failed to release DuckDB memory budget", exc_info=True) + try: + db.close() + except Exception: + logger.debug("Failed to close DuckDB connection", exc_info=True) def __del__(self): self.close() diff --git a/kamidso/engine/core/agent/core/memory/gpts/gpts_memory.py b/kamidso/engine/core/agent/core/memory/gpts/gpts_memory.py index f449e2e..3603400 100644 --- a/kamidso/engine/core/agent/core/memory/gpts/gpts_memory.py +++ b/kamidso/engine/core/agent/core/memory/gpts/gpts_memory.py @@ -37,11 +37,18 @@ class GptsMemory: self._message_memory: GptsMessageMemory = ( message_memory if message_memory is not None else DefaultGptsMessageMemory() ) - self._executor = executor or ThreadPoolExecutor(max_workers=2) + self._owns_executor = executor is None + self._executor = executor or ThreadPoolExecutor( + max_workers=2, thread_name_prefix="gpts-memory" + ) self.messages_cache: defaultdict = defaultdict(list) self.channels: defaultdict = defaultdict(Queue) self.enable_vis_map: defaultdict = defaultdict(bool) self.start_round_map: defaultdict = defaultdict(int) + # DB-GPT chat_messages increments this while it is reading the queue. + # Kamidso's ReAct stream does not, so a vis-disabled turn must not + # enqueue history snapshots that nobody will consume. + self._queue_readers: Dict[str, int] = {} @property def plans_memory(self) -> GptsPlansMemory: @@ -75,7 +82,12 @@ class GptsMemory: return self.channels[conv_id] if conv_id in self.channels else None def clear(self, conv_id: str): - """Clear gpt memory.""" + """Drop one conversation's in-process queue and scratch lists. + + Does not delete MetaDB or any other persisted message history. + DB-GPT still calls this when an application chat finishes. Kamidso + turns use :meth:`dispose_runtime`, which also stops the owned executor. + """ # clear last message queue queue = self.channels.pop(conv_id) # noqa del queue @@ -91,27 +103,133 @@ class GptsMemory: # clear start_roun start_round = self.start_round_map.pop(conv_id) # noqa del start_round + self._queue_readers.pop(conv_id, None) + + def attach_queue_reader(self, conv_id: str) -> None: + """Mark a DB-GPT consumer that is blocked on this conversation queue.""" + self._queue_readers[conv_id] = self._queue_readers.get(conv_id, 0) + 1 + + def detach_queue_reader(self, conv_id: str) -> None: + """Release a queue consumer registered by :meth:`attach_queue_reader`.""" + remaining = self._queue_readers.get(conv_id, 0) - 1 + if remaining <= 0: + self._queue_readers.pop(conv_id, None) + else: + self._queue_readers[conv_id] = remaining + + def dispose_runtime(self, conv_id: Optional[str] = None) -> None: + """Free temporary scratch state without touching persisted history. + + ``message_memory`` and ``plans_memory`` are left intact, including + MetaDB rows. When no conversation scratch remains and this object + created its own executor, that executor is shut down. + """ + if conv_id is None: + conv_ids = ( + set(self.channels) + | set(self.messages_cache) + | set(self.enable_vis_map) + | set(self.start_round_map) + | set(self._queue_readers) + ) + else: + conv_ids = {conv_id} + for cid in conv_ids: + self._drop_runtime_conversation(cid) + if not self._has_runtime_state(): + self.shutdown_executor() + + def shutdown_executor(self) -> None: + """Stop the executor owned by this memory. Shared executors stay up.""" + executor = self._executor + if executor is None or not self._owns_executor: + return + self._executor = None + executor.shutdown(wait=False, cancel_futures=True) + + def _has_runtime_state(self) -> bool: + return bool( + self.channels + or self.messages_cache + or self.enable_vis_map + or self.start_round_map + or self._queue_readers + ) + + def _drop_runtime_conversation(self, conv_id: str) -> None: + queue = self.channels.pop(conv_id, None) + if queue is not None: + while True: + try: + queue.get_nowait() + except asyncio.QueueEmpty: + break + except Exception: + break + self.messages_cache.pop(conv_id, None) + self.enable_vis_map.pop(conv_id, None) + self.start_round_map.pop(conv_id, None) + self._queue_readers.pop(conv_id, None) + + def _latest_simple_view(self, conv_id: str) -> List[dict]: + """One view of the newest non-human message, for a live queue reader.""" + messages = self.messages_cache.get(conv_id) or [] + for message in reversed(messages): + if getattr(message, "sender", None) == "Human": + continue + view_info = getattr(message, "content", None) + action_report_str = getattr(message, "action_report", None) + if action_report_str: + try: + action_out = ActionOutput.from_dict(json.loads(action_report_str)) + except Exception: + action_out = None + if action_out is not None and action_out.content: + view_info = action_out.content + return [ + { + "sender": getattr(message, "sender", None), + "receiver": getattr(message, "receiver", None), + "model": getattr(message, "model_name", None), + "markdown": view_info, + } + ] + return [] async def push_message(self, conv_id: str, temp_msg: Optional[str] = None): - """Push conversation message.""" - queue = self.queue(conv_id) - enable_vis_tag = self.enable_vis_message(conv_id=conv_id) - if enable_vis_tag: - # 如果有临时消息内容需要push 拼接再最末尾，否则直接从短期记忆中发布最后消息 - message_view = await self.app_link_chat_message(conv_id) - if temp_msg: - temp_view = await self.agent_stream_message(temp_msg) - message_view = message_view + "\n" + temp_view - await queue.put(message_view) + """Push conversation message. - else: - # 非VIS消息模式，直接推送简单消息列表即可，不做任何处理 - message_views = await self.simple_message(conv_id) + VIS mode still publishes the DB-GPT view the legacy consumer renders. + With VIS disabled, the full history snapshot is not built. A registered + queue reader receives only the latest message (``chunk[-1]``). Kamidso + does not register a reader, so nothing is queued. + """ + enable_vis_tag = self.enable_vis_message(conv_id=conv_id) + if not enable_vis_tag: + if self._queue_readers.get(conv_id, 0) <= 0: + return + message_views = self._latest_simple_view(conv_id) if temp_msg: temp_view = await self.agent_stream_message(temp_msg, False) - if temp_view and len(temp_view) > 0: + if temp_view: message_views.extend(temp_view) + if not message_views: + return + queue = self.queue(conv_id) + if queue is None: + return await queue.put(message_views) + return + + queue = self.queue(conv_id) + if queue is None: + return + # 如果有临时消息内容需要push 拼接再最末尾，否则直接从短期记忆中发布最后消息 + message_view = await self.app_link_chat_message(conv_id) + if temp_msg: + temp_view = await self.agent_stream_message(temp_msg) + message_view = message_view + "\n" + temp_view + await queue.put(message_view) async def complete(self, conv_id: str): """Complete conversation message.""" @@ -522,14 +640,18 @@ class GptsMemory: conv_id: str, ): """Get chat messages.""" - while True: - queue = self.queue(conv_id) - if not queue: - break - item = await queue.get() - if item == "[DONE]": - queue.task_done() - break - else: - yield item - await asyncio.sleep(0.005) + self.attach_queue_reader(conv_id) + try: + while True: + queue = self.queue(conv_id) + if not queue: + break + item = await queue.get() + if item == "[DONE]": + queue.task_done() + break + else: + yield item + await asyncio.sleep(0.005) + finally: + self.detach_queue_reader(conv_id) diff --git a/kamidso/engine/core/agent/resource/connector/credential.py b/kamidso/engine/core/agent/resource/connector/credential.py index 5d7b103..52a9e64 100644 --- a/kamidso/engine/core/agent/resource/connector/credential.py +++ b/kamidso/engine/core/agent/resource/connector/credential.py @@ -14,16 +14,25 @@ _SENSITIVE_KEYS = frozenset({"password", "token", "secret", "api_key", "github_t class CredentialStore: def __init__(self, system_app: Optional[Any] = None) -> None: self._system_app = system_app + self._durable = False self._master_key = self._resolve_master_key() self._fernet = self._build_fernet(self._master_key) + def has_durable_key(self) -> bool: + """True when credentials can be decrypted after a process restart.""" + return self._durable + def _resolve_master_key(self) -> bytes: config_key: Optional[str] = None if self._system_app is not None and getattr(self._system_app, "config", None): config_key = self._system_app.config.get("kamidso.engine.core.app.global.encrypt_key") raw_key = config_key or os.environ.get("ENCRYPT_KEY") + if isinstance(raw_key, str): + raw_key = raw_key.strip() if raw_key: + self._durable = True return raw_key.encode() + self._durable = False logger.warning( "CredentialStore falling back to ephemeral master key because no " "kamidso.engine.core.app.global.encrypt_key or ENCRYPT_KEY was provided. " diff --git a/kamidso/engine/core/datasource/duckdb_memory.py b/kamidso/engine/core/datasource/duckdb_memory.py new file mode 100644 index 0000000..aa35ceb --- /dev/null +++ b/kamidso/engine/core/datasource/duckdb_memory.py @@ -0,0 +1,223 @@ +"""Cap DuckDB memory for tool connections. + +DuckDB's default limit is about 80% of host RAM per database. Each tool +connection is capped, and open tool connections split one aggregate budget +so several analyses cannot each take the full cap at once. +""" + +from __future__ import annotations + +import logging +import os +import threading +import weakref +from typing import Dict, List, Optional + +logger = logging.getLogger(__name__) + +_DEFAULT_LIMIT = "2GB" +_MEMORY_LIMIT_ENV = "KAMIDSO_DUCKDB_MEMORY_LIMIT" +_AGGREGATE_LIMIT_ENV = "KAMIDSO_DUCKDB_AGGREGATE_MEMORY_LIMIT" + +_LOCK = threading.Lock() +_LIVE: Dict[int, weakref.ReferenceType] = {} +_DEFAULT_CAPPED = False + +_UNITS = { + "tib": 1024**4, + "gib": 1024**3, + "mib": 1024**2, + "kib": 1024, + "tb": 1000**4, + "gb": 1000**3, + "mb": 1000**2, + "kb": 1000, + "ti": 1024**4, + "gi": 1024**3, + "mi": 1024**2, + "ki": 1024, + "t": 1000**4, + "g": 1000**3, + "m": 1000**2, + "k": 1000, + "b": 1, +} + + +def configured_memory_limit() -> str: + """Per-connection cap. ``KAMIDSO_DUCKDB_MEMORY_LIMIT`` overrides the 2GB default.""" + raw = (os.environ.get(_MEMORY_LIMIT_ENV) or "").strip() + if not raw: + return _DEFAULT_LIMIT + try: + parse_memory_limit(raw) + except ValueError: + logger.warning("Ignoring invalid %s=%s; using %s", _MEMORY_LIMIT_ENV, raw, _DEFAULT_LIMIT) + return _DEFAULT_LIMIT + return raw + + +def configured_aggregate_limit() -> str: + """Budget shared by open tool connections. Defaults to the per-connection cap.""" + raw = (os.environ.get(_AGGREGATE_LIMIT_ENV) or "").strip() + if not raw: + return configured_memory_limit() + try: + parse_memory_limit(raw) + except ValueError: + logger.warning( + "Ignoring invalid %s=%s; using the per-connection limit", + _AGGREGATE_LIMIT_ENV, + raw, + ) + return configured_memory_limit() + return raw + + +def parse_memory_limit(value: str) -> int: + """Parse ``2GB``, ``512MiB``, or a bare byte count.""" + text = value.strip().lower().replace(" ", "") + if not text: + raise ValueError("empty memory limit") + if text.isdigit(): + return int(text) + for unit in sorted(_UNITS, key=len, reverse=True): + if text.endswith(unit): + number = text[: -len(unit)] + if not number: + raise ValueError(value) + return int(float(number) * _UNITS[unit]) + raise ValueError(value) + + +def format_duckdb_limit(num_bytes: int) -> str: + """Render a byte count with a unit DuckDB's parser accepts.""" + if num_bytes <= 0: + return "1MiB" + if num_bytes % (1024**3) == 0: + return f"{num_bytes // (1024**3)}GiB" + mebibytes = max(1, num_bytes // (1024**2)) + return f"{mebibytes}MiB" + + +def cap_duckdb_default_connection() -> None: + """Stop the module-level DuckDB connection from taking ~80% of host RAM. + + That connection is not part of the tool aggregate. It only needs a ceiling. + """ + global _DEFAULT_CAPPED + if _DEFAULT_CAPPED: + return + _DEFAULT_CAPPED = True + try: + import duckdb + except Exception: + logger.debug("duckdb is not installed; skip default memory cap", exc_info=True) + return + try: + connection = duckdb.default_connection() + connection.execute(f"SET memory_limit='{configured_memory_limit()}'") + except Exception: + logger.warning("Failed to cap the default DuckDB connection", exc_info=True) + + +def register_tool_duckdb(connection: object) -> None: + """Apply the shared budget to a tool connection and remember it.""" + cap_duckdb_default_connection() + key = id(connection) + + def _dead(ref: weakref.ReferenceType, conn_id: int = key) -> None: + with _LOCK: + current = _LIVE.get(conn_id) + if current is ref: + _LIVE.pop(conn_id, None) + _rebalance_locked() + + with _LOCK: + _LIVE[key] = weakref.ref(connection, _dead) + _rebalance_locked() + + +def release_tool_duckdb(connection: object) -> None: + """Forget a tool connection and give its budget back to the ones still open.""" + key = id(connection) + with _LOCK: + _LIVE.pop(key, None) + _rebalance_locked() + + +def bind_duckdb_memory_limit(engine: object) -> None: + """Set the cap on every DBAPI connection this SQLAlchemy engine opens.""" + if getattr(engine, "_kamidso_duckdb_memory_bound", False): + return + setattr(engine, "_kamidso_duckdb_memory_bound", True) + cap_duckdb_default_connection() + from sqlalchemy import event + + @event.listens_for(engine, "connect") + def _on_connect(dbapi_connection, _connection_record) -> None: # noqa: ANN001 + register_tool_duckdb(dbapi_connection) + + @event.listens_for(engine, "close") + def _on_close(dbapi_connection, _connection_record) -> None: # noqa: ANN001 + release_tool_duckdb(dbapi_connection) + + +def open_tool_duckdb(database: str = ":memory:", read_only: bool = False): + """Open a DuckDB database for a tool and register it under the memory budget.""" + import duckdb + + connection = duckdb.connect(database=database, read_only=read_only) + register_tool_duckdb(connection) + return connection + + +def reset_duckdb_memory_state_for_tests() -> None: + """Drop registry entries. Does not close caller-owned connections.""" + global _DEFAULT_CAPPED + with _LOCK: + _LIVE.clear() + _DEFAULT_CAPPED = False + + +def _share_bytes(open_connections: int) -> int: + count = max(1, open_connections) + per_connection = parse_memory_limit(configured_memory_limit()) + aggregate = parse_memory_limit(configured_aggregate_limit()) + return max(1024 * 1024, min(per_connection, aggregate // count)) + + +def _limit_sql(open_connections: int) -> str: + count = max(1, open_connections) + per_text = configured_memory_limit() + per_bytes = parse_memory_limit(per_text) + share = _share_bytes(count) + if count == 1 and share == per_bytes: + return per_text + return format_duckdb_limit(share) + + +def _rebalance_locked() -> None: + live = _live_connections_locked() + if not live: + return + statement = f"SET memory_limit='{_limit_sql(len(live))}'" + for connection in live: + try: + connection.execute(statement) + except Exception: + logger.debug("Could not update DuckDB memory_limit", exc_info=True) + + +def _live_connections_locked() -> List[object]: + dead: List[int] = [] + live: List[object] = [] + for key, ref in _LIVE.items(): + connection: Optional[object] = ref() + if connection is None: + dead.append(key) + else: + live.append(connection) + for key in dead: + _LIVE.pop(key, None) + return live diff --git a/kamidso/engine/core/model/proxy/llms/chatgpt.py b/kamidso/engine/core/model/proxy/llms/chatgpt.py index f797219..8bcad5a 100644 --- a/kamidso/engine/core/model/proxy/llms/chatgpt.py +++ b/kamidso/engine/core/model/proxy/llms/chatgpt.py @@ -2,6 +2,7 @@ import asyncio import json import logging import os +from contextlib import asynccontextmanager from concurrent.futures import Executor from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any, AsyncIterator, Dict, List, Optional, Type, Union @@ -347,6 +348,9 @@ class OpenAILLMClient(ProxyLLMClient): self._stream_idle_timeout = stream_idle_timeout_seconds() self._api_type = api_type self._client = openai_client + # Tests inject one client. Production streams each open their own so a + # proxy reset on one chat cannot close the other chat's socket. + self._client_is_injected = openai_client is not None self._openai_kwargs = openai_kwargs or {} resolved_reasoning = coerce_optional_bool(reasoning_enabled) if resolved_reasoning is None: @@ -416,6 +420,33 @@ class OpenAILLMClient(ProxyLLMClient): ) return self._client + @asynccontextmanager + async def _open_stream_client(self): + """HTTP client that lives only for this stream. + + A shared client keeps one proxy tunnel. The next user's stream is + written onto that tunnel and both responses reset. + """ + if self._client_is_injected: + yield self._client + return + from kamidso.engine.core.model.utils.chatgpt_utils import _build_openai_client + + _api_type, client = _build_openai_client( + init_params=self._init_params, + timeout=self._stream_idle_timeout, + isolated=True, + ) + try: + yield client + finally: + close = getattr(client, "close", None) + if close is not None: + try: + await close() + except Exception: + logger.debug("Failed to close stream LLM client", exc_info=True) + @property def default_model(self) -> str: model = self._model @@ -540,9 +571,15 @@ class OpenAILLMClient(ProxyLLMClient): async for r in self.generate_stream_v1(messages, payload): yield r - async def _create_chat(self, messages: List[Dict[str, Any]], payload: Dict[str, Any]): + async def _create_chat( + self, + messages: List[Dict[str, Any]], + payload: Dict[str, Any], + client: Optional["ClientType"] = None, + ): + openai_client = client if client is not None else self.client return await asyncio.wait_for( - self.client.chat.completions.create(messages=messages, **payload), + openai_client.chat.completions.create(messages=messages, **payload), timeout=self._stream_idle_timeout, ) @@ -600,49 +637,52 @@ class OpenAILLMClient(ProxyLLMClient): for attempt in range(1, attempts + 1): yielded = False try: - chat_completion = await self._create_chat(messages, payload) - text = "" - reasoning_content = "" - usage = None - tool_calls_acc: Optional[List[Dict[str, Any]]] = None - async for r in _iter_with_idle_timeout( - chat_completion, self._stream_idle_timeout - ): - if len(r.choices) == 0: - continue - # Check for empty 'choices' issue in Azure GPT-4o responses - if r.choices[0] is not None and r.choices[0].delta is None: - continue - delta_obj = r.choices[0].delta - if hasattr(delta_obj, "reasoning_content"): - reasoning_content += delta_obj.reasoning_content or "" - if r.choices[0].delta.content is not None: - text += r.choices[0].delta.content - delta_tool_calls = getattr(delta_obj, "tool_calls", None) or None - if delta_tool_calls: - tool_calls_acc = merge_streaming_tool_calls( - tool_calls_acc, delta_tool_calls - ) - if text or reasoning_content: - if hasattr(r, "usage") and r.usage is not None: - usage = r.usage.dict() + async with self._open_stream_client() as client: + chat_completion = await self._create_chat( + messages, payload, client=client + ) + text = "" + reasoning_content = "" + usage = None + tool_calls_acc: Optional[List[Dict[str, Any]]] = None + async for r in _iter_with_idle_timeout( + chat_completion, self._stream_idle_timeout + ): + if len(r.choices) == 0: + continue + # Check for empty 'choices' issue in Azure GPT-4o responses + if r.choices[0] is not None and r.choices[0].delta is None: + continue + delta_obj = r.choices[0].delta + if hasattr(delta_obj, "reasoning_content"): + reasoning_content += delta_obj.reasoning_content or "" + if r.choices[0].delta.content is not None: + text += r.choices[0].delta.content + delta_tool_calls = getattr(delta_obj, "tool_calls", None) or None + if delta_tool_calls: + tool_calls_acc = merge_streaming_tool_calls( + tool_calls_acc, delta_tool_calls + ) + if text or reasoning_content: + if hasattr(r, "usage") and r.usage is not None: + usage = r.usage.dict() + yielded = True + yield self._drop_thinking_if_disabled( + ModelOutput.build(text, reasoning_content, usage=usage), + payload, + ) + if tool_calls_acc: yielded = True yield self._drop_thinking_if_disabled( - ModelOutput.build(text, reasoning_content, usage=usage), + ModelOutput.build( + text, + reasoning_content, + usage=usage, + tool_calls=tool_calls_acc, + ), payload, ) - if tool_calls_acc: - yielded = True - yield self._drop_thinking_if_disabled( - ModelOutput.build( - text, - reasoning_content, - usage=usage, - tool_calls=tool_calls_acc, - ), - payload, - ) - return + return except Exception as exc: if not _is_stream_stall(exc) or yielded or attempt >= attempts: if _is_stream_stall(exc): diff --git a/kamidso/engine/core/model/proxy/tests/test_chatgpt_stream_retry.py b/kamidso/engine/core/model/proxy/tests/test_chatgpt_stream_retry.py index 56d463f..3a77347 100644 --- a/kamidso/engine/core/model/proxy/tests/test_chatgpt_stream_retry.py +++ b/kamidso/engine/core/model/proxy/tests/test_chatgpt_stream_retry.py @@ -96,3 +96,50 @@ async def test_stalled_stream_retries_the_same_request(monkeypatch): assert texts == ["بازیابی شد"] assert completions_client.completions.calls == 2 + + +@pytest.mark.asyncio +async def test_concurrent_streams_open_separate_http_clients(monkeypatch): + created = [] + + class _OneShot: + default_headers = {} + + def __init__(self, isolated: bool): + self.isolated = isolated + self.closed = False + self.chat = SimpleNamespace(completions=_Completions([_TokenStream("ok")])) + + async def close(self): + self.closed = True + + def _fake_build(init_params, timeout=None, isolated=False): + client = _OneShot(isolated) + created.append(client) + return "open_ai", client + + monkeypatch.setattr( + "kamidso.engine.core.model.utils.chatgpt_utils._build_openai_client", + _fake_build, + ) + owner = OpenAILLMClient(api_key="test-key", model="qwen") + + async def collect(): + texts = [] + async for output in owner.generate_stream_v1( + [], {"model": "qwen", "stream": True} + ): + texts.append(output.text) + return texts + + first, second = await asyncio.gather(collect(), collect()) + + isolated = [client for client in created if client.isolated] + shared = [client for client in created if not client.isolated] + assert first == ["ok"] + assert second == ["ok"] + assert len(shared) == 1 + assert len(isolated) == 2 + assert isolated[0] is not isolated[1] + assert all(client.closed for client in isolated) + assert shared[0].closed is False diff --git a/kamidso/engine/core/model/utils/chatgpt_utils.py b/kamidso/engine/core/model/utils/chatgpt_utils.py index 5cb1713..0718f9a 100644 --- a/kamidso/engine/core/model/utils/chatgpt_utils.py +++ b/kamidso/engine/core/model/utils/chatgpt_utils.py @@ -113,6 +113,8 @@ def _initialize_openai_v1(init_params: OpenAIParameters): def _build_openai_client( init_params: OpenAIParameters, timeout: Optional[float] = None, + *, + isolated: bool = False, ) -> Tuple[str, ClientType]: import httpx @@ -146,13 +148,22 @@ def _build_openai_client( # Remove proxies for httpx AsyncClient when httpx version >= 0.28.0 httpx_version = metadata.version("httpx") - limits = openai_httpx_limits() - client_kwargs = {"limits": limits} + # A stream-scoped client must not park its socket in a shared pool. + # Proxies reuse that socket for the next user and both streams reset. + limits = ( + httpx.Limits(max_connections=4, max_keepalive_connections=0) + if isolated + else openai_httpx_limits() + ) + client_kwargs = {"limits": limits, "http2": False} if http_timeout is not None: client_kwargs["timeout"] = http_timeout + proxy = init_params.proxy + if isinstance(proxy, str): + proxy = proxy.strip() or None if httpx_version >= "0.28.0": - if init_params.proxy: - client_kwargs["proxy"] = init_params.proxy + if proxy: + client_kwargs["proxy"] = proxy http_client = httpx.AsyncClient(**client_kwargs) elif init_params.proxies: client_kwargs["proxies"] = init_params.proxies diff --git a/kamidso/engine/core/model/utils/tests/test_chatgpt_httpx.py b/kamidso/engine/core/model/utils/tests/test_chatgpt_httpx.py index aef62ca..03a6e8e 100644 --- a/kamidso/engine/core/model/utils/tests/test_chatgpt_httpx.py +++ b/kamidso/engine/core/model/utils/tests/test_chatgpt_httpx.py @@ -1,6 +1,12 @@ """httpx connection limits for the OpenAI-compatible proxy client.""" -from kamidso.engine.core.model.utils.chatgpt_utils import openai_httpx_limits +import pytest + +from kamidso.engine.core.model.utils.chatgpt_utils import ( + OpenAIParameters, + _build_openai_client, + openai_httpx_limits, +) def test_openai_httpx_limits_match_proxy_concurrency(): @@ -8,3 +14,18 @@ def test_openai_httpx_limits_match_proxy_concurrency(): assert limits.max_connections == 100 assert limits.max_keepalive_connections == 40 + + +@pytest.mark.asyncio +async def test_isolated_stream_client_does_not_reuse_connections(): + _api_type, client = _build_openai_client( + OpenAIParameters(api_key="test-key", api_base="https://example.invalid/v1"), + timeout=30, + isolated=True, + ) + try: + pool = client._client._transport._pool + assert pool._max_keepalive_connections == 0 + assert pool._max_connections == 4 + finally: + await client.close() diff --git a/kamidso/engine/ext/datasource/rdbms/conn_duckdb.py b/kamidso/engine/ext/datasource/rdbms/conn_duckdb.py index a55e8aa..f8b94be 100644 --- a/kamidso/engine/ext/datasource/rdbms/conn_duckdb.py +++ b/kamidso/engine/ext/datasource/rdbms/conn_duckdb.py @@ -49,6 +49,12 @@ class DuckDbConnector(RDBMSConnector): db_type: str = "duckdb" db_dialect: str = "duckdb" + def __init__(self, engine, **kwargs: Any): + from kamidso.engine.core.datasource.duckdb_memory import bind_duckdb_memory_limit + + bind_duckdb_memory_limit(engine) + super().__init__(engine, **kwargs) + @classmethod def param_class(cls) -> Type[DuckDbConnectorParameters]: """Return the parameter class.""" diff --git a/kamidso/engine/serve/agent/agents/controller.py b/kamidso/engine/serve/agent/agents/controller.py index d5ff969..5647717 100644 --- a/kamidso/engine/serve/agent/agents/controller.py +++ b/kamidso/engine/serve/agent/agents/controller.py @@ -319,6 +319,9 @@ class MultiAgents(BaseComponent, ABC): # init agent memory agent_memory = self.get_or_build_agent_memory(conv_id, gpts_name) + # Register before the agent task starts so early messages still have + # a consumer when VIS is disabled and the simple stream reads chunk[-1]. + self.memory.attach_queue_reader(agent_conv_id) task = None try: task = asyncio.create_task( @@ -420,6 +423,7 @@ class MultiAgents(BaseComponent, ABC): yield task, str(e), agent_conv_id finally: + self.memory.detach_queue_reader(agent_conv_id) self.memory.clear(agent_conv_id) def is_flow_chat(self, gpts_name: str): @@ -735,21 +739,25 @@ class MultiAgents(BaseComponent, ABC): user_code: str = None, system_app: str = None, ): - while True: - queue = self.memory.queue(conv_id) - if not queue: - break - item = await queue.get() - if item == "[DONE]": - queue.task_done() - break - elif isinstance(item, str) and item.startswith("__CONTEXT_STATUS__"): - # Forward context status as a dedicated SSE event - yield item - await asyncio.sleep(0.005) - else: - yield item - await asyncio.sleep(0.005) + self.memory.attach_queue_reader(conv_id) + try: + while True: + queue = self.memory.queue(conv_id) + if not queue: + break + item = await queue.get() + if item == "[DONE]": + queue.task_done() + break + elif isinstance(item, str) and item.startswith("__CONTEXT_STATUS__"): + # Forward context status as a dedicated SSE event + yield item + await asyncio.sleep(0.005) + else: + yield item + await asyncio.sleep(0.005) + finally: + self.memory.detach_queue_reader(conv_id) async def stable_message( self, conv_id: str, user_code: str = None, system_app: str = None diff --git a/kamidso/engine/serve/connector/api/endpoints.py b/kamidso/engine/serve/connector/api/endpoints.py index e538dff..a57d2e7 100644 --- a/kamidso/engine/serve/connector/api/endpoints.py +++ b/kamidso/engine/serve/connector/api/endpoints.py @@ -124,6 +124,33 @@ async def list_connector_types() -> Result[List[Dict[str, Any]]]: ) ) options.append(_CUSTOM_MCP_OPTION) + from kamidso.native_connectors.flags import ( + didar_connector_enabled, + native_connectors_v1_enabled, + ) + from kamidso.native_connectors.registry import get_registry + + if native_connectors_v1_enabled() and didar_connector_enabled(): + registry = get_registry() + if registry.contains("didar"): + provider = registry.get("didar") + options.append( + ConnectorTypeOption( + type=provider.id, + display_name=provider.display_name, + description="Read-only Didar CRM connector", + category=provider.category or "crm", + is_custom=False, + auth_fields=[ + { + "name": provider.credential_field, + "label": "API Key", + "type": "password", + "required": True, + } + ], + ) + ) return Result.succ([o.model_dump() for o in options]) @@ -265,6 +292,22 @@ async def test_connector( return Result.failed(msg=str(e)) +@router.get("/{connector_id}/functions") +async def get_connector_functions( + connector_id: str, + service: ConnectorService = Depends(get_service), +) -> Result[Dict[str, Any]]: + try: + return Result.succ(service.list_native_functions(connector_id)) + except HTTPException: + raise + except ValueError as exc: + raise HTTPException(status_code=404, detail=str(exc)) + except Exception as exc: + logger.exception("Get connector functions failed") + return Result.failed(msg=str(exc)) + + @router.get("/{connector_id}/tools", response_model=Result[ConnectorToolsResponse]) async def get_connector_tools( connector_id: str, diff --git a/kamidso/engine/serve/connector/service/service.py b/kamidso/engine/serve/connector/service/service.py index c254b66..cc09885 100644 --- a/kamidso/engine/serve/connector/service/service.py +++ b/kamidso/engine/serve/connector/service/service.py @@ -4,6 +4,7 @@ import asyncio import json import logging import uuid +from datetime import datetime, timezone from typing import Any, Dict, List, Literal, Optional from fastapi import HTTPException @@ -24,6 +25,25 @@ from ..models.models import ConnectorInstanceDao, ConnectorInstanceEntity logger = logging.getLogger(__name__) +def _utc_stamp() -> str: + return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +class ConnectorProbeFailed(ValueError): + """Health check failed before a native connector credential was saved.""" + + def __init__( + self, + message: str, + *, + status_code: Optional[int] = None, + error_type: Optional[str] = None, + ) -> None: + super().__init__(message) + self.status_code = status_code + self.error_type = error_type + + class ConnectorCreateRequest(BaseModel): """Request model for creating a connector.""" @@ -158,6 +178,109 @@ class ConnectorService( """Initialize the service.""" self._system_app = system_app self._credential_store = CredentialStore(system_app=system_app) + from kamidso.native_connectors.flags import native_connectors_v1_enabled + from kamidso.native_connectors.registry import get_registry + + if native_connectors_v1_enabled(): + get_registry() + + def _is_native_type(self, connector_type: Optional[str]) -> bool: + from kamidso.native_connectors.flags import native_connectors_v1_enabled + from kamidso.native_connectors.registry import get_registry + + if not native_connectors_v1_enabled() or not connector_type: + return False + return get_registry().contains(connector_type) + + def _require_durable_native_key(self) -> None: + from kamidso.native_connectors.flags import is_production + + if is_production() and not self._credential_store.has_durable_key(): + raise ValueError( + "ENCRYPT_KEY must be configured before creating a native connector " + "in production." + ) + + def _native_executor(self): + from kamidso.native_connectors.executor import NativeConnectorExecutor + + return NativeConnectorExecutor(credential_store=self._credential_store) + + def _create_native_connector(self, request: ConnectorCreateRequest) -> ConnectorResponse: + """Persist a native connector only after its catalog health check succeeds.""" + self._require_durable_native_key() + from kamidso.native_connectors.registry import get_registry + + provider = get_registry().get(request.connector_type) + credentials = {k: str(v) for k, v in request.credentials.items()} + if not str(credentials.get(provider.credential_field) or "").strip(): + raise ValueError( + f"Missing credential field: {provider.credential_field}" + ) + loop = get_or_create_event_loop() + probed = loop.run_until_complete( + self._native_executor().probe(request.connector_type, credentials) + ) + if not probed.get("success"): + status_code = probed.get("status_code") + raise ConnectorProbeFailed( + str(probed.get("message") or "Connection test failed"), + status_code=status_code if isinstance(status_code, int) else None, + error_type=str(probed.get("error_type") or "") or None, + ) + + connector_id = str(uuid.uuid4()) + encryption_salt = self._credential_store.generate_salt() + encrypted_credentials = self._credential_store.encrypt( + credentials, + encryption_salt, + ) + config = dict(request.config or {}) + config["last_tested_at"] = _utc_stamp() + config["last_test_success"] = True + config_json = json.dumps(config) + entity = ConnectorInstanceEntity( + connector_id=connector_id, + connector_type=request.connector_type, + display_name=request.display_name, + encrypted_credentials=encrypted_credentials, + encryption_salt=encryption_salt, + status="active", + config_json=config_json, + user_name=request.user_name, + sys_code=request.sys_code, + ) + with self._dao.session() as session: + session.add(entity) + session.flush() + session.refresh(entity) + return _entity_to_response(entity) + + def list_native_functions(self, connector_id: str) -> Dict[str, Any]: + """Return the predefined function catalog summary for one native connector.""" + connector = self.get_connector(connector_id) + if connector is None: + raise HTTPException( + status_code=404, detail=f"Connector '{connector_id}' not found" + ) + if not self._is_native_type(connector.connector_type): + raise ValueError(f"Connector '{connector_id}' is not a native connector") + from kamidso.native_connectors.registry import get_registry + + provider = get_registry().get(connector.connector_type) + return { + "provider": provider.id, + "functions": [ + { + "name": function.name, + "description": function.description, + "domain": function.domain, + "effect": function.effect, + "enabled": function.enabled, + } + for function in provider.functions + ], + } def after_start(self): """Rehydrate active connectors into the runtime connector manager.""" @@ -199,6 +322,8 @@ class ConnectorService( ) for connector in active_connectors: + if self._is_native_type(connector.get("connector_type")): + continue try: credentials = self._credential_store.decrypt( connector["encrypted_credentials"], @@ -280,6 +405,9 @@ class ConnectorService( Returns: ConnectorResponse: The created connector response. """ + if self._is_native_type(request.connector_type): + return self._create_native_connector(request) + connector_id = str(uuid.uuid4()) encryption_salt = self._credential_store.generate_salt() encrypted_credentials = self._credential_store.encrypt( @@ -412,12 +540,23 @@ class ConnectorService( connector_id, ) existing = {} - # New fields override existing; existing fields survive when - # new value is empty/missing. merged = { **existing, **{k: str(v) for k, v in request.credentials.items()}, } + if self._is_native_type(entity.connector_type): + self._require_durable_native_key() + loop = get_or_create_event_loop() + probed = loop.run_until_complete( + self._native_executor().probe(entity.connector_type, merged) + ) + if not probed.get("success"): + status_code = probed.get("status_code") + raise ConnectorProbeFailed( + str(probed.get("message") or "Connection test failed"), + status_code=status_code if isinstance(status_code, int) else None, + error_type=str(probed.get("error_type") or "") or None, + ) entity.encryption_salt = self._credential_store.generate_salt() entity.encrypted_credentials = self._credential_store.encrypt( merged, @@ -430,9 +569,10 @@ class ConnectorService( entity.config_json = json.dumps(request.config) config_changed = True - # P0-1 fix: re-activate manager if credentials/config changed, - # so the connector becomes usable AND the status reflects reality. - if credentials_changed or config_changed: + if self._is_native_type(entity.connector_type): + if credentials_changed or config_changed: + entity.status = "active" + elif credentials_changed or config_changed: previous_status = entity.status entity.status = "active" @@ -510,6 +650,73 @@ class ConnectorService( raise ValueError(f"Connector '{connector_id}' not found") session.delete(entity) + def retire_connector(self, connector_id: str) -> None: + """Hide a connector from new chats without dropping its history row.""" + with self._dao.session() as session: + entity = ( + session.query(ConnectorInstanceEntity) + .filter(ConnectorInstanceEntity.connector_id == connector_id) + .first() + ) + if entity is None: + raise ValueError(f"Connector '{connector_id}' not found") + entity.status = "deleted" + connector_manager = self.system_app.get_component( + "connector_manager", + ConnectorManager, + default_component=None, + ) + if connector_manager is None: + return + loop = get_or_create_event_loop() + try: + loop.run_until_complete(connector_manager.remove_connector(connector_id)) + except Exception: + logger.info("Connector '%s' was not active in the MCP manager", connector_id) + + def record_last_test(self, connector_id: str, *, success: bool) -> None: + """Remember the latest health check without storing credentials.""" + try: + with self._dao.session() as session: + entity = ( + session.query(ConnectorInstanceEntity) + .filter(ConnectorInstanceEntity.connector_id == connector_id) + .first() + ) + if entity is None: + return + config: Dict[str, Any] = {} + if entity.config_json: + try: + loaded = json.loads(entity.config_json) + if isinstance(loaded, dict): + config = loaded + except Exception: + config = {} + config["last_tested_at"] = _utc_stamp() + config["last_test_success"] = success + entity.config_json = json.dumps(config) + except Exception: + logger.exception("Failed to record the last test for '%s'", connector_id) + + async def test_native_credentials( + self, connector_type: str, credentials: Dict[str, Any] + ) -> Dict[str, Any]: + """Probe a native provider without writing a connector row.""" + if not self._is_native_type(connector_type): + raise ValueError("Unknown native provider") + from kamidso.native_connectors.registry import get_registry + + provider = get_registry().get(connector_type) + cleaned = { + str(key): str(value) + for key, value in credentials.items() + if str(value).strip() + } + if not str(cleaned.get(provider.credential_field) or "").strip(): + raise ValueError(f"Missing credential field: {provider.credential_field}") + return await self._native_executor().probe(connector_type, cleaned) + def list_tools(self, connector_id: str) -> ConnectorToolsResponse: """Return the tool summaries for a single connector. @@ -528,6 +735,13 @@ class ConnectorService( raise HTTPException( status_code=404, detail=f"Connector '{connector_id}' not found" ) + current = self.get_connector(connector_id) + if current is not None and self._is_native_type(current.connector_type): + return ConnectorToolsResponse( + connector_id=connector_id, + state="not_mcp", + tools=[], + ) connector_manager = self.system_app.get_component( "connector_manager", @@ -625,6 +839,22 @@ class ConnectorService( if connector is None: raise ValueError(f"Connector '{connector_id}' not found") + if self._is_native_type(connector.connector_type): + probed = await self._native_executor().probe_stored(connector_id) + success = bool(probed.get("success")) + if success: + self._heal_status_to_active(connector_id) + self.record_last_test(connector_id, success=success) + result: Dict[str, Any] = { + "success": success, + "message": str(probed.get("message") or ""), + } + if isinstance(probed.get("status_code"), int): + result["status_code"] = probed["status_code"] + if probed.get("error_type"): + result["error_type"] = probed["error_type"] + return result + cm = self.system_app.get_component( "connector_manager", ConnectorManager, default_component=None ) diff --git a/kamidso/engine/serve/connector/tests/test_service.py b/kamidso/engine/serve/connector/tests/test_service.py index beb50f7..dc63476 100644 --- a/kamidso/engine/serve/connector/tests/test_service.py +++ b/kamidso/engine/serve/connector/tests/test_service.py @@ -1163,3 +1163,160 @@ def test_list_tools_p95_latency_under_100ms_for_50_tools( samples_ms.sort() p95 = samples_ms[int(len(samples_ms) * 0.95) - 1] assert p95 < 100.0, f"p95={p95:.2f}ms exceeds 100ms budget" + + +def _enable_didar(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("KAMIDSO_NATIVE_CONNECTORS_V1", "1") + monkeypatch.setenv("KAMIDSO_DIDAR_CONNECTOR", "1") + from kamidso.native_connectors.registry import reset_registry + + reset_registry() + + +def test_native_create_persists_without_mcp_and_encrypts_api_key( + service: ConnectorService, monkeypatch: pytest.MonkeyPatch +): + _enable_didar(monkeypatch) + from kamidso.native_connectors.executor import NativeConnectorExecutor + + async def _probe(self, provider_id, credentials): + assert credentials["api_key"] == "didar-secret" + return {"success": True, "message": "ok"} + + monkeypatch.setattr(NativeConnectorExecutor, "probe", _probe) + fake_manager = MagicMock() + fake_manager.create_connector = AsyncMock() + + def _fake_get_component(name, component_type, default_component=None): + if name == "connector_manager": + return fake_manager + return default_component + + monkeypatch.setattr(service.system_app, "get_component", _fake_get_component) + response = service.create_connector( + ConnectorCreateRequest( + connector_type="didar", + display_name="دیدار فروش", + credentials={"api_key": "didar-secret"}, + user_name="usr_test", + ) + ) + fake_manager.create_connector.assert_not_called() + assert response.status == "active" + assert "didar-secret" not in response.model_dump_json() + with db.session() as session: + entity = ( + session.query(ConnectorInstanceEntity) + .filter(ConnectorInstanceEntity.connector_id == response.connector_id) + .first() + ) + assert entity is not None + assert entity.connector_type == "didar" + assert "didar-secret" not in (entity.encrypted_credentials or "") + store = CredentialStore(system_app=service.system_app) + assert store.decrypt(entity.encrypted_credentials, entity.encryption_salt) == { + "api_key": "didar-secret" + } + tools = service.list_tools(response.connector_id) + assert tools.state == "not_mcp" + assert tools.tools == [] + functions = service.list_native_functions(response.connector_id) + assert functions["provider"] == "didar" + assert any(item["name"] == "search_deals" and item["enabled"] for item in functions["functions"]) + assert "input_schema" not in json.dumps(functions) + + +def test_native_create_does_not_persist_when_health_check_fails( + service: ConnectorService, monkeypatch: pytest.MonkeyPatch +): + _enable_didar(monkeypatch) + from kamidso.native_connectors.executor import NativeConnectorExecutor + + async def _probe(self, provider_id, credentials): + return {"success": False, "message": "invalid api key"} + + monkeypatch.setattr(NativeConnectorExecutor, "probe", _probe) + with pytest.raises(ValueError, match="invalid api key"): + service.create_connector( + ConnectorCreateRequest( + connector_type="didar", + display_name="دیدار فروش", + credentials={"api_key": "bad"}, + ) + ) + with db.session() as session: + assert session.query(ConnectorInstanceEntity).count() == 0 + + +def test_native_create_requires_durable_key_in_production( + service: ConnectorService, monkeypatch: pytest.MonkeyPatch +): + _enable_didar(monkeypatch) + monkeypatch.setenv("KAMIDSO_ENV", "production") + monkeypatch.setattr(service._credential_store, "has_durable_key", lambda: False) + from kamidso.native_connectors.executor import NativeConnectorExecutor + + called = {"probe": False} + + async def _probe(self, provider_id, credentials): + called["probe"] = True + return {"success": True, "message": "ok"} + + monkeypatch.setattr(NativeConnectorExecutor, "probe", _probe) + with pytest.raises(ValueError, match="ENCRYPT_KEY"): + service.create_connector( + ConnectorCreateRequest( + connector_type="didar", + display_name="دیدار فروش", + credentials={"api_key": "didar-secret"}, + ) + ) + assert called["probe"] is False + + +def test_after_start_skips_native_connectors( + service: ConnectorService, monkeypatch: pytest.MonkeyPatch +): + _enable_didar(monkeypatch) + store = CredentialStore(system_app=service.system_app) + salt = store.generate_salt() + with db.session() as session: + session.add( + ConnectorInstanceEntity( + connector_id="didar-1", + connector_type="didar", + display_name="دیدار فروش", + encrypted_credentials=store.encrypt({"api_key": "didar-secret"}, salt), + encryption_salt=salt, + status="active", + ) + ) + fake_manager = MagicMock() + fake_manager.create_connector = AsyncMock() + + def _fake_get_component(name, component_type, default_component=None): + if name == "connector_manager": + return fake_manager + return default_component + + monkeypatch.setattr(service.system_app, "get_component", _fake_get_component) + service.after_start() + fake_manager.create_connector.assert_not_called() + + +def test_retire_connector_keeps_the_row(service: ConnectorService): + with db.session() as session: + session.add( + ConnectorInstanceEntity( + connector_id="didar-keep", + connector_type="didar", + display_name="دیدار فروش", + status="active", + user_name="usr_1", + ) + ) + service.retire_connector("didar-keep") + kept = service.get_connector("didar-keep") + assert kept is not None + assert kept.status == "deleted" + diff --git a/kamidso/integrations/engine/engine.py b/kamidso/integrations/engine/engine.py index 08a79b1..eacdbb9 100644 --- a/kamidso/integrations/engine/engine.py +++ b/kamidso/integrations/engine/engine.py @@ -13,7 +13,11 @@ from kamidso.contracts.chat import KamidsoChatRequest from kamidso.contracts.resource import ResourceRef from kamidso.contracts.run import KamidsoRun from kamidso.contracts.user import CurrentUser -from kamidso.integrations.engine.events import map_engine_event +from kamidso.integrations.engine.events import ( + begin_adapter_run, + map_engine_event, + release_adapter_run, +) from kamidso.integrations.engine.runtime import EngineRuntime from kamidso.resources.resolver import MaterializedResource from kamidso.settings.flags import KamidsoSettings @@ -74,6 +78,32 @@ class KamidsoEngine: } return + begin_adapter_run(run.id) + try: + async for event in self._execute_ready( + user=user, + run=run, + request=request, + materialized=materialized, + prior_history=prior_history, + last_report=last_report, + ): + yield event + finally: + # Failure, cancellation, and client disconnect all land here, + # after the last event has already been yielded. + release_adapter_run(run.id) + + async def _execute_ready( + self, + *, + user: CurrentUser, + run: KamidsoRun, + request: KamidsoChatRequest, + materialized: List[MaterializedResource], + prior_history: Optional[List[Dict[str, str]]] = None, + last_report: Optional[Dict[str, Any]] = None, + ) -> AsyncIterator[Dict[str, Any]]: from kamidso.engine.app.openapi.api_v1.agentic_data_api import ( _open_turn_attachments, _react_agent_stream, @@ -109,6 +139,9 @@ class KamidsoEngine: ext_info["knowledge_space_name"] = kb_ids[0] if kb_label: ext_info["knowledge_space_label"] = kb_label + connector_refs = [ref for ref in request.resources if ref.type == "connector" and ref.id] + if connector_refs: + ext_info.update(connector_selection(connector_refs)) context = self._builder.build( user, @@ -226,6 +259,15 @@ class KamidsoEngine: logger.warning("Failed to close ReAct agent stream", exc_info=True) +def connector_selection(resources: List[Any]) -> Dict[str, List[str]]: + """Map attached connector resources onto the existing connector_ids field.""" + refs = [ref for ref in resources if getattr(ref, "type", None) == "connector" and getattr(ref, "id", None)] + return { + "connector_ids": [ref.id for ref in refs], + "connector_labels": [ref.name or ref.id for ref in refs], + } + + def _stage_files_for_engine( materialized: List[MaterializedResource], ) -> tuple[List[str], List[str]]: diff --git a/kamidso/integrations/engine/events.py b/kamidso/integrations/engine/events.py index cbd796d..4b8724c 100644 --- a/kamidso/integrations/engine/events.py +++ b/kamidso/integrations/engine/events.py @@ -1,15 +1,121 @@ +"""Map engine SSE payloads to product events, and forget finished runs.""" + from __future__ import annotations -from typing import Any, Dict, List +import logging +import os +import threading +import time +from typing import Any, Dict, List, Optional from kamidso.integrations.event_adapter import EventAdapterState +logger = logging.getLogger("kamidso.engine") + +_ADAPTER_CACHE: Dict[str, EventAdapterState] = {} +_ACTIVE_RUNS: set[str] = set() +_TOUCHED: Dict[str, float] = {} +_LOCK = threading.Lock() +_SWEEPER_STARTED = False +_DEFAULT_TTL_SECONDS = 900.0 + + +def adapter_cache_ttl_seconds() -> float: + raw = (os.environ.get("KAMIDSO_ADAPTER_CACHE_TTL_SECONDS") or "").strip() + if not raw: + return _DEFAULT_TTL_SECONDS + try: + return max(1.0, float(raw)) + except ValueError: + logger.warning("Ignoring invalid KAMIDSO_ADAPTER_CACHE_TTL_SECONDS=%s", raw) + return _DEFAULT_TTL_SECONDS + + +def begin_adapter_run(run_id: str) -> None: + """Keep this run's adapter alive for the whole execution, including long tools.""" + with _LOCK: + _ACTIVE_RUNS.add(run_id) + _TOUCHED[run_id] = time.monotonic() + _ensure_sweeper_locked() + + +def release_adapter_run(run_id: str) -> None: + """Drop adapter state after the last event for this run has been produced. + + Safe on success, failure, cancellation, and generator close. Does not + remove a different run that is still active. + """ + with _LOCK: + _ACTIVE_RUNS.discard(run_id) + _ADAPTER_CACHE.pop(run_id, None) + _TOUCHED.pop(run_id, None) + + +def sweep_adapter_cache(*, now: Optional[float] = None, keep: Optional[str] = None) -> List[str]: + """Forget inactive adapters that have not been touched within the TTL. + + Active runs are kept even if they have been quiet, so a long SQL or code + step is not discarded mid-flight. + """ + moment = time.monotonic() if now is None else now + ttl = adapter_cache_ttl_seconds() + removed: List[str] = [] + with _LOCK: + stale = [ + run_id + for run_id, touched in _TOUCHED.items() + if run_id != keep and run_id not in _ACTIVE_RUNS and moment - touched >= ttl + ] + for run_id in stale: + _ADAPTER_CACHE.pop(run_id, None) + _TOUCHED.pop(run_id, None) + removed.append(run_id) + return removed + def map_engine_event( run_id: str, engine_event: Dict[str, Any], locale: str = "fa-IR" ) -> List[Dict[str, Any]]: - adapter = _ADAPTER_CACHE.setdefault(run_id, EventAdapterState(run_id, locale=locale)) - return adapter.map(engine_event) + with _LOCK: + _TOUCHED[run_id] = time.monotonic() + adapter = _ADAPTER_CACHE.get(run_id) + if adapter is None: + adapter = EventAdapterState(run_id, locale=locale) + _ADAPTER_CACHE[run_id] = adapter + _ensure_sweeper_locked() + events = adapter.map(engine_event) + sweep_adapter_cache(keep=run_id) + return events -_ADAPTER_CACHE: Dict[str, EventAdapterState] = {} +def adapter_cache_size() -> int: + with _LOCK: + return len(_ADAPTER_CACHE) + + +def adapter_run_is_active(run_id: str) -> bool: + with _LOCK: + return run_id in _ACTIVE_RUNS + + +def _ensure_sweeper_locked() -> None: + global _SWEEPER_STARTED + if _SWEEPER_STARTED: + return + _SWEEPER_STARTED = True + thread = threading.Thread( + target=_sweep_loop, + name="kamidso-adapter-cache", + daemon=True, + ) + thread.start() + + +def _sweep_loop() -> None: + while True: + ttl = adapter_cache_ttl_seconds() + time.sleep(min(60.0, ttl)) + try: + sweep_adapter_cache() + except Exception: + logger.debug("adapter cache sweep failed", exc_info=True) diff --git a/kamidso/native_connectors/__init__.py b/kamidso/native_connectors/__init__.py new file mode 100644 index 0000000..994378d --- /dev/null +++ b/kamidso/native_connectors/__init__.py @@ -0,0 +1,5 @@ +"""Predefined business-app connectors. Independent of MCP.""" + +from kamidso.native_connectors.registry import get_registry, reset_registry + +__all__ = ["get_registry", "reset_registry"] diff --git a/kamidso/native_connectors/errors.py b/kamidso/native_connectors/errors.py new file mode 100644 index 0000000..412d3d1 --- /dev/null +++ b/kamidso/native_connectors/errors.py @@ -0,0 +1,144 @@ +"""Error taxonomy for native connector calls.""" + +from __future__ import annotations + +from typing import Any, Dict, List, Optional + + +class NativeConnectorError(Exception): + error_type = "connector_error" + http_sent = False + + def __init__( + self, + message: str, + *, + function: Optional[str] = None, + details: Optional[Dict[str, Any]] = None, + ) -> None: + super().__init__(message) + self.message = message + self.function = function + self.details = details or {} + + def to_payload(self) -> Dict[str, Any]: + payload: Dict[str, Any] = { + "ok": False, + "error_type": self.error_type, + "message": self.message, + "http_sent": self.http_sent, + } + if self.function: + payload["function"] = self.function + payload.update(self.details) + return payload + + +class CatalogError(NativeConnectorError): + error_type = "catalog_error" + + +class ConnectorNotAttached(NativeConnectorError): + error_type = "connector_not_attached" + + +class ConnectorNotFound(NativeConnectorError): + error_type = "connector_not_found" + + +class ConnectorAmbiguous(NativeConnectorError): + error_type = "connector_ambiguous" + + def __init__(self, names: List[str]) -> None: + listing = "\n".join(f"- {name}" for name in names) + super().__init__( + "Several native connectors are attached.\n\nSpecify connection:\n" + listing, + details={"connections": names}, + ) + + +class ProviderNotFound(NativeConnectorError): + error_type = "provider_not_found" + + +class FunctionNotFound(NativeConnectorError): + error_type = "unknown_function" + + def __init__(self, function: str, provider_label: str, suggestions: List[str]) -> None: + super().__init__( + f"Function '{function}' does not exist for {provider_label}.", + function=function, + details={ + "suggestions": suggestions, + "instruction": "Use only predefined function names.", + }, + ) + + +class FunctionDisabled(NativeConnectorError): + error_type = "function_disabled" + + +class OperationNotAllowed(NativeConnectorError): + error_type = "operation_not_allowed" + + def __init__(self, function: str, effect: str) -> None: + super().__init__( + "This connector is read-only. No HTTP request was sent.", + function=function, + details={"effect": effect}, + ) + + +class ValidationFailed(NativeConnectorError): + error_type = "validation_error" + + +class AuthenticationFailed(NativeConnectorError): + """Local credential failure. Remote 401 uses RemoteCallFailed.""" + + error_type = "authentication_error" + + +class RemoteCallFailed(NativeConnectorError): + def __init__( + self, + message: str, + *, + error_type: str, + function: Optional[str] = None, + details: Optional[Dict[str, Any]] = None, + ) -> None: + super().__init__(message, function=function, details=details) + self.error_type = error_type + self.http_sent = True + + +class TransportFailed(NativeConnectorError): + error_type = "transport_error" + + +class CallTimeout(NativeConnectorError): + error_type = "timeout_error" + + +class ResponseTooLarge(NativeConnectorError): + error_type = "response_too_large" + http_sent = True + + +class InvalidJSONResponse(NativeConnectorError): + error_type = "invalid_json_response" + http_sent = True + + +class ResultMaterializationError(NativeConnectorError): + error_type = "result_materialization_error" + http_sent = True + + +class AccuracyGuard(NativeConnectorError): + """An exact management figure cannot be verified. No substitute total is included.""" + + error_type = "accuracy_guard" + http_sent = True diff --git a/kamidso/native_connectors/executor.py b/kamidso/native_connectors/executor.py new file mode 100644 index 0000000..d2e706d --- /dev/null +++ b/kamidso/native_connectors/executor.py @@ -0,0 +1,482 @@ +"""Call a predefined native connector function. Credentials stay inside this module.""" + +from __future__ import annotations + +import json +import logging +import time +from typing import Any, Callable, Dict, Optional +from urllib.parse import urlparse + +import httpx + +from kamidso.native_connectors.errors import ( + AuthenticationFailed, + CallTimeout, + ConnectorNotFound, + FunctionDisabled, + FunctionNotFound, + InvalidJSONResponse, + NativeConnectorError, + OperationNotAllowed, + ProviderNotFound, + RemoteCallFailed, + ResponseTooLarge, + ResultMaterializationError, + TransportFailed, +) +from kamidso.native_connectors.models import ConnectorFunction, NativeProvider +from kamidso.native_connectors.redact import redact, redact_text, strip_sensitive_fields +from kamidso.native_connectors.registry import NativeConnectorRegistry, get_registry +from kamidso.native_connectors.result_store import ( + materialize_connector_result, + render_success_observation, +) +from kamidso.native_connectors.validator import validate_body, validation_payload + +logger = logging.getLogger(__name__) + +_MAX_RESPONSE_BYTES = 5 * 1024 * 1024 +_ATTEMPT_TIMEOUT = 30.0 +_TOTAL_BUDGET = 60.0 +_RETRY_STATUSES = {500, 502, 503} +RowLoader = Callable[[str], Optional[Dict[str, Any]]] + + +def _status_error_type(status_code: int) -> str: + if status_code == 401: + return "authentication_error" + if status_code == 403: + return "permission_error" + if status_code == 429: + return "rate_limit_error" + if status_code == 400: + return "remote_validation_error" + if status_code >= 500: + return "remote_server_error" + return "http_error" + + +def suggest_functions(provider: NativeProvider, name: str) -> list: + import difflib + + enabled = [item.name for item in provider.enabled_reads()] + suggestions = list(provider.aliases.get(name) or []) + if "lead" in name.lower(): + for candidate in ("search_persons", "search_deals"): + if candidate in enabled and candidate not in suggestions: + suggestions.append(candidate) + close = difflib.get_close_matches(name, enabled, n=3, cutoff=0.5) + for candidate in close: + if candidate not in suggestions: + suggestions.append(candidate) + return [item for item in suggestions if item in enabled][:3] + + +def default_row_loader(connector_id: str) -> Optional[Dict[str, Any]]: + from kamidso.engine.serve.connector.config import ServeConfig + from kamidso.engine.serve.connector.models.models import ( + ConnectorInstanceDao, + ConnectorInstanceEntity, + ) + + dao = ConnectorInstanceDao(ServeConfig()) + with dao.session() as session: + entity = ( + session.query(ConnectorInstanceEntity) + .filter(ConnectorInstanceEntity.connector_id == connector_id) + .first() + ) + if entity is None: + return None + config: Dict[str, Any] = {} + raw_config = entity.config_json + if isinstance(raw_config, str) and raw_config.strip(): + try: + parsed = json.loads(raw_config) + except json.JSONDecodeError: + parsed = None + if isinstance(parsed, dict): + config = parsed + return { + "connector_id": entity.connector_id, + "connector_type": entity.connector_type, + "display_name": entity.display_name or "", + "status": entity.status or "", + "encrypted_credentials": entity.encrypted_credentials, + "encryption_salt": entity.encryption_salt, + "user_name": entity.user_name, + "sys_code": entity.sys_code, + "config": config, + } + + +class NativeConnectorExecutor: + def __init__( + self, + registry: Optional[NativeConnectorRegistry] = None, + *, + client: Optional[httpx.AsyncClient] = None, + credential_store: Any = None, + row_loader: Optional[RowLoader] = None, + ) -> None: + self._registry = registry + self._client = client + self._credential_store = credential_store + self._row_loader = row_loader or default_row_loader + + def load_row(self, connector_id: str) -> Optional[Dict[str, Any]]: + return self._row_loader(connector_id) + + @property + def registry(self) -> NativeConnectorRegistry: + if self._registry is None: + self._registry = get_registry() + return self._registry + + async def probe(self, provider_id: str, credentials: Dict[str, str]) -> Dict[str, Any]: + try: + provider = self.registry.get(provider_id) + except ProviderNotFound as exc: + return {"success": False, "message": exc.message} + try: + result = await self._call( + provider=provider, + function_name=provider.health_function, + body=dict(provider.health_body), + credentials=credentials, + react_state=None, + materialize=False, + display_name=provider.display_name, + ) + except NativeConnectorError as exc: + payload = exc.to_payload() + status_code = payload.get("status_code") + return { + "success": False, + "message": redact_text(exc.message), + "status_code": status_code if isinstance(status_code, int) else None, + "error_type": exc.error_type, + } + if result.get("ok"): + return {"success": True, "message": "Connection successful"} + return { + "success": False, + "message": redact_text(str(result.get("message") or "Connection test failed")), + } + + async def probe_stored(self, connector_id: str) -> Dict[str, Any]: + row = self._row_loader(connector_id) + if row is None: + return {"success": False, "message": f"Connector '{connector_id}' not found"} + try: + credentials = self._decrypt(row) + except NativeConnectorError as exc: + return {"success": False, "message": exc.message} + return await self.probe(str(row.get("connector_type") or ""), credentials) + + async def execute( + self, + *, + connector_id: str, + function_name: str, + body: Any, + react_state: Optional[Dict[str, Any]], + display_name: Optional[str] = None, + ) -> Dict[str, Any]: + row = self._row_loader(connector_id) + if row is None or str(row.get("status") or "") != "active": + raise ConnectorNotFound(f"Connector '{connector_id}' is not available.") + provider = self.registry.get(str(row.get("connector_type") or "")) + label = display_name or str(row.get("display_name") or provider.display_name) + return await self._call( + provider=provider, + function_name=function_name, + body=body, + credentials=None, + react_state=react_state, + materialize=True, + display_name=label, + audit_user=str((react_state or {}).get("user_name") or row.get("user_name") or ""), + connector_id=connector_id, + encrypted_row=row, + ) + + def _decrypt(self, row: Dict[str, Any]) -> Dict[str, str]: + if self._credential_store is None: + raise AuthenticationFailed("Credential store is not configured.") + try: + decrypted = self._credential_store.decrypt( + row.get("encrypted_credentials"), + row.get("encryption_salt"), + ) + except Exception as exc: + raise AuthenticationFailed("Stored credentials could not be decrypted.") from exc + if not isinstance(decrypted, dict): + raise AuthenticationFailed("Stored credentials could not be decrypted.") + return {str(key): str(value) for key, value in decrypted.items()} + + async def _call( + self, + *, + provider: NativeProvider, + function_name: str, + body: Any, + credentials: Optional[Dict[str, str]], + react_state: Optional[Dict[str, Any]], + materialize: bool, + display_name: str, + audit_user: str = "", + connector_id: str = "", + encrypted_row: Optional[Dict[str, Any]] = None, + ) -> Dict[str, Any]: + function = provider.function(function_name) + if function is None: + raise FunctionNotFound( + function_name, + provider.display_name, + suggest_functions(provider, function_name), + ) + if function.effect != "read": + raise OperationNotAllowed(function.name, function.effect) + if not function.enabled: + raise FunctionDisabled( + f"Function '{function.name}' is disabled.", + function=function.name, + ) + normalized, issues = validate_body(function, body) + if issues: + return validation_payload( + connector_name=display_name, + function=function, + issues=issues, + ) + assert normalized is not None + if credentials is None: + if encrypted_row is None: + raise AuthenticationFailed( + "The connector credential is missing.", + function=function.name, + ) + credentials = self._decrypt(encrypted_row) + secret = str(credentials.get(provider.credential_field) or "").strip() + if not secret: + raise AuthenticationFailed( + "The connector credential is missing.", + function=function.name, + ) + started = time.perf_counter() + response, request_size = await self._send( + provider, function, strip_sensitive_fields(normalized), secret + ) + duration_ms = int((time.perf_counter() - started) * 1000) + response_size = len(response.content or b"") + self._audit( + react_state=react_state, + user_name=audit_user, + connector_id=connector_id, + connector_type=provider.id, + function=function, + status_code=response.status_code, + duration_ms=duration_ms, + request_size=request_size, + response_size=response_size, + ) + if response_size > _MAX_RESPONSE_BYTES: + raise ResponseTooLarge( + "Use narrower filters or pagination.", + function=function.name, + details={"status_code": response.status_code}, + ) + if response.status_code in {301, 302, 303, 307, 308}: + raise RemoteCallFailed( + "The connector refused a redirect.", + error_type="http_error", + function=function.name, + details={ + "provider": provider.id, + "status_code": response.status_code, + "body": None, + }, + ) + payload = self._decode_json(response, function.name) + if response.status_code >= 400: + details: Dict[str, Any] = { + "provider": provider.id, + "status_code": response.status_code, + "body": redact(payload), + } + retry_after = response.headers.get("Retry-After") + if response.status_code == 429 and retry_after: + details["retry_after"] = retry_after + raise RemoteCallFailed( + f"The connector returned HTTP {response.status_code}.", + error_type=_status_error_type(response.status_code), + function=function.name, + details=details, + ) + if not materialize or react_state is None: + return { + "ok": True, + "http_sent": True, + "status_code": response.status_code, + "body": payload, + } + try: + meta = materialize_connector_result(react_state, payload) + except OSError as exc: + raise ResultMaterializationError( + "The connector response could not be saved.", + function=function.name, + ) from exc + observation = render_success_observation( + provider=provider, + function=function, + body=normalized, + status_code=response.status_code, + payload=payload, + relative_path=meta["relative_path"], + ) + return { + "ok": True, + "http_sent": True, + "status_code": response.status_code, + "relative_path": meta["relative_path"], + "observation": observation, + } + + def _decode_json(self, response: httpx.Response, function_name: str) -> Any: + if not response.content: + return {} + try: + return response.json() + except (json.JSONDecodeError, ValueError) as exc: + raise InvalidJSONResponse( + "The connector returned a non-JSON response.", + function=function_name, + details={"status_code": response.status_code}, + ) from exc + + async def _send( + self, + provider: NativeProvider, + function: ConnectorFunction, + body: Dict[str, Any], + secret: str, + ): + url = _safe_url(provider, function.path) + params = {provider.auth_parameter: secret} + headers = dict(provider.default_headers) + request_size = len(json.dumps(body, ensure_ascii=False).encode("utf-8")) + deadline = time.monotonic() + _TOTAL_BUDGET + attempt = 0 + while True: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise CallTimeout( + "The connector request exceeded the time budget.", + function=function.name, + ) + timeout = httpx.Timeout(min(_ATTEMPT_TIMEOUT, remaining)) + try: + response = await self._request( + function.method, + url, + params=params, + headers=headers, + body=body, + timeout=timeout, + ) + except httpx.TimeoutException as exc: + raise CallTimeout( + "The connector request timed out.", + function=function.name, + ) from exc + except httpx.TransportError as exc: + if attempt == 0 and _retryable_transport(exc) and deadline - time.monotonic() > 0: + attempt += 1 + continue + raise TransportFailed( + redact_text(str(exc)) or "The connector request failed.", + function=function.name, + ) from exc + if response.status_code in _RETRY_STATUSES and attempt == 0 and deadline - time.monotonic() > 0: + attempt += 1 + continue + return response, request_size + + async def _request( + self, + method: str, + url: str, + *, + params: Dict[str, str], + headers: Dict[str, str], + body: Dict[str, Any], + timeout: httpx.Timeout, + ) -> httpx.Response: + kwargs: Dict[str, Any] = { + "params": params, + "headers": headers, + "timeout": timeout, + "follow_redirects": False, + } + if method.upper() != "GET": + kwargs["json"] = body + + async def _send(client: httpx.AsyncClient) -> httpx.Response: + return await client.request(method, url, **kwargs) + + if self._client is not None: + return await _send(self._client) + async with httpx.AsyncClient(follow_redirects=False) as client: + return await _send(client) + + def _audit( + self, + *, + react_state: Optional[Dict[str, Any]], + user_name: str, + connector_id: str, + connector_type: str, + function: ConnectorFunction, + status_code: int, + duration_ms: int, + request_size: int, + response_size: int, + ) -> None: + logger.info( + "native_connector conversation_id=%s user_name=%s connector_id=%s " + "connector_type=%s function=%s effect=%s http_status=%s duration_ms=%s " + "request_size=%s response_size=%s cache_hit=%s", + (react_state or {}).get("conv_id") or "", + user_name, + connector_id, + connector_type, + function.name, + function.effect, + status_code, + duration_ms, + request_size, + response_size, + False, + ) + + +def _retryable_transport(exc: httpx.TransportError) -> bool: + if isinstance(exc, httpx.TimeoutException): + return False + return isinstance(exc, (httpx.NetworkError, httpx.RemoteProtocolError)) + + +def _safe_url(provider: NativeProvider, path: str) -> str: + base = urlparse(provider.base_url) + if base.scheme != "https" or not base.hostname: + raise TransportFailed("The connector host is not allowed.") + if not path.startswith("/") or "://" in path or ".." in path.split("/"): + raise TransportFailed("The connector path is not allowed.") + url = provider.base_url.rstrip("/") + path + parsed = urlparse(url) + if parsed.scheme != "https" or parsed.hostname != base.hostname: + raise TransportFailed("The connector host is not allowed.") + return url diff --git a/kamidso/native_connectors/flags.py b/kamidso/native_connectors/flags.py new file mode 100644 index 0000000..5cb66dd --- /dev/null +++ b/kamidso/native_connectors/flags.py @@ -0,0 +1,21 @@ +"""Feature flags for native business connectors.""" + +from __future__ import annotations + +import os + + +def _truthy(name: str, default: str = "0") -> bool: + return os.environ.get(name, default).strip().lower() in {"1", "true", "yes", "on"} + + +def native_connectors_v1_enabled() -> bool: + return _truthy("KAMIDSO_NATIVE_CONNECTORS_V1", "0") + + +def didar_connector_enabled() -> bool: + return _truthy("KAMIDSO_DIDAR_CONNECTOR", "0") + + +def is_production() -> bool: + return os.environ.get("KAMIDSO_ENV", "").strip().lower() == "production" diff --git a/kamidso/native_connectors/models.py b/kamidso/native_connectors/models.py new file mode 100644 index 0000000..55f72d7 --- /dev/null +++ b/kamidso/native_connectors/models.py @@ -0,0 +1,59 @@ +"""In-memory catalog models. Loaded from provider files, not from the network.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Dict, List, Optional + + +@dataclass(frozen=True) +class ConnectorFunction: + name: str + title: str + description: str + domain: str + effect: str + enabled: bool + method: str + path: str + input_schema: Dict[str, Any] + response_root: str + records_path: Optional[str] = None + summary_fields: List[str] = field(default_factory=list) + semantic_hints: List[str] = field(default_factory=list) + pagination: Optional[Dict[str, Any]] = None + content_type: str = "application/json" + timeout_seconds: int = 30 + + +@dataclass(frozen=True) +class NativeProvider: + id: str + version: str + display_name: str + category: str + base_url: str + auth_type: str + auth_parameter: str + credential_field: str + default_headers: Dict[str, str] + mode: str + health_function: str + health_body: Dict[str, Any] + aliases: Dict[str, List[str]] + guidance: str + functions: List[ConnectorFunction] + description: str = "" + + def function(self, name: str) -> Optional[ConnectorFunction]: + for item in self.functions: + if item.name == name: + return item + return None + + def enabled_reads(self) -> List[ConnectorFunction]: + return [ + item + for item in self.functions + if item.enabled and item.effect == "read" + ] diff --git a/kamidso/native_connectors/prompt.py b/kamidso/native_connectors/prompt.py new file mode 100644 index 0000000..9b550eb --- /dev/null +++ b/kamidso/native_connectors/prompt.py @@ -0,0 +1,110 @@ +"""Prompt text for attached native connectors. Names and descriptions only.""" + +from __future__ import annotations + +import re +from typing import Any, Dict, Iterable, List + +from kamidso.native_connectors.models import ConnectorFunction +from kamidso.native_connectors.registry import get_registry + +_PATH_MENTION = re.compile( + r"(?i)\b(?:GET|POST|PUT|PATCH|DELETE|HEAD)\s+/api/\S+|/api/\S+" +) + +_DOMAIN_LABELS = { + "activity": "Activities", + "case": "Cases", + "company": "Companies", + "deal": "Deals", + "invoice": "Invoices", + "location": "Locations", + "note": "Notes", + "person": "People", + "pipeline": "Pipelines", + "product": "Products", + "user": "Users", +} + +_RULES = """Native connector functions are predefined. +Never invent a connector function. +Use connector_call to fetch business data. +Do not use code_interpreter to call the external API. +Use code_interpreter only after connector_call when computation, dataframe operations, merging, charts, statistics or export are required. +If connector_call returns validation_error: +1. Read the returned schema. +2. Fix the body. +3. Retry the SAME function. +Do not create a different function to work around a malformed request. +A validation_error means the selected function exists and is appropriate enough to validate. +Prefer server-provided totals and aggregates from connector responses. +Do not paginate through all records merely to count them. +Didar data is fetched with connector_call. Database data uses database(). Excel and CSV files use code_interpreter. +For Didar management questions, call a semantic operation and pass a business body. Do not build Didar Criteria. +sales_summary answers فروش in a period and means deals that became Won in that period. Example body: {"period": "مهر ۱۴۰۵"}. +deals_created answers معاملات in a period and means deals registered in that period. State the period basis. +Other semantic operations: won_deals, lost_deals, pending_pipeline_snapshot, sales_by_owner, sales_over_time. +Keep every accuracy warning in the answer, including mode, date basis, currency, and panel parity. +Current deal status comes only from Status. ChangeToWonTime never overrides Pending. +Response.List is one page. Totals come from aggregates such as WonDealsCount, PendingDealsCount, and LostDealsCount. +Do not invent get_leads, search_leads, or list_leads. lead_count stays disabled. +Do not sum amounts that have different CurrencyId values, and do not guess a currency name. +On raw search calls, From and Limit are top-level siblings of Criteria, not fields inside Criteria. +""" + + +def _domain_label(domain: str) -> str: + return _DOMAIN_LABELS.get(domain, domain.replace("_", " ").title() or "Other") + + +def _public_description(description: str) -> str: + cleaned = _PATH_MENTION.sub("", description) + return " ".join(cleaned.split()) + + +def _function_lines(functions: Iterable[ConnectorFunction]) -> List[str]: + grouped: Dict[str, List[ConnectorFunction]] = {} + for function in functions: + grouped.setdefault(function.domain or "other", []).append(function) + lines: List[str] = [] + for domain, items in grouped.items(): + lines.append(_domain_label(domain)) + for function in items: + lines.append(f"- {function.name} — {_public_description(function.description)}") + lines.append("") + return lines + + +def render_native_prompt(instances: List[Dict[str, Any]]) -> str: + if not instances: + return "" + registry = get_registry() + blocks: List[str] = [ + "## Connected Business Apps", + "", + _RULES.strip(), + "", + ] + by_type: Dict[str, List[Dict[str, Any]]] = {} + for instance in instances: + by_type.setdefault(str(instance.get("connector_type") or ""), []).append(instance) + for connector_type, group in by_type.items(): + if not registry.contains(connector_type): + continue + provider = registry.get(connector_type) + if provider.guidance: + blocks.append(provider.guidance) + blocks.append("") + for instance in group: + blocks.extend( + [ + f"### {instance.get('display_name') or instance.get('connector_id')}", + f"Provider: {provider.display_name}", + "Access: Read only", + "", + "Available functions:", + "", + ] + ) + blocks.extend(_function_lines(provider.enabled_reads())) + return "\n".join(blocks).strip() diff --git a/kamidso/native_connectors/providers/didar/calibration_schema.json b/kamidso/native_connectors/providers/didar/calibration_schema.json new file mode 100644 index 0000000..a30ee75 --- /dev/null +++ b/kamidso/native_connectors/providers/didar/calibration_schema.json @@ -0,0 +1,94 @@ +{ + "schema_version": "1.0", + "provider": "didar", + "storage_target": "connector_instance.config_json.semantic_calibration", + "schema": { + "type": "object", + "properties": { + "timezone": { + "type": "string", + "default": "Asia/Tehran" + }, + "calendar": { + "type": "string", + "enum": [ + "jalali", + "gregorian" + ], + "default": "jalali" + }, + "base_currency_id": { + "type": [ + "integer", + "string", + "null" + ] + }, + "base_currency_code": { + "type": [ + "string", + "null" + ] + }, + "base_currency_name": { + "type": [ + "string", + "null" + ] + }, + "display_unit": { + "type": [ + "string", + "null" + ] + }, + "currency_map": { + "type": "object" + }, + "verified_reports": { + "type": "object", + "additionalProperties": { + "type": "object", + "properties": { + "verified": { + "type": "boolean" + }, + "verified_at": { + "type": [ + "string", + "null" + ] + }, + "panel_sample_id": { + "type": [ + "string", + "null" + ] + }, + "notes": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "verified" + ] + } + } + }, + "additionalProperties": false + }, + "default": { + "timezone": "Asia/Tehran", + "calendar": "jalali", + "base_currency_id": null, + "base_currency_code": null, + "base_currency_name": null, + "display_unit": null, + "currency_map": {}, + "verified_reports": {} + }, + "rule": "Calibration is connection-specific. Do not put account-specific currency mappings or verification flags in the global provider files." +} \ No newline at end of file diff --git a/kamidso/native_connectors/providers/didar/date_rules.json b/kamidso/native_connectors/providers/didar/date_rules.json new file mode 100644 index 0000000..f56612a --- /dev/null +++ b/kamidso/native_connectors/providers/didar/date_rules.json @@ -0,0 +1,93 @@ +{ + "schema_version": "1.0", + "provider": "didar", + "default_user_calendar": "jalali", + "default_business_timezone": "Asia/Tehran", + "api_timestamp_timezone": "UTC", + "range_semantics": "[start, end)", + "parser_requirements": { + "accept_persian_digits": true, + "accept_latin_digits": true, + "accept_persian_month_names": true, + "accept_relative_periods": [ + "امروز", + "دیروز", + "این هفته", + "هفته قبل", + "این ماه", + "ماه قبل", + "امسال", + "سال قبل", + "سه ماه اخیر", + "شش ماه اخیر" + ] + }, + "jalali_months": { + "فروردین": 1, + "اردیبهشت": 2, + "خرداد": 3, + "تیر": 4, + "مرداد": 5, + "شهریور": 6, + "مهر": 7, + "آبان": 8, + "آذر": 9, + "دی": 10, + "بهمن": 11, + "اسفند": 12 + }, + "period_resolution": { + "year_omitted": "Use the current Jalali year in the conversation/business timezone unless context clearly names another year.", + "month_period": "Start at 00:00:00 local time on day 1; end at 00:00:00 local time on day 1 of the next Jalali month.", + "utc_conversion": "Convert local start/end to UTC before calling Didar.", + "output_metadata": [ + "input_text", + "calendar", + "timezone", + "start_local", + "end_local_exclusive", + "start_utc", + "end_utc_exclusive" + ] + }, + "deal_date_basis": { + "sales": "won_at", + "won_deals": "won_at", + "lost_deals": "lost_at", + "deals_created": "created_at", + "pending_pipeline_snapshot": "current_snapshot" + }, + "didar_search_deals_strategy": { + "created_at": { + "Sort": 0, + "SearchFromTime": "start_utc", + "SearchToTime": "end_utc_exclusive", + "confidence": "documented" + }, + "won_at": { + "Sort": 9, + "SearchFromTime": "start_utc", + "SearchToTime": "end_utc_exclusive", + "confidence": "needs_real_account_reconciliation", + "verification": [ + "Every returned row used for a won-period report must have Status == Won.", + "Every returned row used for a won-period report must have ChangeToWonTime inside [start_utc, end_utc_exclusive).", + "If returned rows violate the period basis, do not silently report the aggregate as exact." + ], + "fallback": { + "mode": "exact_local_filter_when_feasible", + "description": "Fetch Won deals in controlled pages and post-filter by ChangeToWonTime when the server date-filter semantics are not reconciled.", + "max_records": 5000, + "over_cap": "Return an accuracy_guard error rather than an unverified exact number." + } + }, + "lost_at": { + "confidence": "not_verified_by_public_contract", + "fallback": "Use Status=Lost and locally filter ChangeToLossTime after controlled pagination until a real-account contract test establishes server-side date behavior." + } + }, + "implementation": { + "recommended_python_dependency": "jdatetime or another tested Jalali conversion library", + "rule": "Pin the chosen Jalali library version and cover Esfand/leap-year/month-boundary behavior with tests." + } +} \ No newline at end of file diff --git a/kamidso/native_connectors/providers/didar/enums.json b/kamidso/native_connectors/providers/didar/enums.json new file mode 100644 index 0000000..9e0e2ff --- /dev/null +++ b/kamidso/native_connectors/providers/didar/enums.json @@ -0,0 +1,67 @@ +{ + "schema_version": "1.0", + "provider": "didar", + "authoritative": { + "deal_status": { + "Pending": { + "canonical": "pending", + "fa": "جاری", + "meaning": "Current/open deal" + }, + "Won": { + "canonical": "won", + "fa": "موفق", + "meaning": "Won/successful deal" + }, + "Lost": { + "canonical": "lost", + "fa": "ناموفق", + "meaning": "Lost/failed deal" + } + }, + "deal_search_sort": { + "0": "register_time", + "1": "last_tracking_time", + "2": "next_tracking_time", + "3": "has_quote", + "4": "has_invoice", + "5": "without_invoice", + "6": "has_void_invoice", + "7": "has_quote_with_price", + "8": "has_quote_without_price", + "9": "won_time", + "10": "next_activity_time" + }, + "custom_field_operators": [ + "NotSet", + "Set", + "EqualToAny", + "NotEqualAny", + "ContainsAnyString", + "NotContainsAnyString", + "StartsWith", + "LessThan", + "GreaterThan", + "Between" + ] + }, + "observed_not_authoritative": { + "contact_type": [ + "General", + "Customer", + "Prospect" + ], + "lead_type_examples": [ + "General" + ], + "lead_status_examples": [ + "None", + "NotHandle" + ] + }, + "rules": [ + "Deal Status is authoritative for current deal status.", + "IsDeleted is a separate deletion dimension and must not be collapsed into deal status.", + "Do not infer Won from PipelineStageId, Probability, labels, or ChangeToWonTime." + ] +} \ No newline at end of file diff --git a/kamidso/native_connectors/providers/didar/function_accuracy_patch.json b/kamidso/native_connectors/providers/didar/function_accuracy_patch.json new file mode 100644 index 0000000..8fb0f17 --- /dev/null +++ b/kamidso/native_connectors/providers/didar/function_accuracy_patch.json @@ -0,0 +1,398 @@ +{ + "schema_version": "1.0", + "provider": "didar", + "purpose": "Merge these accuracy-critical contracts into the existing 49-function catalog. Do not replace correct existing definitions blindly.", + "patches": [ + { + "name": "search_deals", + "method": "POST", + "path": "/api/deal/search_v2", + "effect": "read", + "input_schema": { + "type": "object", + "properties": { + "Criteria": { + "type": "object", + "properties": { + "SearchFromTime": { + "type": "string", + "format": "date-time" + }, + "SearchToTime": { + "type": "string", + "format": "date-time" + }, + "Sort": { + "type": "integer", + "minimum": 0, + "maximum": 10 + }, + "ContactIds": { + "type": "array", + "items": { + "type": "string" + } + }, + "CompanyIds": { + "type": "array", + "items": { + "type": "string" + } + }, + "OtherContactIds": { + "type": "array", + "items": { + "type": "string" + } + }, + "Keywords": { + "type": "string" + }, + "LabelIds": { + "type": "array", + "items": { + "type": "string" + } + }, + "Status": { + "type": "string", + "enum": [ + "Pending", + "Won", + "Lost" + ] + }, + "CustomFields": { + "type": "array", + "items": { + "type": "object", + "properties": { + "CustomFieldId": { + "type": "string" + }, + "OR": { + "type": "array", + "items": { + "type": "object", + "properties": { + "Type": { + "type": "string", + "enum": [ + "NotSet", + "Set", + "EqualToAny", + "NotEqualAny", + "ContainsAnyString", + "NotContainsAnyString", + "StartsWith", + "LessThan", + "GreaterThan", + "Between" + ] + }, + "Value": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "Type", + "Value" + ], + "additionalProperties": false + } + } + }, + "required": [ + "CustomFieldId", + "OR" + ], + "additionalProperties": false + } + }, + "PipelineId": { + "type": "string" + }, + "PipelineStageId": { + "type": "string" + }, + "SourceId": { + "type": "string" + }, + "IsDeleted": { + "type": "boolean" + }, + "HasActivity": { + "type": "boolean" + }, + "LostReasonId": { + "type": "string" + }, + "NextActivityFrom": { + "type": "string", + "format": "date-time" + }, + "NextActivityTo": { + "type": "string", + "format": "date-time" + }, + "IsIdle": { + "type": "boolean" + }, + "IsRotten": { + "type": "boolean" + }, + "Probability": { + "type": "string" + }, + "OwnerIds": { + "type": "array", + "items": { + "type": "string" + } + }, + "PipelineStageIds": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "additionalProperties": false + }, + "From": { + "type": "integer", + "minimum": 0 + }, + "Limit": { + "type": "integer", + "minimum": 1 + } + }, + "required": [ + "Criteria", + "From", + "Limit" + ], + "additionalProperties": false + }, + "response_contract": { + "root": "Response", + "records_path": "Response.List", + "total_count": "Response.TotalCount", + "aggregates": [ + "Response.TotalPrice", + "Response.AllDealsTotalPrice", + "Response.AllDealsCount", + "Response.PendingDealsTotalPrice", + "Response.PendingDealsCount", + "Response.WonDealsTotalPrice", + "Response.WonDealsCount", + "Response.LostDealsTotalPrice", + "Response.LostDealsCount" + ] + }, + "semantic_hints": [ + "From and Limit are top-level fields, not inside Criteria.", + "Status accepts Pending, Won, Lost.", + "Sort=0 means registration time; Sort=9 means won time.", + "Use aggregate counts for totals; Response.List is only the current page.", + "Current status must come from each deal's Status field." + ] + }, + { + "name": "search_persons", + "method": "POST", + "path": "/api/contact/PersonSearch", + "effect": "read", + "input_schema": { + "type": "object", + "properties": { + "Criteria": { + "type": "object", + "properties": { + "IsDeleted": { + "type": "integer", + "enum": [ + 0, + 1 + ] + }, + "IsPinned": { + "type": "integer" + }, + "IsVIP": { + "type": "integer" + }, + "SortOrder": { + "type": "integer" + }, + "Keywords": { + "type": [ + "string", + "null" + ] + }, + "MobilePhone": { + "type": "string" + }, + "WorkPhone": { + "type": "string" + }, + "Email": { + "type": "string" + }, + "NationalCode": { + "type": "string" + }, + "ZipCode": { + "type": "string" + }, + "CustomerCode": { + "type": "string" + }, + "SearchFromTime": { + "type": "string", + "format": "date-time" + }, + "SearchToTime": { + "type": "string", + "format": "date-time" + }, + "CustomFields": { + "type": "array" + } + }, + "additionalProperties": true + }, + "From": { + "type": "integer", + "minimum": 0 + }, + "Limit": { + "type": "integer", + "minimum": 1 + } + }, + "required": [ + "Criteria", + "From", + "Limit" + ], + "additionalProperties": false + }, + "response_contract": { + "root": "Response", + "records_path": "Response.List", + "total_count": "Response.TotalCount" + } + }, + { + "name": "get_users", + "method": "POST", + "path": "/api/User/List", + "effect": "read", + "input_schema": { + "type": "object", + "properties": {}, + "additionalProperties": false + }, + "response_contract": { + "root": "Response", + "records_path": "Response", + "owner_key": "UserId", + "display_field": "DisplayName" + } + }, + { + "name": "get_deal", + "method": "POST", + "path": "/api/deal/getdealdetail", + "effect": "read", + "input_schema": { + "type": "object", + "properties": { + "Id": { + "type": "string" + } + }, + "required": [ + "Id" + ], + "additionalProperties": false + }, + "semantic_hints": [ + "Status is authoritative for current deal status.", + "Do not infer Won merely because ChangeToWonTime has a value; official response examples can contain ChangeToWonTime while Status is Pending." + ] + }, + { + "name": "get_deal_pipelines", + "method": "POST", + "path": "/api/pipeline/list/0", + "effect": "read", + "input_schema": { + "type": "object", + "properties": {}, + "additionalProperties": false + }, + "response_contract": { + "root": "Response", + "records_path": "Response", + "pipeline_id": "Response[].Id", + "stage_id": "Response[].Stages[].Id", + "stage_title": "Response[].Stages[].Title" + } + }, + { + "name": "search_products", + "method": "POST", + "path": "/api/product/search", + "effect": "read", + "input_schema": { + "type": "object", + "properties": { + "Criteria": { + "type": "object", + "properties": { + "Keywords": { + "type": "string" + } + }, + "required": [ + "Keywords" + ], + "additionalProperties": false + }, + "From": { + "type": "integer", + "minimum": 0 + }, + "Limit": { + "type": "integer", + "minimum": 1 + } + }, + "required": [ + "Criteria", + "From", + "Limit" + ], + "additionalProperties": false + } + }, + { + "name": "get_product_categories", + "method": "POST", + "path": "/api/product/categories", + "effect": "read", + "input_schema": { + "type": "object", + "properties": {}, + "additionalProperties": false + } + } + ] +} \ No newline at end of file diff --git a/kamidso/native_connectors/providers/didar/functions.json b/kamidso/native_connectors/providers/didar/functions.json new file mode 100644 index 0000000..9aa950b --- /dev/null +++ b/kamidso/native_connectors/providers/didar/functions.json @@ -0,0 +1,5102 @@ +[ + { + "name": "create_product", + "title": "Create Product", + "description": "Create a product. Disabled in v1.", + "entity": "product", + "effect": "write", + "enabled": false, + "method": "POST", + "path": "/api/product/save", + "input_schema": { + "type": "object", + "description": "Create a product.", + "properties": { + "Product": { + "type": "object", + "description": "Product payload.", + "properties": { + "Title": { + "type": "string", + "description": "Product title." + }, + "Code": { + "type": "string", + "description": "Product code. Official sample sends a numeric-looking string." + }, + "Unit": { + "type": "string", + "description": "Unit name." + }, + "ProductCategoryId": { + "type": "string", + "description": "Category id from get_product_categories." + }, + "TitleForInvoice": { + "type": "string", + "description": "Title printed on invoices." + }, + "Description": { + "type": "string", + "description": "Description." + }, + "UnitPrice": { + "type": "number", + "description": "Unit price." + }, + "IsActive": { + "type": "boolean", + "description": "Whether the product is active." + }, + "Fields": { + "type": "object", + "description": "Product custom fields keyed by field Key.", + "additionalProperties": true + }, + "ProductPrices": { + "type": "array", + "description": "Prices per currency.", + "items": { + "type": "object", + "description": "One currency price.", + "properties": { + "CurrencyId": { + "type": "string", + "description": "Currency id." + }, + "Price": { + "type": "number", + "description": "Price in that currency." + }, + "decimalLength": { + "type": "number", + "description": "Decimal length." + } + }, + "required": [ + "CurrencyId", + "Price" + ] + } + }, + "Variants": { + "type": "array", + "description": "Product variants.", + "items": { + "type": "object", + "description": "One product variant.", + "properties": { + "Title": { + "type": "string", + "description": "Variant title." + }, + "TitleForInvoice": { + "type": "string", + "description": "Variant title on invoices." + }, + "IsDeleted": { + "type": "boolean", + "description": "Whether the variant is deleted." + }, + "IsDefault": { + "type": "boolean", + "description": "Whether this is the default variant." + }, + "UnitPrice": { + "type": "number", + "description": "Variant unit price." + }, + "VariantCode": { + "type": "string", + "description": "Variant code." + }, + "Fields": { + "type": "object", + "description": "Variant custom fields.", + "additionalProperties": true + } + }, + "required": [ + "Title", + "TitleForInvoice", + "IsDefault", + "UnitPrice", + "VariantCode" + ] + } + } + }, + "required": [ + "Title", + "Unit", + "ProductCategoryId" + ] + } + }, + "required": [ + "Product" + ] + }, + "tags": [ + "product" + ], + "content_type": "application/json", + "response_root": "Response", + "timeout_seconds": 30 + }, + { + "name": "search_products", + "title": "Search Products", + "description": "Search products by title keywords with From/Limit pagination.", + "entity": "product", + "effect": "read", + "enabled": true, + "method": "POST", + "path": "/api/product/search", + "input_schema": { + "type": "object", + "description": "Search products.", + "properties": { + "Criteria": { + "type": "object", + "description": "Product search criteria.", + "properties": { + "Keywords": { + "type": "string", + "description": "Title text to match." + } + } + }, + "From": { + "type": "integer", + "description": "Zero-based offset." + }, + "Limit": { + "type": "integer", + "description": "Page size." + } + } + }, + "pagination": { + "type": "offset_limit", + "offset_field": "From", + "limit_field": "Limit" + }, + "tags": [ + "product" + ], + "content_type": "application/json", + "response_root": "Response", + "timeout_seconds": 30 + }, + { + "name": "get_product_categories", + "title": "Get Product Categories", + "description": "List product categories. Official docs: no request body.", + "entity": "product", + "effect": "read", + "enabled": true, + "method": "POST", + "path": "/api/product/categories", + "input_schema": { + "type": "object", + "description": "Official Didar docs: this request has no body.", + "additionalProperties": false + }, + "tags": [ + "product" + ], + "content_type": "application/json", + "response_root": "Response", + "timeout_seconds": 30 + }, + { + "name": "get_products_by_code", + "title": "Get Products By Code", + "description": "Fetch products by exact product codes.", + "entity": "product", + "effect": "read", + "enabled": true, + "method": "POST", + "path": "/api/product/getproductbycodes", + "input_schema": { + "type": "object", + "description": "Products by code.", + "properties": { + "Code": { + "type": "array", + "description": "Product codes.", + "items": { + "type": "string", + "description": "Exact product code." + } + } + }, + "required": [ + "Code" + ] + }, + "tags": [ + "product" + ], + "content_type": "application/json", + "response_root": "Response", + "timeout_seconds": 30 + }, + { + "name": "get_activity_types", + "title": "Get Activity Types", + "description": "List activity types. Official docs: no request body.", + "entity": "activity", + "effect": "read", + "enabled": true, + "method": "POST", + "path": "/api/activity/GetActivityType", + "input_schema": { + "type": "object", + "description": "Official Didar docs: this request has no body.", + "additionalProperties": false + }, + "tags": [ + "activity" + ], + "content_type": "application/json", + "response_root": "Response", + "timeout_seconds": 30 + }, + { + "name": "create_activity", + "title": "Create Activity", + "description": "Create an activity. Disabled in v1.", + "entity": "activity", + "effect": "write", + "enabled": false, + "method": "POST", + "path": "/api/activity/save", + "input_schema": { + "type": "object", + "description": "Create or update an activity.", + "properties": { + "Activity": { + "type": "object", + "description": "Activity payload.", + "properties": { + "ActivityTypeId": { + "type": "string", + "description": "Activity type id from get_activity_types." + }, + "Title": { + "type": "string", + "description": "Activity title." + }, + "OwnerId": { + "type": "string", + "description": "UserId of the owner." + }, + "Note": { + "type": "string", + "description": "Activity note." + }, + "IsDone": { + "type": "boolean", + "description": "Whether the activity is done." + }, + "ResultNote": { + "type": "string", + "description": "Result note." + }, + "IsDeleted": { + "type": "boolean", + "description": "Whether the activity is deleted." + }, + "IsPinned": { + "type": "boolean", + "description": "Whether the activity is pinned." + }, + "DueDate": { + "type": "string", + "description": "Due date/time." + }, + "Duration": { + "type": "number", + "description": "Duration." + }, + "DoneDate": { + "type": "string", + "description": "Done date/time." + }, + "DealId": { + "type": "string", + "description": "Related deal id." + }, + "CaseId": { + "type": "string", + "description": "Related case id." + }, + "ContactIds": { + "type": "array", + "description": "Related contact ids.", + "items": { + "type": "string", + "description": "Contact id." + } + }, + "CreatorId": { + "type": "string", + "description": "UserId of the creator." + }, + "RecurrenceType": { + "type": "string", + "description": "Recurrence type. Official sample uses OneTime." + }, + "RecurrenceData": { + "type": "number", + "description": "Recurrence data." + }, + "RecurrenceEndDate": { + "type": "string", + "description": "Recurrence end date/time." + }, + "Tags": { + "type": "array", + "description": "Tags.", + "items": { + "type": "object", + "description": "One tag reference.", + "properties": { + "TagId": { + "type": "string", + "description": "Tag id from get_tags." + } + } + } + } + }, + "required": [ + "ActivityTypeId", + "Title" + ] + } + }, + "required": [ + "Activity" + ] + }, + "tags": [ + "activity" + ], + "content_type": "application/json", + "response_root": "Response", + "timeout_seconds": 30 + }, + { + "name": "update_activity", + "title": "Update Activity", + "description": "Update an activity. Disabled in v1.", + "entity": "activity", + "effect": "write", + "enabled": false, + "method": "POST", + "path": "/api/activity/save", + "input_schema": { + "type": "object", + "description": "Create or update an activity.", + "properties": { + "Activity": { + "type": "object", + "description": "Activity payload.", + "properties": { + "Id": { + "type": "string", + "description": "Activity id." + }, + "ActivityTypeId": { + "type": "string", + "description": "Activity type id from get_activity_types." + }, + "Title": { + "type": "string", + "description": "Activity title." + }, + "OwnerId": { + "type": "string", + "description": "UserId of the owner." + }, + "Note": { + "type": "string", + "description": "Activity note." + }, + "IsDone": { + "type": "boolean", + "description": "Whether the activity is done." + }, + "ResultNote": { + "type": "string", + "description": "Result note." + }, + "IsDeleted": { + "type": "boolean", + "description": "Whether the activity is deleted." + }, + "IsPinned": { + "type": "boolean", + "description": "Whether the activity is pinned." + }, + "DueDate": { + "type": "string", + "description": "Due date/time." + }, + "Duration": { + "type": "number", + "description": "Duration." + }, + "DoneDate": { + "type": "string", + "description": "Done date/time." + }, + "DealId": { + "type": "string", + "description": "Related deal id." + }, + "CaseId": { + "type": "string", + "description": "Related case id." + }, + "ContactIds": { + "type": "array", + "description": "Related contact ids.", + "items": { + "type": "string", + "description": "Contact id." + } + }, + "CreatorId": { + "type": "string", + "description": "UserId of the creator." + }, + "RecurrenceType": { + "type": "string", + "description": "Recurrence type. Official sample uses OneTime." + }, + "RecurrenceData": { + "type": "number", + "description": "Recurrence data." + }, + "RecurrenceEndDate": { + "type": "string", + "description": "Recurrence end date/time." + }, + "Tags": { + "type": "array", + "description": "Tags.", + "items": { + "type": "object", + "description": "One tag reference.", + "properties": { + "TagId": { + "type": "string", + "description": "Tag id from get_tags." + } + } + } + } + }, + "required": [ + "Id", + "ActivityTypeId", + "Title" + ] + } + }, + "required": [ + "Activity" + ] + }, + "tags": [ + "activity" + ], + "content_type": "application/json", + "response_root": "Response", + "timeout_seconds": 30 + }, + { + "name": "search_activities", + "title": "Search Activities", + "description": "Search activities by time range, type, contact, owner, tag, and related records.", + "entity": "activity", + "effect": "read", + "enabled": true, + "method": "POST", + "path": "/api/activity/search", + "input_schema": { + "type": "object", + "description": "Search activities. Filters are top-level, not inside Criteria.", + "properties": { + "FromDate": { + "type": "string", + "description": "Range start." + }, + "ToDate": { + "type": "string", + "description": "Range end." + }, + "SortType": { + "type": "integer", + "description": "Sort mode." + }, + "From": { + "type": "integer", + "description": "Zero-based offset." + }, + "Limit": { + "type": "integer", + "description": "Page size." + }, + "ActivityTypeIds": { + "type": "array", + "description": "Activity type filter.", + "items": { + "type": "string", + "description": "Activity type id." + } + }, + "ContactIds": { + "type": "array", + "description": "Related contacts.", + "items": { + "type": "string", + "description": "Person or company id." + } + }, + "TagIds": { + "type": "array", + "description": "Tag filter.", + "items": { + "type": "string", + "description": "Tag id." + } + }, + "OwnerId": { + "type": "string", + "description": "Owner UserId." + }, + "CreatorId": { + "type": "string", + "description": "Creator UserId." + }, + "IsDone": { + "type": "boolean", + "description": "true returns done activities. false returns not-done activities." + }, + "IsDeleted": { + "type": "boolean", + "description": "true returns deleted activities. false returns active activities." + }, + "DealId": { + "type": "string", + "description": "Related deal id." + }, + "CaseId": { + "type": "string", + "description": "Related case id." + } + } + }, + "pagination": { + "type": "offset_limit", + "offset_field": "From", + "limit_field": "Limit" + }, + "tags": [ + "activity" + ], + "content_type": "application/json", + "response_root": "Response", + "timeout_seconds": 30 + }, + { + "name": "attach_file_to_activity", + "title": "Attach File To Activity", + "description": "Attach files to an activity with multipart form fields activityId and uploads. Disabled in v1.", + "entity": "activity", + "effect": "write", + "enabled": false, + "method": "POST", + "path": "/api/activity/AttachFilesToActivity", + "content_type": "multipart/form-data", + "input_schema": { + "type": "object", + "description": "Multipart attachment.", + "properties": { + "activityId": { + "type": "string", + "description": "Activity id." + }, + "uploads": { + "type": "string", + "description": "File upload. The official request is multipart/form-data." + } + }, + "required": [ + "activityId", + "uploads" + ] + }, + "tags": [ + "activity" + ], + "response_root": "Response", + "timeout_seconds": 30 + }, + { + "name": "get_activity", + "title": "Get Activity", + "description": "Get one activity by id.", + "entity": "activity", + "effect": "read", + "enabled": true, + "method": "POST", + "path": "/api/Activity/GetActivityAndNoteById", + "input_schema": { + "type": "object", + "description": "Get activity.", + "properties": { + "Id": { + "type": "string", + "description": "Activity id." + } + }, + "required": [ + "Id" + ] + }, + "tags": [ + "activity" + ], + "content_type": "application/json", + "response_root": "Response", + "timeout_seconds": 30 + }, + { + "name": "create_note", + "title": "Create Note", + "description": "Create a note through the activity save endpoint. Disabled in v1.", + "entity": "note", + "effect": "write", + "enabled": false, + "method": "POST", + "path": "/api/activity/save", + "input_schema": { + "type": "object", + "description": "Create or update a note.", + "properties": { + "Activity": { + "type": "object", + "description": "Note payload stored through the activity endpoint.", + "properties": { + "ActivityTypeId": { + "type": "string", + "description": "Activity type id. Official note samples use the zero GUID." + }, + "OwnerId": { + "type": "string", + "description": "UserId of the owner." + }, + "ResultNote": { + "type": "string", + "description": "Note text." + }, + "IsDone": { + "type": "boolean", + "description": "Official note docs require true for a note." + }, + "IsPinned": { + "type": "boolean", + "description": "Whether the note is pinned." + }, + "IsDeleted": { + "type": "boolean", + "description": "Whether the note is deleted." + }, + "DealId": { + "type": "string", + "description": "Related deal id." + }, + "CaseId": { + "type": "string", + "description": "Related case id." + }, + "ContactIds": { + "type": "array", + "description": "Related person or company ids.", + "items": { + "type": "string", + "description": "Contact id." + } + }, + "Tags": { + "type": "array", + "description": "Tags.", + "items": { + "type": "object", + "description": "One tag reference.", + "properties": { + "TagId": { + "type": "string", + "description": "Tag id from get_tags." + } + } + } + } + }, + "required": [ + "ActivityTypeId", + "IsDone" + ] + } + }, + "required": [ + "Activity" + ] + }, + "tags": [ + "note" + ], + "content_type": "application/json", + "response_root": "Response", + "timeout_seconds": 30 + }, + { + "name": "update_note", + "title": "Update Note", + "description": "Update a note through the activity save endpoint. Disabled in v1.", + "entity": "note", + "effect": "write", + "enabled": false, + "method": "POST", + "path": "/api/activity/save", + "input_schema": { + "type": "object", + "description": "Create or update a note.", + "properties": { + "Activity": { + "type": "object", + "description": "Note payload stored through the activity endpoint.", + "properties": { + "Id": { + "type": "string", + "description": "Note id." + }, + "ActivityTypeId": { + "type": "string", + "description": "Activity type id. Official note samples use the zero GUID." + }, + "OwnerId": { + "type": "string", + "description": "UserId of the owner." + }, + "ResultNote": { + "type": "string", + "description": "Note text." + }, + "IsDone": { + "type": "boolean", + "description": "Official note docs require true for a note." + }, + "IsPinned": { + "type": "boolean", + "description": "Whether the note is pinned." + }, + "IsDeleted": { + "type": "boolean", + "description": "Whether the note is deleted." + }, + "DealId": { + "type": "string", + "description": "Related deal id." + }, + "CaseId": { + "type": "string", + "description": "Related case id." + }, + "ContactIds": { + "type": "array", + "description": "Related person or company ids.", + "items": { + "type": "string", + "description": "Contact id." + } + }, + "Tags": { + "type": "array", + "description": "Tags.", + "items": { + "type": "object", + "description": "One tag reference.", + "properties": { + "TagId": { + "type": "string", + "description": "Tag id from get_tags." + } + } + } + } + }, + "required": [ + "Id", + "ActivityTypeId", + "IsDone" + ] + } + }, + "required": [ + "Activity" + ] + }, + "tags": [ + "note" + ], + "content_type": "application/json", + "response_root": "Response", + "timeout_seconds": 30 + }, + { + "name": "attach_file_to_note", + "title": "Attach File To Note", + "description": "Attach files to a note using the activity attachment endpoint. Disabled in v1.", + "entity": "note", + "effect": "write", + "enabled": false, + "method": "POST", + "path": "/api/activity/AttachFilesToActivity", + "content_type": "multipart/form-data", + "input_schema": { + "type": "object", + "description": "Multipart note attachment.", + "properties": { + "activityId": { + "type": "string", + "description": "Note id. Official docs send it as activityId." + }, + "uploads": { + "type": "string", + "description": "File upload. The official request is multipart/form-data." + } + }, + "required": [ + "activityId", + "uploads" + ] + }, + "tags": [ + "note" + ], + "response_root": "Response", + "timeout_seconds": 30 + }, + { + "name": "get_note", + "title": "Get Note", + "description": "Get one note by id.", + "entity": "note", + "effect": "read", + "enabled": true, + "method": "POST", + "path": "/api/Activity/GetActivityAndNoteById", + "input_schema": { + "type": "object", + "description": "Get note.", + "properties": { + "Id": { + "type": "string", + "description": "Note id." + } + }, + "required": [ + "Id" + ] + }, + "tags": [ + "note" + ], + "content_type": "application/json", + "response_root": "Response", + "timeout_seconds": 30 + }, + { + "name": "didaryab_search", + "title": "Didaryab Search", + "description": "Search contacts, companies, deals, cases, and attachments with one keyword.", + "entity": "search", + "effect": "read", + "enabled": true, + "method": "POST", + "path": "/api/search/search", + "input_schema": { + "type": "object", + "description": "Didaryab search.", + "properties": { + "Keyword": { + "type": "string", + "description": "Text to search." + }, + "Types": { + "type": "array", + "description": "One or more record types.", + "items": { + "type": "string", + "description": "Record type.", + "enum": [ + "contact", + "company", + "deal", + "case", + "attachment" + ] + } + } + }, + "required": [ + "Keyword", + "Types" + ] + }, + "tags": [ + "search" + ], + "content_type": "application/json", + "response_root": "Response", + "timeout_seconds": 30 + }, + { + "name": "get_custom_fields", + "title": "Get Custom Fields", + "description": "List custom fields. Official docs: no request body.", + "entity": "custom_field", + "effect": "read", + "enabled": true, + "method": "POST", + "path": "/api/customfield/GetCustomfieldList", + "input_schema": { + "type": "object", + "description": "Official Didar docs: this request has no body.", + "additionalProperties": false + }, + "tags": [ + "custom_field" + ], + "content_type": "application/json", + "response_root": "Response", + "timeout_seconds": 30 + }, + { + "name": "get_users", + "title": "Get Users", + "description": "List Didar users. Official docs: POST with no request body. Used to test the API key.", + "entity": "user", + "effect": "read", + "enabled": true, + "method": "POST", + "path": "/api/User/List", + "input_schema": { + "type": "object", + "description": "Official Didar docs: this request has no body.", + "additionalProperties": false + }, + "tags": [ + "user" + ], + "content_type": "application/json", + "response_root": "Response", + "timeout_seconds": 30 + }, + { + "name": "create_person", + "title": "Create Person", + "description": "Create a person. Disabled in v1.", + "entity": "person", + "effect": "write", + "enabled": false, + "method": "POST", + "path": "/api/contact/save", + "input_schema": { + "type": "object", + "description": "Create a person.", + "properties": { + "Contact": { + "type": "object", + "description": "Person contact payload.", + "properties": { + "Type": { + "type": "string", + "description": "Person contact type.", + "enum": [ + "Person" + ] + }, + "FirstName": { + "type": "string", + "description": "Given name." + }, + "LastName": { + "type": "string", + "description": "Family name. Required by the official create/update person docs." + }, + "Title": { + "type": "string", + "description": "Salutation or title." + }, + "OwnerId": { + "type": "string", + "description": "UserId of the owner from get_users." + }, + "CompanyName": { + "type": "string", + "description": "Related company name." + }, + "CompanyId": { + "type": "string", + "description": "Id of the related company." + }, + "BirthDate": { + "type": "string", + "description": "Birth date/time." + }, + "BirthDateMessageId": { + "type": "string", + "description": "Birth-date message id." + }, + "MobilePhone": { + "type": "string", + "description": "Mobile phone." + }, + "WorkPhone": { + "type": "string", + "description": "Work phone." + }, + "WorkPhoneExtension": { + "type": "string", + "description": "Work phone extension." + }, + "Fax": { + "type": "string", + "description": "Fax." + }, + "Email": { + "type": "string", + "description": "Email." + }, + "IsVIP": { + "type": "boolean", + "description": "Whether the person is marked VIP." + }, + "BackgroundInfo": { + "type": "string", + "description": "Background notes." + }, + "Position": { + "type": "string", + "description": "Job title." + }, + "CustomerCode": { + "type": "string", + "description": "Customer code." + }, + "NationalCode": { + "type": "string", + "description": "National code." + }, + "ZipCode": { + "type": "string", + "description": "Postal code." + }, + "CityId": { + "type": "string", + "description": "City id from get_locations." + }, + "ProvinceId": { + "type": "string", + "description": "Province id from get_locations." + }, + "VisibilityType": { + "type": "string", + "description": "Visibility. Documented examples include Owner, OwnerGroup, OwnerSubGroup, All." + }, + "Fields": { + "type": "object", + "description": "Custom field values keyed by the field Key from get_custom_fields.", + "additionalProperties": true + }, + "OtherPhones": { + "type": "object", + "description": "Grouped key/value collection documented on contact save.", + "properties": { + "KeyValues": { + "type": "array", + "description": "Labeled key/value rows.", + "items": { + "type": "object", + "description": "One labeled value.", + "properties": { + "Key": { + "type": "string", + "description": "Label key." + }, + "Value": { + "type": "string", + "description": "Value." + }, + "Title": { + "type": "string", + "description": "Optional display title." + } + } + } + } + } + }, + "OtherEmails": { + "type": "object", + "description": "Grouped key/value collection documented on contact save.", + "properties": { + "KeyValues": { + "type": "array", + "description": "Labeled key/value rows.", + "items": { + "type": "object", + "description": "One labeled value.", + "properties": { + "Key": { + "type": "string", + "description": "Label key." + }, + "Value": { + "type": "string", + "description": "Value." + }, + "Title": { + "type": "string", + "description": "Optional display title." + } + } + } + } + } + }, + "Websites": { + "type": "object", + "description": "Grouped key/value collection documented on contact save.", + "properties": { + "KeyValues": { + "type": "array", + "description": "Labeled key/value rows.", + "items": { + "type": "object", + "description": "One labeled value.", + "properties": { + "Key": { + "type": "string", + "description": "Label key." + }, + "Value": { + "type": "string", + "description": "Value." + }, + "Title": { + "type": "string", + "description": "Optional display title." + } + } + } + } + } + }, + "BankAccounts": { + "type": "object", + "description": "Bank accounts object.", + "properties": { + "Values": { + "type": "array", + "description": "Bank account rows.", + "items": { + "type": "object", + "description": "One bank-account row.", + "properties": { + "Value1": { + "type": "string", + "description": "Account row type, such as Card or Account." + }, + "Value2": { + "type": "string", + "description": "Second documented value slot." + }, + "Value3": { + "type": "string", + "description": "Third documented value slot." + }, + "Value4": { + "type": "string", + "description": "Fourth documented value slot." + } + } + } + } + } + }, + "Addresses": { + "type": "object", + "description": "Grouped key/value collection documented on contact save.", + "properties": { + "KeyValues": { + "type": "array", + "description": "Labeled key/value rows.", + "items": { + "type": "object", + "description": "One labeled value.", + "properties": { + "Key": { + "type": "string", + "description": "Label key." + }, + "Value": { + "type": "string", + "description": "Value." + }, + "Title": { + "type": "string", + "description": "Optional display title." + } + } + } + } + } + } + }, + "required": [ + "LastName", + "OwnerId" + ] + }, + "SegmentIds": { + "type": "array", + "description": "Segment ids.", + "items": { + "type": "string", + "description": "Segment id from get_segments." + } + } + }, + "required": [ + "Contact" + ] + }, + "tags": [ + "person" + ], + "content_type": "application/json", + "response_root": "Response", + "timeout_seconds": 30 + }, + { + "name": "update_person", + "title": "Update Person", + "description": "Update a person. Disabled in v1.", + "entity": "person", + "effect": "write", + "enabled": false, + "method": "POST", + "path": "/api/contact/save", + "input_schema": { + "type": "object", + "description": "Update a person.", + "properties": { + "Contact": { + "type": "object", + "description": "Person contact payload.", + "properties": { + "Id": { + "type": "string", + "description": "Person id." + }, + "Type": { + "type": "string", + "description": "Person contact type.", + "enum": [ + "Person" + ] + }, + "FirstName": { + "type": "string", + "description": "Given name." + }, + "LastName": { + "type": "string", + "description": "Family name. Required by the official create/update person docs." + }, + "Title": { + "type": "string", + "description": "Salutation or title." + }, + "OwnerId": { + "type": "string", + "description": "UserId of the owner from get_users." + }, + "CompanyName": { + "type": "string", + "description": "Related company name." + }, + "CompanyId": { + "type": "string", + "description": "Id of the related company." + }, + "BirthDate": { + "type": "string", + "description": "Birth date/time." + }, + "BirthDateMessageId": { + "type": "string", + "description": "Birth-date message id." + }, + "MobilePhone": { + "type": "string", + "description": "Mobile phone." + }, + "WorkPhone": { + "type": "string", + "description": "Work phone." + }, + "WorkPhoneExtension": { + "type": "string", + "description": "Work phone extension." + }, + "Fax": { + "type": "string", + "description": "Fax." + }, + "Email": { + "type": "string", + "description": "Email." + }, + "IsVIP": { + "type": "boolean", + "description": "Whether the person is marked VIP." + }, + "BackgroundInfo": { + "type": "string", + "description": "Background notes." + }, + "Position": { + "type": "string", + "description": "Job title." + }, + "CustomerCode": { + "type": "string", + "description": "Customer code." + }, + "NationalCode": { + "type": "string", + "description": "National code." + }, + "ZipCode": { + "type": "string", + "description": "Postal code." + }, + "CityId": { + "type": "string", + "description": "City id from get_locations." + }, + "ProvinceId": { + "type": "string", + "description": "Province id from get_locations." + }, + "VisibilityType": { + "type": "string", + "description": "Visibility. Documented examples include Owner, OwnerGroup, OwnerSubGroup, All." + }, + "Fields": { + "type": "object", + "description": "Custom field values keyed by the field Key from get_custom_fields.", + "additionalProperties": true + }, + "OtherPhones": { + "type": "object", + "description": "Grouped key/value collection documented on contact save.", + "properties": { + "KeyValues": { + "type": "array", + "description": "Labeled key/value rows.", + "items": { + "type": "object", + "description": "One labeled value.", + "properties": { + "Key": { + "type": "string", + "description": "Label key." + }, + "Value": { + "type": "string", + "description": "Value." + }, + "Title": { + "type": "string", + "description": "Optional display title." + } + } + } + } + } + }, + "OtherEmails": { + "type": "object", + "description": "Grouped key/value collection documented on contact save.", + "properties": { + "KeyValues": { + "type": "array", + "description": "Labeled key/value rows.", + "items": { + "type": "object", + "description": "One labeled value.", + "properties": { + "Key": { + "type": "string", + "description": "Label key." + }, + "Value": { + "type": "string", + "description": "Value." + }, + "Title": { + "type": "string", + "description": "Optional display title." + } + } + } + } + } + }, + "Websites": { + "type": "object", + "description": "Grouped key/value collection documented on contact save.", + "properties": { + "KeyValues": { + "type": "array", + "description": "Labeled key/value rows.", + "items": { + "type": "object", + "description": "One labeled value.", + "properties": { + "Key": { + "type": "string", + "description": "Label key." + }, + "Value": { + "type": "string", + "description": "Value." + }, + "Title": { + "type": "string", + "description": "Optional display title." + } + } + } + } + } + }, + "BankAccounts": { + "type": "object", + "description": "Bank accounts object.", + "properties": { + "Values": { + "type": "array", + "description": "Bank account rows.", + "items": { + "type": "object", + "description": "One bank-account row.", + "properties": { + "Value1": { + "type": "string", + "description": "Account row type, such as Card or Account." + }, + "Value2": { + "type": "string", + "description": "Second documented value slot." + }, + "Value3": { + "type": "string", + "description": "Third documented value slot." + }, + "Value4": { + "type": "string", + "description": "Fourth documented value slot." + } + } + } + } + } + }, + "Addresses": { + "type": "object", + "description": "Grouped key/value collection documented on contact save.", + "properties": { + "KeyValues": { + "type": "array", + "description": "Labeled key/value rows.", + "items": { + "type": "object", + "description": "One labeled value.", + "properties": { + "Key": { + "type": "string", + "description": "Label key." + }, + "Value": { + "type": "string", + "description": "Value." + }, + "Title": { + "type": "string", + "description": "Optional display title." + } + } + } + } + } + } + }, + "required": [ + "Id", + "LastName", + "OwnerId" + ] + }, + "SegmentIds": { + "type": "array", + "description": "Segment ids.", + "items": { + "type": "string", + "description": "Segment id from get_segments." + } + } + }, + "required": [ + "Contact" + ] + }, + "tags": [ + "person" + ], + "content_type": "application/json", + "response_root": "Response", + "timeout_seconds": 30 + }, + { + "name": "get_person_by_id", + "title": "Get Person By Id", + "description": "Get one person by id.", + "entity": "person", + "effect": "read", + "enabled": true, + "method": "POST", + "path": "/api/contact/GetContactDetail", + "input_schema": { + "type": "object", + "description": "Get person.", + "properties": { + "Id": { + "type": "string", + "description": "Person id." + } + }, + "required": [ + "Id" + ] + }, + "tags": [ + "person" + ], + "content_type": "application/json", + "response_root": "Response", + "timeout_seconds": 30 + }, + { + "name": "get_person_by_mobile", + "title": "Get Person By Mobile", + "description": "Find a person by the stored mobile number.", + "entity": "person", + "effect": "read", + "enabled": true, + "method": "POST", + "path": "/api/contact/getbyphonenumber", + "input_schema": { + "type": "object", + "description": "Find person by mobile.", + "properties": { + "MobilePhone": { + "type": "string", + "description": "Mobile number in the stored format." + } + }, + "required": [ + "MobilePhone" + ] + }, + "tags": [ + "person" + ], + "content_type": "application/json", + "response_root": "Response", + "timeout_seconds": 30 + }, + { + "name": "search_persons", + "title": "Search Persons", + "description": "Search persons by name, phone, email, codes, and registration time.", + "entity": "person", + "effect": "read", + "enabled": true, + "method": "POST", + "path": "/api/contact/PersonSearch", + "input_schema": { + "type": "object", + "description": "Search persons.", + "properties": { + "Criteria": { + "type": "object", + "description": "Person search criteria from the official Search Persons page.", + "properties": { + "IsDeleted": { + "type": "integer", + "description": "0 keeps active records. 1 keeps deleted records.", + "enum": [ + 0, + 1 + ] + }, + "IsPinned": { + "type": "integer", + "description": "-1 is not pinned. 1 is pinned.", + "enum": [ + -1, + 1 + ] + }, + "IsVIP": { + "type": "integer", + "description": "-1 is not VIP. 1 is VIP.", + "enum": [ + -1, + 1 + ] + }, + "SortOrder": { + "type": "integer", + "description": "Sort direction." + }, + "Keywords": { + "type": "string", + "description": "Text matched against the documented name fields." + }, + "MobilePhone": { + "type": "string", + "description": "Mobile phone filter." + }, + "WorkPhone": { + "type": "string", + "description": "Work phone filter." + }, + "Email": { + "type": "string", + "description": "Email filter." + }, + "NationalCode": { + "type": "string", + "description": "National code filter shown in the official sample." + }, + "ZipCode": { + "type": "string", + "description": "Postal code filter." + }, + "CustomerCode": { + "type": "string", + "description": "Customer code filter." + }, + "SearchFromTime": { + "type": "string", + "description": "Inclusive range start." + }, + "SearchToTime": { + "type": "string", + "description": "Inclusive range end." + }, + "CustomFields": { + "type": "array", + "description": "Custom-field filters.", + "items": { + "type": "object", + "description": "Official person/company search sample sends objects whose keys are custom-field keys such as Field_996_0_226.", + "additionalProperties": true + } + } + } + }, + "From": { + "type": "integer", + "description": "Zero-based offset." + }, + "Limit": { + "type": "integer", + "description": "Page size." + } + } + }, + "pagination": { + "type": "offset_limit", + "offset_field": "From", + "limit_field": "Limit" + }, + "tags": [ + "person" + ], + "content_type": "application/json", + "response_root": "Response", + "timeout_seconds": 30 + }, + { + "name": "create_company", + "title": "Create Company", + "description": "Create a company. Disabled in v1.", + "entity": "company", + "effect": "write", + "enabled": false, + "method": "POST", + "path": "/api/contact/save", + "input_schema": { + "type": "object", + "description": "Create a company.", + "properties": { + "Contact": { + "type": "object", + "description": "Company contact payload.", + "properties": { + "Type": { + "type": "string", + "description": "Must be Company.", + "enum": [ + "Company" + ] + }, + "FirstName": { + "type": "string", + "description": "Company name. Required by the official company docs." + }, + "Title": { + "type": "string", + "description": "Title." + }, + "OwnerId": { + "type": "string", + "description": "UserId of the owner from get_users." + }, + "BirthDate": { + "type": "string", + "description": "Date field accepted by the official company sample." + }, + "BirthDateMessageId": { + "type": "string", + "description": "Message id accepted by the official company sample." + }, + "MobilePhone": { + "type": "string", + "description": "Mobile or secretary phone." + }, + "WorkPhone": { + "type": "string", + "description": "Work phone." + }, + "WorkPhoneExtension": { + "type": "string", + "description": "Work phone extension." + }, + "Fax": { + "type": "string", + "description": "Fax." + }, + "Email": { + "type": "string", + "description": "Email." + }, + "IsVIP": { + "type": "boolean", + "description": "Whether the company is marked VIP." + }, + "BackgroundInfo": { + "type": "string", + "description": "Background notes." + }, + "Position": { + "type": "string", + "description": "Position field accepted by the official company sample." + }, + "CustomerCode": { + "type": "string", + "description": "Customer code." + }, + "EconomicIssue": { + "type": "string", + "description": "Economic code." + }, + "RegistrationNumber": { + "type": "string", + "description": "Company registration number." + }, + "NationalID": { + "type": "string", + "description": "Company national id." + }, + "ZipCode": { + "type": "string", + "description": "Postal code." + }, + "CityId": { + "type": "string", + "description": "City id from get_locations." + }, + "ProvinceId": { + "type": "string", + "description": "Province id from get_locations." + }, + "VisibilityType": { + "type": "string", + "description": "Visibility. Documented sample uses Owner." + }, + "Fields": { + "type": "object", + "description": "Custom field values keyed by field Key.", + "additionalProperties": true + }, + "OtherPhones": { + "type": "object", + "description": "Grouped key/value collection documented on contact save.", + "properties": { + "KeyValues": { + "type": "array", + "description": "Labeled key/value rows.", + "items": { + "type": "object", + "description": "One labeled value.", + "properties": { + "Key": { + "type": "string", + "description": "Label key." + }, + "Value": { + "type": "string", + "description": "Value." + }, + "Title": { + "type": "string", + "description": "Optional display title." + } + } + } + } + } + }, + "OtherEmails": { + "type": "object", + "description": "Grouped key/value collection documented on contact save.", + "properties": { + "KeyValues": { + "type": "array", + "description": "Labeled key/value rows.", + "items": { + "type": "object", + "description": "One labeled value.", + "properties": { + "Key": { + "type": "string", + "description": "Label key." + }, + "Value": { + "type": "string", + "description": "Value." + }, + "Title": { + "type": "string", + "description": "Optional display title." + } + } + } + } + } + }, + "Websites": { + "type": "object", + "description": "Grouped key/value collection documented on contact save.", + "properties": { + "KeyValues": { + "type": "array", + "description": "Labeled key/value rows.", + "items": { + "type": "object", + "description": "One labeled value.", + "properties": { + "Key": { + "type": "string", + "description": "Label key." + }, + "Value": { + "type": "string", + "description": "Value." + }, + "Title": { + "type": "string", + "description": "Optional display title." + } + } + } + } + } + }, + "BankAccounts": { + "type": "object", + "description": "Bank accounts object.", + "properties": { + "Values": { + "type": "array", + "description": "Bank account rows.", + "items": { + "type": "object", + "description": "One bank-account row.", + "properties": { + "Value1": { + "type": "string", + "description": "Account row type, such as Card or Account." + }, + "Value2": { + "type": "string", + "description": "Second documented value slot." + }, + "Value3": { + "type": "string", + "description": "Third documented value slot." + }, + "Value4": { + "type": "string", + "description": "Fourth documented value slot." + } + } + } + } + } + }, + "Addresses": { + "type": "object", + "description": "Grouped key/value collection documented on contact save.", + "properties": { + "KeyValues": { + "type": "array", + "description": "Labeled key/value rows.", + "items": { + "type": "object", + "description": "One labeled value.", + "properties": { + "Key": { + "type": "string", + "description": "Label key." + }, + "Value": { + "type": "string", + "description": "Value." + }, + "Title": { + "type": "string", + "description": "Optional display title." + } + } + } + } + } + } + }, + "required": [ + "Type", + "FirstName", + "OwnerId" + ] + }, + "SegmentIds": { + "type": "array", + "description": "Segment ids.", + "items": { + "type": "string", + "description": "Segment id from get_segments." + } + } + }, + "required": [ + "Contact" + ] + }, + "tags": [ + "company" + ], + "content_type": "application/json", + "response_root": "Response", + "timeout_seconds": 30 + }, + { + "name": "update_company", + "title": "Update Company", + "description": "Update a company. Disabled in v1.", + "entity": "company", + "effect": "write", + "enabled": false, + "method": "POST", + "path": "/api/contact/save", + "input_schema": { + "type": "object", + "description": "Update a company.", + "properties": { + "Contact": { + "type": "object", + "description": "Company contact payload.", + "properties": { + "Id": { + "type": "string", + "description": "Company id." + }, + "Type": { + "type": "string", + "description": "Must be Company.", + "enum": [ + "Company" + ] + }, + "FirstName": { + "type": "string", + "description": "Company name. Required by the official company docs." + }, + "Title": { + "type": "string", + "description": "Title." + }, + "OwnerId": { + "type": "string", + "description": "UserId of the owner from get_users." + }, + "BirthDate": { + "type": "string", + "description": "Date field accepted by the official company sample." + }, + "BirthDateMessageId": { + "type": "string", + "description": "Message id accepted by the official company sample." + }, + "MobilePhone": { + "type": "string", + "description": "Mobile or secretary phone." + }, + "WorkPhone": { + "type": "string", + "description": "Work phone." + }, + "WorkPhoneExtension": { + "type": "string", + "description": "Work phone extension." + }, + "Fax": { + "type": "string", + "description": "Fax." + }, + "Email": { + "type": "string", + "description": "Email." + }, + "IsVIP": { + "type": "boolean", + "description": "Whether the company is marked VIP." + }, + "BackgroundInfo": { + "type": "string", + "description": "Background notes." + }, + "Position": { + "type": "string", + "description": "Position field accepted by the official company sample." + }, + "CustomerCode": { + "type": "string", + "description": "Customer code." + }, + "EconomicIssue": { + "type": "string", + "description": "Economic code." + }, + "RegistrationNumber": { + "type": "string", + "description": "Company registration number." + }, + "NationalID": { + "type": "string", + "description": "Company national id." + }, + "ZipCode": { + "type": "string", + "description": "Postal code." + }, + "CityId": { + "type": "string", + "description": "City id from get_locations." + }, + "ProvinceId": { + "type": "string", + "description": "Province id from get_locations." + }, + "VisibilityType": { + "type": "string", + "description": "Visibility. Documented sample uses Owner." + }, + "Fields": { + "type": "object", + "description": "Custom field values keyed by field Key.", + "additionalProperties": true + }, + "OtherPhones": { + "type": "object", + "description": "Grouped key/value collection documented on contact save.", + "properties": { + "KeyValues": { + "type": "array", + "description": "Labeled key/value rows.", + "items": { + "type": "object", + "description": "One labeled value.", + "properties": { + "Key": { + "type": "string", + "description": "Label key." + }, + "Value": { + "type": "string", + "description": "Value." + }, + "Title": { + "type": "string", + "description": "Optional display title." + } + } + } + } + } + }, + "OtherEmails": { + "type": "object", + "description": "Grouped key/value collection documented on contact save.", + "properties": { + "KeyValues": { + "type": "array", + "description": "Labeled key/value rows.", + "items": { + "type": "object", + "description": "One labeled value.", + "properties": { + "Key": { + "type": "string", + "description": "Label key." + }, + "Value": { + "type": "string", + "description": "Value." + }, + "Title": { + "type": "string", + "description": "Optional display title." + } + } + } + } + } + }, + "Websites": { + "type": "object", + "description": "Grouped key/value collection documented on contact save.", + "properties": { + "KeyValues": { + "type": "array", + "description": "Labeled key/value rows.", + "items": { + "type": "object", + "description": "One labeled value.", + "properties": { + "Key": { + "type": "string", + "description": "Label key." + }, + "Value": { + "type": "string", + "description": "Value." + }, + "Title": { + "type": "string", + "description": "Optional display title." + } + } + } + } + } + }, + "BankAccounts": { + "type": "object", + "description": "Bank accounts object.", + "properties": { + "Values": { + "type": "array", + "description": "Bank account rows.", + "items": { + "type": "object", + "description": "One bank-account row.", + "properties": { + "Value1": { + "type": "string", + "description": "Account row type, such as Card or Account." + }, + "Value2": { + "type": "string", + "description": "Second documented value slot." + }, + "Value3": { + "type": "string", + "description": "Third documented value slot." + }, + "Value4": { + "type": "string", + "description": "Fourth documented value slot." + } + } + } + } + } + }, + "Addresses": { + "type": "object", + "description": "Grouped key/value collection documented on contact save.", + "properties": { + "KeyValues": { + "type": "array", + "description": "Labeled key/value rows.", + "items": { + "type": "object", + "description": "One labeled value.", + "properties": { + "Key": { + "type": "string", + "description": "Label key." + }, + "Value": { + "type": "string", + "description": "Value." + }, + "Title": { + "type": "string", + "description": "Optional display title." + } + } + } + } + } + } + }, + "required": [ + "Id", + "Type", + "FirstName", + "OwnerId" + ] + }, + "SegmentIds": { + "type": "array", + "description": "Segment ids.", + "items": { + "type": "string", + "description": "Segment id from get_segments." + } + } + }, + "required": [ + "Contact" + ] + }, + "tags": [ + "company" + ], + "content_type": "application/json", + "response_root": "Response", + "timeout_seconds": 30 + }, + { + "name": "get_company", + "title": "Get Company", + "description": "Get one company by id.", + "entity": "company", + "effect": "read", + "enabled": true, + "method": "POST", + "path": "/api/contact/GetContactDetail", + "input_schema": { + "type": "object", + "description": "Get company.", + "properties": { + "Id": { + "type": "string", + "description": "Company id." + } + }, + "required": [ + "Id" + ] + }, + "tags": [ + "company" + ], + "content_type": "application/json", + "response_root": "Response", + "timeout_seconds": 30 + }, + { + "name": "search_companies", + "title": "Search Companies", + "description": "Search companies. The official Search Companies page uses POST /api/contact/PersonSearch with Criteria, From, and Limit.", + "entity": "company", + "effect": "read", + "enabled": true, + "method": "POST", + "path": "/api/contact/PersonSearch", + "input_schema": { + "type": "object", + "description": "Search companies. Didar documents this on POST /api/contact/PersonSearch.", + "properties": { + "Criteria": { + "type": "object", + "description": "Company search criteria from the official Search Companies page.", + "properties": { + "IsDeleted": { + "type": "integer", + "description": "0 keeps active records. 1 keeps deleted records.", + "enum": [ + 0, + 1 + ] + }, + "IsPinned": { + "type": "integer", + "description": "-1 is not pinned. 1 is pinned.", + "enum": [ + -1, + 1 + ] + }, + "IsVIP": { + "type": "integer", + "description": "-1 is not VIP. 1 is VIP.", + "enum": [ + -1, + 1 + ] + }, + "SortOrder": { + "type": "integer", + "description": "Sort direction." + }, + "Keywords": { + "type": "string", + "description": "Text matched against the company name." + }, + "MobilePhone": { + "type": "string", + "description": "Phone filter." + }, + "WorkPhone": { + "type": "string", + "description": "Work phone filter." + }, + "Email": { + "type": "string", + "description": "Email filter." + }, + "NationalCode": { + "type": "string", + "description": "Field present on the official Search Companies sample." + }, + "NationalID": { + "type": "string", + "description": "Company national id named on the Search Companies page." + }, + "RegistrationNumber": { + "type": "string", + "description": "Registration number named on the Search Companies page." + }, + "EconomicIssue": { + "type": "string", + "description": "Economic code named on the Search Companies page." + }, + "ZipCode": { + "type": "string", + "description": "Postal code filter." + }, + "CustomerCode": { + "type": "string", + "description": "Customer code filter." + }, + "SearchFromTime": { + "type": "string", + "description": "Inclusive range start." + }, + "SearchToTime": { + "type": "string", + "description": "Inclusive range end." + }, + "CustomFields": { + "type": "array", + "description": "Custom-field filters.", + "items": { + "type": "object", + "description": "Official person/company search sample sends objects whose keys are custom-field keys such as Field_996_0_226.", + "additionalProperties": true + } + } + } + }, + "From": { + "type": "integer", + "description": "Zero-based offset." + }, + "Limit": { + "type": "integer", + "description": "Page size." + } + } + }, + "pagination": { + "type": "offset_limit", + "offset_field": "From", + "limit_field": "Limit" + }, + "tags": [ + "company" + ], + "content_type": "application/json", + "response_root": "Response", + "timeout_seconds": 30 + }, + { + "name": "create_deal", + "title": "Create Deal", + "description": "Create a deal. Disabled in v1.", + "entity": "deal", + "effect": "write", + "enabled": false, + "method": "POST", + "path": "/api/deal/save_v2", + "input_schema": { + "type": "object", + "description": "Create or update a deal.", + "properties": { + "Deal": { + "type": "object", + "description": "Deal payload.", + "properties": { + "Title": { + "type": "string", + "description": "Deal title." + }, + "Description": { + "type": "string", + "description": "Deal description." + }, + "PersonId": { + "type": "string", + "description": "Primary person id." + }, + "CompanyId": { + "type": "string", + "description": "Related company id." + }, + "PipelineId": { + "type": "string", + "description": "Deal pipeline id from get_deal_pipelines." + }, + "PipelineStageId": { + "type": "string", + "description": "Stage id from get_deal_pipelines." + }, + "OwnerId": { + "type": "string", + "description": "UserId of the owner." + }, + "Status": { + "type": "string", + "description": "Deal status.", + "enum": [ + "Pending", + "Won", + "Lost" + ] + }, + "Price": { + "type": "string", + "description": "Price as documented in the official sample." + }, + "LabelIds": { + "type": "array", + "description": "Label ids.", + "items": { + "type": "string", + "description": "Deal label id from get_deal_labels." + } + }, + "SourceId": { + "type": "string", + "description": "Source id from get_sources." + }, + "SourceOther": { + "type": "string", + "description": "Free-text source." + }, + "ReferrerId": { + "type": "string", + "description": "Referrer contact id." + }, + "ExpectedCloseDate": { + "description": "Expected close date, or null." + }, + "LostReasonId": { + "type": "string", + "description": "Lost reason id from get_lost_reasons. Used when Status is Lost." + }, + "LostReasonNote": { + "type": "string", + "description": "Extra lost-reason note." + }, + "LostReasonOther": { + "type": "string", + "description": "Other lost-reason text." + }, + "VisibilityType": { + "type": "string", + "description": "Visibility. Official sample uses Owner." + }, + "CurrencyId": { + "type": "string", + "description": "Currency id." + }, + "Fields": { + "type": "object", + "description": "Custom field values keyed by field Key.", + "additionalProperties": true + }, + "CreatorId": { + "type": "string", + "description": "UserId of the creator." + }, + "OtherContactIds": { + "type": "string", + "description": "Comma-separated other contact ids, as in the official sample." + }, + "IsPaid": { + "type": "boolean", + "description": "Whether the deal is paid." + }, + "UserLosing": { + "type": "string", + "description": "UserId recorded when the deal is lost." + }, + "ChangeToWonTime": { + "type": "string", + "description": "Time the deal became Won." + }, + "ChangeToLossTime": { + "type": "string", + "description": "Time the deal became Lost." + }, + "ContactId": { + "type": "string", + "description": "Contact id included in the official sample." + }, + "IsDeleted": { + "type": "boolean", + "description": "Whether the deal is deleted." + }, + "LabelId": { + "type": "string", + "description": "Single label id included in the official sample." + }, + "EstimatedPrice": { + "type": "number", + "description": "Estimated price included in the official sample." + }, + "SegmentIds": { + "type": "array", + "description": "Segment ids included in the official sample.", + "items": { + "type": "string", + "description": "Segment id." + } + } + }, + "required": [ + "Title", + "PersonId", + "PipelineId", + "PipelineStageId" + ] + }, + "DealItems": { + "type": "array", + "description": "Deal product lines.", + "items": { + "type": "object", + "description": "One deal line.", + "properties": { + "ProductId": { + "type": "string", + "description": "Product id." + }, + "Quantity": { + "type": "string", + "description": "Quantity." + }, + "UnitPrice": { + "type": "string", + "description": "Unit price." + }, + "Price": { + "type": "string", + "description": "Line price." + }, + "Discount": { + "type": "string", + "description": "Discount amount." + }, + "DiscountPercent": { + "type": "string", + "description": "Discount percent." + }, + "TaxPercent": { + "type": "string", + "description": "Tax percent." + }, + "Description": { + "type": "string", + "description": "Line description." + }, + "VariantId": { + "type": "string", + "description": "Product variant id." + } + }, + "required": [ + "ProductId", + "Quantity" + ] + } + } + }, + "required": [ + "Deal" + ] + }, + "tags": [ + "deal" + ], + "content_type": "application/json", + "response_root": "Response", + "timeout_seconds": 30 + }, + { + "name": "get_deal", + "title": "Get Deal", + "description": "Get one deal by id.", + "entity": "deal", + "effect": "read", + "enabled": true, + "method": "POST", + "path": "/api/deal/getdealdetail", + "input_schema": { + "type": "object", + "description": "Get deal.", + "properties": { + "Id": { + "type": "string", + "description": "Deal id." + } + }, + "required": [ + "Id" + ] + }, + "tags": [ + "deal" + ], + "content_type": "application/json", + "response_root": "Response", + "timeout_seconds": 30 + }, + { + "name": "update_deal", + "title": "Update Deal", + "description": "Update a deal. Disabled in v1.", + "entity": "deal", + "effect": "write", + "enabled": false, + "method": "POST", + "path": "/api/deal/save_v2", + "input_schema": { + "type": "object", + "description": "Create or update a deal.", + "properties": { + "Deal": { + "type": "object", + "description": "Deal payload.", + "properties": { + "Id": { + "type": "string", + "description": "Deal id." + }, + "Code": { + "type": "string", + "description": "Deal code." + }, + "Title": { + "type": "string", + "description": "Deal title." + }, + "Description": { + "type": "string", + "description": "Deal description." + }, + "PersonId": { + "type": "string", + "description": "Primary person id." + }, + "CompanyId": { + "type": "string", + "description": "Related company id." + }, + "PipelineId": { + "type": "string", + "description": "Deal pipeline id from get_deal_pipelines." + }, + "PipelineStageId": { + "type": "string", + "description": "Stage id from get_deal_pipelines." + }, + "OwnerId": { + "type": "string", + "description": "UserId of the owner." + }, + "Status": { + "type": "string", + "description": "Deal status.", + "enum": [ + "Pending", + "Won", + "Lost" + ] + }, + "Price": { + "type": "string", + "description": "Price as documented in the official sample." + }, + "LabelIds": { + "type": "array", + "description": "Label ids.", + "items": { + "type": "string", + "description": "Deal label id from get_deal_labels." + } + }, + "SourceId": { + "type": "string", + "description": "Source id from get_sources." + }, + "SourceOther": { + "type": "string", + "description": "Free-text source." + }, + "ReferrerId": { + "type": "string", + "description": "Referrer contact id." + }, + "ExpectedCloseDate": { + "description": "Expected close date, or null." + }, + "LostReasonId": { + "type": "string", + "description": "Lost reason id from get_lost_reasons. Used when Status is Lost." + }, + "LostReasonNote": { + "type": "string", + "description": "Extra lost-reason note." + }, + "LostReasonOther": { + "type": "string", + "description": "Other lost-reason text." + }, + "VisibilityType": { + "type": "string", + "description": "Visibility. Official sample uses Owner." + }, + "CurrencyId": { + "type": "string", + "description": "Currency id." + }, + "Fields": { + "type": "object", + "description": "Custom field values keyed by field Key.", + "additionalProperties": true + }, + "CreatorId": { + "type": "string", + "description": "UserId of the creator." + }, + "OtherContactIds": { + "type": "string", + "description": "Comma-separated other contact ids, as in the official sample." + }, + "IsPaid": { + "type": "boolean", + "description": "Whether the deal is paid." + }, + "UserLosing": { + "type": "string", + "description": "UserId recorded when the deal is lost." + }, + "ChangeToWonTime": { + "type": "string", + "description": "Time the deal became Won." + }, + "ChangeToLossTime": { + "type": "string", + "description": "Time the deal became Lost." + }, + "ContactId": { + "type": "string", + "description": "Contact id included in the official sample." + }, + "IsDeleted": { + "type": "boolean", + "description": "Whether the deal is deleted." + }, + "LabelId": { + "type": "string", + "description": "Single label id included in the official sample." + }, + "EstimatedPrice": { + "type": "number", + "description": "Estimated price included in the official sample." + }, + "SegmentIds": { + "type": "array", + "description": "Segment ids included in the official sample.", + "items": { + "type": "string", + "description": "Segment id." + } + } + }, + "required": [ + "Id", + "Title", + "PersonId", + "PipelineId", + "PipelineStageId" + ] + }, + "DealItems": { + "type": "array", + "description": "Deal product lines.", + "items": { + "type": "object", + "description": "One deal line.", + "properties": { + "ProductId": { + "type": "string", + "description": "Product id." + }, + "Quantity": { + "type": "string", + "description": "Quantity." + }, + "UnitPrice": { + "type": "string", + "description": "Unit price." + }, + "Price": { + "type": "string", + "description": "Line price." + }, + "Discount": { + "type": "string", + "description": "Discount amount." + }, + "DiscountPercent": { + "type": "string", + "description": "Discount percent." + }, + "TaxPercent": { + "type": "string", + "description": "Tax percent." + }, + "Description": { + "type": "string", + "description": "Line description." + }, + "VariantId": { + "type": "string", + "description": "Product variant id." + } + }, + "required": [ + "ProductId", + "Quantity" + ] + } + } + }, + "required": [ + "Deal" + ] + }, + "tags": [ + "deal" + ], + "content_type": "application/json", + "response_root": "Response", + "timeout_seconds": 30 + }, + { + "name": "search_deals", + "title": "Search Deals", + "description": "Search deals with Criteria plus From/Limit pagination. Status values documented by Didar are Pending, Won, and Lost. The response root contains List plus aggregate counts and totals.", + "entity": "deal", + "effect": "read", + "enabled": true, + "method": "POST", + "path": "/api/deal/search_v2", + "input_schema": { + "type": "object", + "description": "Search deals.", + "properties": { + "Criteria": { + "type": "object", + "description": "Deal search criteria from the official Search Deals sample.", + "properties": { + "SearchFromTime": { + "type": "string", + "description": "Range start." + }, + "SearchToTime": { + "type": "string", + "description": "Range end." + }, + "Sort": { + "type": "integer", + "description": "Sort mode." + }, + "ContactIds": { + "type": "array", + "description": "Primary contact filter.", + "items": { + "type": "string", + "description": "Person id." + } + }, + "CompanyIds": { + "type": "array", + "description": "Company filter.", + "items": { + "type": "string", + "description": "Company id." + } + }, + "OtherContactIds": { + "type": "array", + "description": "Other contact filter.", + "items": { + "type": "string", + "description": "Other contact id." + } + }, + "Keywords": { + "type": "string", + "description": "Keyword filter." + }, + "LabelIds": { + "type": "array", + "description": "Label filter.", + "items": { + "type": "string", + "description": "Deal label id." + } + }, + "Status": { + "type": "string", + "description": "Deal status.", + "enum": [ + "Pending", + "Won", + "Lost" + ] + }, + "CustomFields": { + "type": "array", + "description": "Deal custom-field filters.", + "items": { + "type": "object", + "description": "One custom-field filter from the Search Deals sample.", + "properties": { + "CustomFieldId": { + "type": "string", + "description": "Id of a deal custom field from get_custom_fields." + }, + "OR": { + "type": "array", + "description": "OR comparisons for this custom field.", + "items": { + "type": "object", + "description": "One custom-field comparison.", + "properties": { + "Type": { + "type": "string", + "description": "Comparison shown in the official sample: ContainsAnyString, NotContainsAnyString, StartsWith, GreaterThan, LessThan, Between." + }, + "Value": { + "type": "array", + "description": "Values for this comparison.", + "items": { + "type": "string", + "description": "Comparison value." + } + } + } + } + } + } + } + }, + "PipelineId": { + "type": "string", + "description": "Pipeline id." + }, + "PipelineStageId": { + "type": "string", + "description": "Single pipeline stage id." + }, + "PipelineStageIds": { + "type": "array", + "description": "Several stage ids.", + "items": { + "type": "string", + "description": "Pipeline stage id." + } + }, + "SourceId": { + "type": "string", + "description": "Source id." + }, + "IsDeleted": { + "type": "boolean", + "description": "Deleted-record filter." + }, + "HasActivity": { + "type": "boolean", + "description": "Whether a related activity exists." + }, + "LostReasonId": { + "type": "string", + "description": "Lost reason id." + }, + "NextActivityFrom": { + "type": "string", + "description": "Next activity range start." + }, + "NextActivityTo": { + "type": "string", + "description": "Next activity range end." + }, + "IsIdle": { + "type": "boolean", + "description": "Idle-deal filter." + }, + "IsRotten": { + "type": "boolean", + "description": "Rotten-deal filter." + }, + "Probability": { + "type": "string", + "description": "Probability filter as shown in the official sample." + }, + "OwnerIds": { + "type": "array", + "description": "Owner filter.", + "items": { + "type": "string", + "description": "Owner UserId." + } + } + }, + "additionalProperties": false + }, + "From": { + "type": "integer", + "minimum": 0, + "description": "Zero-based offset." + }, + "Limit": { + "type": "integer", + "minimum": 1, + "maximum": 100, + "description": "Page size." + } + }, + "required": [ + "Criteria", + "From", + "Limit" + ], + "additionalProperties": false + }, + "pagination": { + "type": "offset_limit", + "offset_field": "From", + "limit_field": "Limit" + }, + "records_path": "Response.List", + "summary_fields": [ + "Response.TotalCount", + "Response.TotalPrice", + "Response.AllDealsCount", + "Response.WonDealsCount", + "Response.LostDealsCount", + "Response.PendingDealsCount" + ], + "semantic_hints": [ + "From and Limit are top-level fields, not inside Criteria.", + "Use server totals when only counts are needed." + ], + "tags": [ + "deal" + ], + "content_type": "application/json", + "response_root": "Response", + "timeout_seconds": 30 + }, + { + "name": "create_invoice", + "title": "Create Invoice", + "description": "Create an invoice. Disabled in v1.", + "entity": "invoice", + "effect": "write", + "enabled": false, + "method": "POST", + "path": "/api/AccountingInvoice/Create", + "input_schema": { + "type": "object", + "description": "Invoice payload.", + "properties": { + "Items": { + "type": "array", + "description": "Document lines.", + "items": { + "type": "object", + "description": "One invoice or quote line.", + "properties": { + "Id": { + "type": "string", + "description": "Existing line id when updating." + }, + "ProductId": { + "type": "string", + "description": "Product id." + }, + "ProductCode": { + "type": "string", + "description": "Product code." + }, + "VariantId": { + "type": "string", + "description": "Variant id." + }, + "VariantTitle": { + "type": "string", + "description": "Variant title." + }, + "VariantCode": { + "type": "string", + "description": "Variant code." + }, + "Title": { + "type": "string", + "description": "Line title." + }, + "TitleForInvoice": { + "type": "string", + "description": "Title printed on the document." + }, + "newTitle": { + "type": "string", + "description": "Replacement title accepted by the quote sample." + }, + "Description": { + "type": "string", + "description": "Line description." + }, + "Unit": { + "type": "string", + "description": "Unit name." + }, + "UnitPrice": { + "type": "string", + "description": "Unit price." + }, + "Quantity": { + "type": "number", + "description": "Quantity." + }, + "Discount": { + "type": "string", + "description": "Discount amount." + }, + "DiscountPercent": { + "type": "number", + "description": "Discount percent." + }, + "DiscountType": { + "type": "string", + "description": "Discount type. Official samples use Amount or Percent." + }, + "TaxPercent": { + "type": "string", + "description": "Tax percent." + }, + "SaleTypeId": { + "type": "string", + "description": "Sale type id." + }, + "Price": { + "type": "number", + "description": "Line price included in the invoice sample." + } + } + } + }, + "SellerInfo": { + "type": "object", + "description": "Seller info.", + "properties": { + "SellerName": { + "type": "string", + "description": "Seller name." + }, + "BizDomainId": { + "type": "string", + "description": "Seller business-domain id." + }, + "QuoteExpirationDays": { + "type": "number", + "description": "Default quote validity days." + }, + "SellerEconomicCode": { + "type": "string", + "description": "Seller economic code." + }, + "SellerNationalCode": { + "type": "string", + "description": "Seller national code." + }, + "SellerPostalCode": { + "type": "string", + "description": "Seller postal code." + }, + "SellerProvince": { + "type": "string", + "description": "Seller province." + }, + "SellerGrandCity": { + "type": "string", + "description": "Seller grand city." + }, + "SellerCity": { + "type": "string", + "description": "Seller city." + }, + "SellerAddress": { + "type": "string", + "description": "Seller address." + }, + "SellerPhone": { + "type": "string", + "description": "Seller phone." + }, + "InvoiceConditions": { + "type": "string", + "description": "Invoice conditions text." + }, + "QuoteConditions": { + "type": "string", + "description": "Quote conditions text." + } + } + }, + "BuyerInfo": { + "type": "object", + "description": "Buyer info.", + "properties": { + "BuyerType": { + "type": "string", + "description": "Buyer type. Official sample uses Company." + }, + "BuyerTitle": { + "type": "string", + "description": "Buyer title." + }, + "BuyerName": { + "type": "string", + "description": "Buyer name." + }, + "BuyerEconomicCode": { + "type": "string", + "description": "Buyer economic code." + }, + "BuyerNationalCode": { + "type": "string", + "description": "Buyer national code." + }, + "BuyerNationalID": { + "type": "string", + "description": "Buyer national id." + }, + "BuyerRegistrationNumber": { + "type": "string", + "description": "Buyer registration number." + }, + "BuyerProvince": { + "type": "string", + "description": "Buyer province." + }, + "BuyerGrandCity": { + "type": "string", + "description": "Buyer grand city." + }, + "BuyerCity": { + "type": "string", + "description": "Buyer city." + }, + "BuyerPostalCode": { + "type": "string", + "description": "Buyer postal code." + }, + "BuyerMobilePhone": { + "type": "string", + "description": "Buyer mobile phone." + }, + "BuyerFaxNumber": { + "type": "string", + "description": "Buyer fax." + }, + "BuyerAddress": { + "type": "object", + "description": "Grouped key/value collection documented on contact save.", + "properties": { + "KeyValues": { + "type": "array", + "description": "Labeled key/value rows.", + "items": { + "type": "object", + "description": "One labeled value.", + "properties": { + "Key": { + "type": "string", + "description": "Label key." + }, + "Value": { + "type": "string", + "description": "Value." + }, + "Title": { + "type": "string", + "description": "Optional display title." + } + } + } + } + } + }, + "BuyerPhone": { + "type": "string", + "description": "Buyer phone." + }, + "BuyerEmail": { + "type": "string", + "description": "Buyer email." + } + } + }, + "BizdomainProvider": { + "type": "object", + "description": "Optional provider object accepted by the official docs.", + "properties": {} + }, + "CreatorId": { + "type": "string", + "description": "UserId of the creator." + }, + "IssueDate": { + "type": "string", + "description": "Issue date/time." + }, + "Adjustment": { + "type": "string", + "description": "Adjustment amount." + }, + "DiscountRows": { + "type": "array", + "description": "Document-level discount rows.", + "items": { + "type": "object", + "description": "One discount row.", + "properties": { + "DiscountName": { + "type": "string", + "description": "Discount row name." + }, + "RowPrice": { + "type": "string", + "description": "Discount row price." + } + } + } + }, + "BizDomainId": { + "type": "string", + "description": "Business-domain id." + }, + "CurrencyId": { + "type": "number", + "description": "Currency id." + }, + "DealId": { + "type": "string", + "description": "Related deal id." + }, + "Description": { + "type": "string", + "description": "Document description." + } + }, + "required": [ + "Items", + "SellerInfo", + "BuyerInfo" + ] + }, + "tags": [ + "invoice" + ], + "content_type": "application/json", + "response_root": "Response", + "timeout_seconds": 30 + }, + { + "name": "create_quote", + "title": "Create Quote", + "description": "Create a quote. Disabled in v1.", + "entity": "quote", + "effect": "write", + "enabled": false, + "method": "POST", + "path": "/api/AccountingQuote/CreateQuote", + "input_schema": { + "type": "object", + "description": "Quote payload.", + "properties": { + "Items": { + "type": "array", + "description": "Document lines.", + "items": { + "type": "object", + "description": "One invoice or quote line.", + "properties": { + "Id": { + "type": "string", + "description": "Existing line id when updating." + }, + "ProductId": { + "type": "string", + "description": "Product id." + }, + "ProductCode": { + "type": "string", + "description": "Product code." + }, + "VariantId": { + "type": "string", + "description": "Variant id." + }, + "VariantTitle": { + "type": "string", + "description": "Variant title." + }, + "VariantCode": { + "type": "string", + "description": "Variant code." + }, + "Title": { + "type": "string", + "description": "Line title." + }, + "TitleForInvoice": { + "type": "string", + "description": "Title printed on the document." + }, + "newTitle": { + "type": "string", + "description": "Replacement title accepted by the quote sample." + }, + "Description": { + "type": "string", + "description": "Line description." + }, + "Unit": { + "type": "string", + "description": "Unit name." + }, + "UnitPrice": { + "type": "string", + "description": "Unit price." + }, + "Quantity": { + "type": "number", + "description": "Quantity." + }, + "Discount": { + "type": "string", + "description": "Discount amount." + }, + "DiscountPercent": { + "type": "number", + "description": "Discount percent." + }, + "DiscountType": { + "type": "string", + "description": "Discount type. Official samples use Amount or Percent." + }, + "TaxPercent": { + "type": "string", + "description": "Tax percent." + }, + "SaleTypeId": { + "type": "string", + "description": "Sale type id." + }, + "Price": { + "type": "number", + "description": "Line price included in the invoice sample." + } + } + } + }, + "SellerInfo": { + "type": "object", + "description": "Seller info.", + "properties": { + "SellerName": { + "type": "string", + "description": "Seller name." + }, + "BizDomainId": { + "type": "string", + "description": "Seller business-domain id." + }, + "QuoteExpirationDays": { + "type": "number", + "description": "Default quote validity days." + }, + "SellerEconomicCode": { + "type": "string", + "description": "Seller economic code." + }, + "SellerNationalCode": { + "type": "string", + "description": "Seller national code." + }, + "SellerPostalCode": { + "type": "string", + "description": "Seller postal code." + }, + "SellerProvince": { + "type": "string", + "description": "Seller province." + }, + "SellerGrandCity": { + "type": "string", + "description": "Seller grand city." + }, + "SellerCity": { + "type": "string", + "description": "Seller city." + }, + "SellerAddress": { + "type": "string", + "description": "Seller address." + }, + "SellerPhone": { + "type": "string", + "description": "Seller phone." + }, + "InvoiceConditions": { + "type": "string", + "description": "Invoice conditions text." + }, + "QuoteConditions": { + "type": "string", + "description": "Quote conditions text." + } + } + }, + "BuyerInfo": { + "type": "object", + "description": "Buyer info.", + "properties": { + "BuyerType": { + "type": "string", + "description": "Buyer type. Official sample uses Company." + }, + "BuyerTitle": { + "type": "string", + "description": "Buyer title." + }, + "BuyerName": { + "type": "string", + "description": "Buyer name." + }, + "BuyerEconomicCode": { + "type": "string", + "description": "Buyer economic code." + }, + "BuyerNationalCode": { + "type": "string", + "description": "Buyer national code." + }, + "BuyerNationalID": { + "type": "string", + "description": "Buyer national id." + }, + "BuyerRegistrationNumber": { + "type": "string", + "description": "Buyer registration number." + }, + "BuyerProvince": { + "type": "string", + "description": "Buyer province." + }, + "BuyerGrandCity": { + "type": "string", + "description": "Buyer grand city." + }, + "BuyerCity": { + "type": "string", + "description": "Buyer city." + }, + "BuyerPostalCode": { + "type": "string", + "description": "Buyer postal code." + }, + "BuyerMobilePhone": { + "type": "string", + "description": "Buyer mobile phone." + }, + "BuyerFaxNumber": { + "type": "string", + "description": "Buyer fax." + }, + "BuyerAddress": { + "type": "object", + "description": "Grouped key/value collection documented on contact save.", + "properties": { + "KeyValues": { + "type": "array", + "description": "Labeled key/value rows.", + "items": { + "type": "object", + "description": "One labeled value.", + "properties": { + "Key": { + "type": "string", + "description": "Label key." + }, + "Value": { + "type": "string", + "description": "Value." + }, + "Title": { + "type": "string", + "description": "Optional display title." + } + } + } + } + } + }, + "BuyerPhone": { + "type": "string", + "description": "Buyer phone." + }, + "BuyerEmail": { + "type": "string", + "description": "Buyer email." + } + } + }, + "BizdomainProvider": { + "type": "object", + "description": "Optional provider object accepted by the official docs.", + "properties": {} + }, + "CreatorId": { + "type": "string", + "description": "UserId of the creator." + }, + "IssueDate": { + "type": "string", + "description": "Issue date/time." + }, + "Adjustment": { + "type": "string", + "description": "Adjustment amount." + }, + "DiscountRows": { + "type": "array", + "description": "Document-level discount rows.", + "items": { + "type": "object", + "description": "One discount row.", + "properties": { + "DiscountName": { + "type": "string", + "description": "Discount row name." + }, + "RowPrice": { + "type": "string", + "description": "Discount row price." + } + } + } + }, + "BizDomainId": { + "type": "string", + "description": "Business-domain id." + }, + "CurrencyId": { + "type": "number", + "description": "Currency id." + }, + "DealId": { + "type": "string", + "description": "Related deal id." + }, + "Description": { + "type": "string", + "description": "Document description." + }, + "QuoteExpirationDays": { + "type": "number", + "description": "Quote validity in days." + } + }, + "required": [ + "Items", + "CreatorId", + "BizDomainId", + "CurrencyId" + ] + }, + "tags": [ + "quote" + ], + "content_type": "application/json", + "response_root": "Response", + "timeout_seconds": 30 + }, + { + "name": "update_invoice", + "title": "Update Invoice", + "description": "Update an invoice. Disabled in v1.", + "entity": "invoice", + "effect": "write", + "enabled": false, + "method": "POST", + "path": "/api/AccountingInvoice/Update", + "input_schema": { + "type": "object", + "description": "Invoice payload.", + "properties": { + "Id": { + "type": "number", + "description": "Invoice or quote id." + }, + "Items": { + "type": "array", + "description": "Document lines.", + "items": { + "type": "object", + "description": "One invoice or quote line.", + "properties": { + "Id": { + "type": "string", + "description": "Existing line id when updating." + }, + "ProductId": { + "type": "string", + "description": "Product id." + }, + "ProductCode": { + "type": "string", + "description": "Product code." + }, + "VariantId": { + "type": "string", + "description": "Variant id." + }, + "VariantTitle": { + "type": "string", + "description": "Variant title." + }, + "VariantCode": { + "type": "string", + "description": "Variant code." + }, + "Title": { + "type": "string", + "description": "Line title." + }, + "TitleForInvoice": { + "type": "string", + "description": "Title printed on the document." + }, + "newTitle": { + "type": "string", + "description": "Replacement title accepted by the quote sample." + }, + "Description": { + "type": "string", + "description": "Line description." + }, + "Unit": { + "type": "string", + "description": "Unit name." + }, + "UnitPrice": { + "type": "string", + "description": "Unit price." + }, + "Quantity": { + "type": "number", + "description": "Quantity." + }, + "Discount": { + "type": "string", + "description": "Discount amount." + }, + "DiscountPercent": { + "type": "number", + "description": "Discount percent." + }, + "DiscountType": { + "type": "string", + "description": "Discount type. Official samples use Amount or Percent." + }, + "TaxPercent": { + "type": "string", + "description": "Tax percent." + }, + "SaleTypeId": { + "type": "string", + "description": "Sale type id." + }, + "Price": { + "type": "number", + "description": "Line price included in the invoice sample." + } + } + } + }, + "SellerInfo": { + "type": "object", + "description": "Seller info.", + "properties": { + "SellerName": { + "type": "string", + "description": "Seller name." + }, + "BizDomainId": { + "type": "string", + "description": "Seller business-domain id." + }, + "QuoteExpirationDays": { + "type": "number", + "description": "Default quote validity days." + }, + "SellerEconomicCode": { + "type": "string", + "description": "Seller economic code." + }, + "SellerNationalCode": { + "type": "string", + "description": "Seller national code." + }, + "SellerPostalCode": { + "type": "string", + "description": "Seller postal code." + }, + "SellerProvince": { + "type": "string", + "description": "Seller province." + }, + "SellerGrandCity": { + "type": "string", + "description": "Seller grand city." + }, + "SellerCity": { + "type": "string", + "description": "Seller city." + }, + "SellerAddress": { + "type": "string", + "description": "Seller address." + }, + "SellerPhone": { + "type": "string", + "description": "Seller phone." + }, + "InvoiceConditions": { + "type": "string", + "description": "Invoice conditions text." + }, + "QuoteConditions": { + "type": "string", + "description": "Quote conditions text." + } + } + }, + "BuyerInfo": { + "type": "object", + "description": "Buyer info.", + "properties": { + "BuyerType": { + "type": "string", + "description": "Buyer type. Official sample uses Company." + }, + "BuyerTitle": { + "type": "string", + "description": "Buyer title." + }, + "BuyerName": { + "type": "string", + "description": "Buyer name." + }, + "BuyerEconomicCode": { + "type": "string", + "description": "Buyer economic code." + }, + "BuyerNationalCode": { + "type": "string", + "description": "Buyer national code." + }, + "BuyerNationalID": { + "type": "string", + "description": "Buyer national id." + }, + "BuyerRegistrationNumber": { + "type": "string", + "description": "Buyer registration number." + }, + "BuyerProvince": { + "type": "string", + "description": "Buyer province." + }, + "BuyerGrandCity": { + "type": "string", + "description": "Buyer grand city." + }, + "BuyerCity": { + "type": "string", + "description": "Buyer city." + }, + "BuyerPostalCode": { + "type": "string", + "description": "Buyer postal code." + }, + "BuyerMobilePhone": { + "type": "string", + "description": "Buyer mobile phone." + }, + "BuyerFaxNumber": { + "type": "string", + "description": "Buyer fax." + }, + "BuyerAddress": { + "type": "object", + "description": "Grouped key/value collection documented on contact save.", + "properties": { + "KeyValues": { + "type": "array", + "description": "Labeled key/value rows.", + "items": { + "type": "object", + "description": "One labeled value.", + "properties": { + "Key": { + "type": "string", + "description": "Label key." + }, + "Value": { + "type": "string", + "description": "Value." + }, + "Title": { + "type": "string", + "description": "Optional display title." + } + } + } + } + } + }, + "BuyerPhone": { + "type": "string", + "description": "Buyer phone." + }, + "BuyerEmail": { + "type": "string", + "description": "Buyer email." + } + } + }, + "BizdomainProvider": { + "type": "object", + "description": "Optional provider object accepted by the official docs.", + "properties": {} + }, + "CreatorId": { + "type": "string", + "description": "UserId of the creator." + }, + "IssueDate": { + "type": "string", + "description": "Issue date/time." + }, + "Adjustment": { + "type": "string", + "description": "Adjustment amount." + }, + "DiscountRows": { + "type": "array", + "description": "Document-level discount rows.", + "items": { + "type": "object", + "description": "One discount row.", + "properties": { + "DiscountName": { + "type": "string", + "description": "Discount row name." + }, + "RowPrice": { + "type": "string", + "description": "Discount row price." + } + } + } + }, + "BizDomainId": { + "type": "string", + "description": "Business-domain id." + }, + "CurrencyId": { + "type": "number", + "description": "Currency id." + }, + "DealId": { + "type": "string", + "description": "Related deal id." + }, + "Description": { + "type": "string", + "description": "Document description." + } + }, + "required": [ + "Id", + "Items" + ] + }, + "tags": [ + "invoice" + ], + "content_type": "application/json", + "response_root": "Response", + "timeout_seconds": 30 + }, + { + "name": "update_quote", + "title": "Update Quote", + "description": "Update a quote. Disabled in v1.", + "entity": "quote", + "effect": "write", + "enabled": false, + "method": "POST", + "path": "/api/AccountingQuote/UpdateQuote", + "input_schema": { + "type": "object", + "description": "Quote payload.", + "properties": { + "Id": { + "type": "number", + "description": "Invoice or quote id." + }, + "Items": { + "type": "array", + "description": "Document lines.", + "items": { + "type": "object", + "description": "One invoice or quote line.", + "properties": { + "Id": { + "type": "string", + "description": "Existing line id when updating." + }, + "ProductId": { + "type": "string", + "description": "Product id." + }, + "ProductCode": { + "type": "string", + "description": "Product code." + }, + "VariantId": { + "type": "string", + "description": "Variant id." + }, + "VariantTitle": { + "type": "string", + "description": "Variant title." + }, + "VariantCode": { + "type": "string", + "description": "Variant code." + }, + "Title": { + "type": "string", + "description": "Line title." + }, + "TitleForInvoice": { + "type": "string", + "description": "Title printed on the document." + }, + "newTitle": { + "type": "string", + "description": "Replacement title accepted by the quote sample." + }, + "Description": { + "type": "string", + "description": "Line description." + }, + "Unit": { + "type": "string", + "description": "Unit name." + }, + "UnitPrice": { + "type": "string", + "description": "Unit price." + }, + "Quantity": { + "type": "number", + "description": "Quantity." + }, + "Discount": { + "type": "string", + "description": "Discount amount." + }, + "DiscountPercent": { + "type": "number", + "description": "Discount percent." + }, + "DiscountType": { + "type": "string", + "description": "Discount type. Official samples use Amount or Percent." + }, + "TaxPercent": { + "type": "string", + "description": "Tax percent." + }, + "SaleTypeId": { + "type": "string", + "description": "Sale type id." + }, + "Price": { + "type": "number", + "description": "Line price included in the invoice sample." + } + } + } + }, + "SellerInfo": { + "type": "object", + "description": "Seller info.", + "properties": { + "SellerName": { + "type": "string", + "description": "Seller name." + }, + "BizDomainId": { + "type": "string", + "description": "Seller business-domain id." + }, + "QuoteExpirationDays": { + "type": "number", + "description": "Default quote validity days." + }, + "SellerEconomicCode": { + "type": "string", + "description": "Seller economic code." + }, + "SellerNationalCode": { + "type": "string", + "description": "Seller national code." + }, + "SellerPostalCode": { + "type": "string", + "description": "Seller postal code." + }, + "SellerProvince": { + "type": "string", + "description": "Seller province." + }, + "SellerGrandCity": { + "type": "string", + "description": "Seller grand city." + }, + "SellerCity": { + "type": "string", + "description": "Seller city." + }, + "SellerAddress": { + "type": "string", + "description": "Seller address." + }, + "SellerPhone": { + "type": "string", + "description": "Seller phone." + }, + "InvoiceConditions": { + "type": "string", + "description": "Invoice conditions text." + }, + "QuoteConditions": { + "type": "string", + "description": "Quote conditions text." + } + } + }, + "BuyerInfo": { + "type": "object", + "description": "Buyer info.", + "properties": { + "BuyerType": { + "type": "string", + "description": "Buyer type. Official sample uses Company." + }, + "BuyerTitle": { + "type": "string", + "description": "Buyer title." + }, + "BuyerName": { + "type": "string", + "description": "Buyer name." + }, + "BuyerEconomicCode": { + "type": "string", + "description": "Buyer economic code." + }, + "BuyerNationalCode": { + "type": "string", + "description": "Buyer national code." + }, + "BuyerNationalID": { + "type": "string", + "description": "Buyer national id." + }, + "BuyerRegistrationNumber": { + "type": "string", + "description": "Buyer registration number." + }, + "BuyerProvince": { + "type": "string", + "description": "Buyer province." + }, + "BuyerGrandCity": { + "type": "string", + "description": "Buyer grand city." + }, + "BuyerCity": { + "type": "string", + "description": "Buyer city." + }, + "BuyerPostalCode": { + "type": "string", + "description": "Buyer postal code." + }, + "BuyerMobilePhone": { + "type": "string", + "description": "Buyer mobile phone." + }, + "BuyerFaxNumber": { + "type": "string", + "description": "Buyer fax." + }, + "BuyerAddress": { + "type": "object", + "description": "Grouped key/value collection documented on contact save.", + "properties": { + "KeyValues": { + "type": "array", + "description": "Labeled key/value rows.", + "items": { + "type": "object", + "description": "One labeled value.", + "properties": { + "Key": { + "type": "string", + "description": "Label key." + }, + "Value": { + "type": "string", + "description": "Value." + }, + "Title": { + "type": "string", + "description": "Optional display title." + } + } + } + } + } + }, + "BuyerPhone": { + "type": "string", + "description": "Buyer phone." + }, + "BuyerEmail": { + "type": "string", + "description": "Buyer email." + } + } + }, + "BizdomainProvider": { + "type": "object", + "description": "Optional provider object accepted by the official docs.", + "properties": {} + }, + "CreatorId": { + "type": "string", + "description": "UserId of the creator." + }, + "IssueDate": { + "type": "string", + "description": "Issue date/time." + }, + "Adjustment": { + "type": "string", + "description": "Adjustment amount." + }, + "DiscountRows": { + "type": "array", + "description": "Document-level discount rows.", + "items": { + "type": "object", + "description": "One discount row.", + "properties": { + "DiscountName": { + "type": "string", + "description": "Discount row name." + }, + "RowPrice": { + "type": "string", + "description": "Discount row price." + } + } + } + }, + "BizDomainId": { + "type": "string", + "description": "Business-domain id." + }, + "CurrencyId": { + "type": "number", + "description": "Currency id." + }, + "DealId": { + "type": "string", + "description": "Related deal id." + }, + "Description": { + "type": "string", + "description": "Document description." + }, + "QuoteExpirationDays": { + "type": "number", + "description": "Quote validity in days." + } + }, + "required": [ + "Id", + "Items" + ] + }, + "tags": [ + "quote" + ], + "content_type": "application/json", + "response_root": "Response", + "timeout_seconds": 30 + }, + { + "name": "remove_invoice", + "title": "Remove Invoice", + "description": "Remove an invoice by numeric id. Disabled in v1.", + "entity": "invoice", + "effect": "delete", + "enabled": false, + "method": "POST", + "path": "/api/AccountingInvoice/Remove", + "input_schema": { + "type": "object", + "description": "Remove invoice.", + "properties": { + "id": { + "type": "number", + "description": "Invoice id." + } + }, + "required": [ + "id" + ] + }, + "tags": [ + "invoice" + ], + "content_type": "application/json", + "response_root": "Response", + "timeout_seconds": 30 + }, + { + "name": "remove_quote", + "title": "Remove Quote", + "description": "Remove a quote by numeric id. Disabled in v1.", + "entity": "quote", + "effect": "delete", + "enabled": false, + "method": "POST", + "path": "/api/AccountingQuote/RemoveQuote", + "input_schema": { + "type": "object", + "description": "Remove quote.", + "properties": { + "id": { + "type": "number", + "description": "Quote id." + } + }, + "required": [ + "id" + ] + }, + "tags": [ + "quote" + ], + "content_type": "application/json", + "response_root": "Response", + "timeout_seconds": 30 + }, + { + "name": "get_invoice_or_quote", + "title": "Get Invoice Or Quote", + "description": "Get one invoice or quote. Send exactly one of Id, InvoiceFormattedNumber, InvoiceNumber, QuoteFormattedNumber, or QuoteNumber.", + "entity": "invoice", + "effect": "read", + "enabled": true, + "method": "POST", + "path": "/api/AccountingInvoice/Get", + "input_schema": { + "type": "object", + "description": "Exactly one lookup field is valid.", + "properties": { + "Id": { + "type": "number", + "description": "Invoice or quote id." + }, + "InvoiceFormattedNumber": { + "type": "string", + "description": "Formatted invoice number, for example I-23." + }, + "InvoiceNumber": { + "type": "string", + "description": "Invoice number, for example 23.0." + }, + "QuoteFormattedNumber": { + "type": "string", + "description": "Formatted quote number, for example Q-21." + }, + "QuoteNumber": { + "type": "string", + "description": "Quote number, for example 21.0." + } + } + }, + "tags": [ + "invoice", + "quote" + ], + "content_type": "application/json", + "response_root": "Response", + "timeout_seconds": 30 + }, + { + "name": "create_case", + "title": "Create Case", + "description": "Create a case. Disabled in v1.", + "entity": "case", + "effect": "write", + "enabled": false, + "method": "POST", + "path": "/api/Case/Save_v2", + "input_schema": { + "type": "object", + "description": "Create or update a case.", + "properties": { + "Case": { + "type": "object", + "description": "Case payload.", + "properties": { + "Title": { + "type": "string", + "description": "Case title." + }, + "PipelineStageId": { + "type": "string", + "description": "Case pipeline stage id from get_case_pipelines." + }, + "OwnerId": { + "type": "string", + "description": "UserId of the owner." + }, + "Description": { + "type": "string", + "description": "Case description." + }, + "CompanyId": { + "type": "string", + "description": "Related company id." + }, + "PersonId": { + "type": "string", + "description": "Related person id." + }, + "CaseCategoryId": { + "type": "string", + "description": "Case category id." + }, + "DealId": { + "type": "string", + "description": "Related deal id." + }, + "IsDeleted": { + "type": "boolean", + "description": "Whether the case is deleted." + }, + "Status": { + "type": "string", + "description": "Case status.", + "enum": [ + "InProgress", + "Done" + ] + }, + "DueDate": { + "type": "string", + "description": "Due date/time." + }, + "Priority": { + "type": "number", + "description": "Priority. Documented values are -2, -1, 0, and 1." + }, + "UserIds": { + "type": "array", + "description": "Additional user ids.", + "items": { + "type": "string", + "description": "Related UserId." + } + }, + "OtherContactIds": { + "type": "string", + "description": "Other contact ids." + }, + "ResultNote": { + "type": "string", + "description": "Result note." + }, + "VisibilityType": { + "type": "string", + "description": "Visibility. Documented values include Owner, OwnerGroup, OwnerSubGroup, and All." + }, + "Fields": { + "type": "object", + "description": "Custom field values keyed by field Key. FieldType must be Case.", + "additionalProperties": true + }, + "LabelIds": { + "type": "array", + "description": "Case label ids.", + "items": { + "type": "string", + "description": "Case label id from get_case_labels." + } + } + }, + "required": [ + "Title", + "PipelineStageId", + "CaseCategoryId" + ] + } + }, + "required": [ + "Case" + ] + }, + "tags": [ + "case" + ], + "content_type": "application/json", + "response_root": "Response", + "timeout_seconds": 30 + }, + { + "name": "update_case", + "title": "Update Case", + "description": "Update a case. Disabled in v1.", + "entity": "case", + "effect": "write", + "enabled": false, + "method": "POST", + "path": "/api/Case/Save_v2", + "input_schema": { + "type": "object", + "description": "Create or update a case.", + "properties": { + "Case": { + "type": "object", + "description": "Case payload.", + "properties": { + "Id": { + "type": "string", + "description": "Case id." + }, + "Title": { + "type": "string", + "description": "Case title." + }, + "PipelineStageId": { + "type": "string", + "description": "Case pipeline stage id from get_case_pipelines." + }, + "OwnerId": { + "type": "string", + "description": "UserId of the owner." + }, + "Description": { + "type": "string", + "description": "Case description." + }, + "CompanyId": { + "type": "string", + "description": "Related company id." + }, + "PersonId": { + "type": "string", + "description": "Related person id." + }, + "CaseCategoryId": { + "type": "string", + "description": "Case category id." + }, + "DealId": { + "type": "string", + "description": "Related deal id." + }, + "IsDeleted": { + "type": "boolean", + "description": "Whether the case is deleted." + }, + "Status": { + "type": "string", + "description": "Case status.", + "enum": [ + "InProgress", + "Done" + ] + }, + "DueDate": { + "type": "string", + "description": "Due date/time." + }, + "Priority": { + "type": "number", + "description": "Priority. Documented values are -2, -1, 0, and 1." + }, + "UserIds": { + "type": "array", + "description": "Additional user ids.", + "items": { + "type": "string", + "description": "Related UserId." + } + }, + "OtherContactIds": { + "type": "string", + "description": "Other contact ids." + }, + "ResultNote": { + "type": "string", + "description": "Result note." + }, + "VisibilityType": { + "type": "string", + "description": "Visibility. Documented values include Owner, OwnerGroup, OwnerSubGroup, and All." + }, + "Fields": { + "type": "object", + "description": "Custom field values keyed by field Key. FieldType must be Case.", + "additionalProperties": true + }, + "LabelIds": { + "type": "array", + "description": "Case label ids.", + "items": { + "type": "string", + "description": "Case label id from get_case_labels." + } + } + }, + "required": [ + "Id", + "Title", + "PipelineStageId", + "CaseCategoryId" + ] + } + }, + "required": [ + "Case" + ] + }, + "tags": [ + "case" + ], + "content_type": "application/json", + "response_root": "Response", + "timeout_seconds": 30 + }, + { + "name": "search_cases", + "title": "Search Cases", + "description": "Search cases by title, time range, contact, owner, stage, and related deal.", + "entity": "case", + "effect": "read", + "enabled": true, + "method": "POST", + "path": "/api/Case/search", + "input_schema": { + "type": "object", + "description": "Search cases.", + "properties": { + "Criteria": { + "type": "object", + "description": "Case search criteria.", + "properties": { + "SearchTimeField": { + "type": "integer", + "description": "Which date field SearchFromTime and SearchToTime apply to." + }, + "SearchFromTime": { + "type": "string", + "description": "Range start." + }, + "SearchToTime": { + "type": "string", + "description": "Range end." + }, + "ContactIds": { + "type": "array", + "description": "Contact filter.", + "items": { + "type": "string", + "description": "Person id." + } + }, + "Keywords": { + "type": "string", + "description": "Text matched against the case title." + }, + "OwnerId": { + "type": "string", + "description": "Owner UserId." + }, + "PipelineStageId": { + "type": "string", + "description": "Case pipeline stage id." + }, + "DealId": { + "type": "string", + "description": "Related deal id." + }, + "CustomFields": { + "type": "array", + "description": "Custom-field filters.", + "items": { + "type": "object", + "description": "Case custom-field filter. FieldType must be Case.", + "additionalProperties": true + } + } + } + }, + "From": { + "type": "integer", + "description": "Zero-based offset." + }, + "Limit": { + "type": "integer", + "description": "Page size." + } + } + }, + "pagination": { + "type": "offset_limit", + "offset_field": "From", + "limit_field": "Limit" + }, + "tags": [ + "case" + ], + "content_type": "application/json", + "response_root": "Response", + "timeout_seconds": 30 + }, + { + "name": "get_deal_pipelines", + "title": "Get Deal Pipelines", + "description": "List deal pipelines and stages. Official docs: no request body.", + "entity": "pipeline", + "effect": "read", + "enabled": true, + "method": "POST", + "path": "/api/pipeline/list/0", + "input_schema": { + "type": "object", + "description": "Official Didar docs: this request has no body.", + "additionalProperties": false + }, + "tags": [ + "pipeline" + ], + "content_type": "application/json", + "response_root": "Response", + "timeout_seconds": 30 + }, + { + "name": "get_case_pipelines", + "title": "Get Case Pipelines", + "description": "List case pipelines and stages. Official docs: no request body.", + "entity": "pipeline", + "effect": "read", + "enabled": true, + "method": "POST", + "path": "/api/pipeline/list/1", + "input_schema": { + "type": "object", + "description": "Official Didar docs: this request has no body.", + "additionalProperties": false + }, + "tags": [ + "pipeline" + ], + "content_type": "application/json", + "response_root": "Response", + "timeout_seconds": 30 + }, + { + "name": "get_segments", + "title": "Get Segments", + "description": "List contact segments, or the children of ParentId.", + "entity": "segment", + "effect": "read", + "enabled": true, + "method": "POST", + "path": "/api/Segment/GetList", + "input_schema": { + "type": "object", + "description": "List segments.", + "properties": { + "ParentId": { + "type": "string", + "description": "Optional parent segment id. Omit it to list root segments." + } + } + }, + "tags": [ + "segment" + ], + "content_type": "application/json", + "response_root": "Response", + "timeout_seconds": 30 + }, + { + "name": "get_case_labels", + "title": "Get Case Labels", + "description": "List case labels. Official docs: GET with no request body.", + "entity": "label", + "effect": "read", + "enabled": true, + "method": "GET", + "path": "/api/Label/GetCaseLabels", + "input_schema": { + "type": "object", + "description": "Official Didar docs: this request has no body.", + "additionalProperties": false + }, + "tags": [ + "label" + ], + "content_type": "application/json", + "response_root": "Response", + "timeout_seconds": 30 + }, + { + "name": "get_deal_labels", + "title": "Get Deal Labels", + "description": "List deal labels. Official docs: GET with no request body.", + "entity": "label", + "effect": "read", + "enabled": true, + "method": "GET", + "path": "/api/Label/GetDealLabels", + "input_schema": { + "type": "object", + "description": "Official Didar docs: this request has no body.", + "additionalProperties": false + }, + "tags": [ + "label" + ], + "content_type": "application/json", + "response_root": "Response", + "timeout_seconds": 30 + }, + { + "name": "get_tags", + "title": "Get Tags", + "description": "List tags. Official docs: no request body.", + "entity": "tag", + "effect": "read", + "enabled": true, + "method": "POST", + "path": "/api/Tag/GetTagList", + "input_schema": { + "type": "object", + "description": "Official Didar docs: this request has no body.", + "additionalProperties": false + }, + "tags": [ + "tag" + ], + "content_type": "application/json", + "response_root": "Response", + "timeout_seconds": 30 + }, + { + "name": "get_lost_reasons", + "title": "Get Lost Reasons", + "description": "List deal lost reasons. Official docs: no request body.", + "entity": "deal", + "effect": "read", + "enabled": true, + "method": "POST", + "path": "/api/Deal/GetLostReasons", + "input_schema": { + "type": "object", + "description": "Official Didar docs: this request has no body.", + "additionalProperties": false + }, + "tags": [ + "deal" + ], + "content_type": "application/json", + "response_root": "Response", + "timeout_seconds": 30 + }, + { + "name": "get_sources", + "title": "Get Sources", + "description": "List lead sources. Official docs: no request body.", + "entity": "source", + "effect": "read", + "enabled": true, + "method": "POST", + "path": "/api/Source/GetList", + "input_schema": { + "type": "object", + "description": "Official Didar docs: this request has no body.", + "additionalProperties": false + }, + "tags": [ + "source" + ], + "content_type": "application/json", + "response_root": "Response", + "timeout_seconds": 30 + }, + { + "name": "get_locations", + "title": "Get Locations", + "description": "List countries, provinces, and cities. Official docs: no request body.", + "entity": "location", + "effect": "read", + "enabled": true, + "method": "POST", + "path": "/api/shared/GetLocations", + "input_schema": { + "type": "object", + "description": "Official Didar docs: this request has no body.", + "additionalProperties": false + }, + "tags": [ + "location" + ], + "content_type": "application/json", + "response_root": "Response", + "timeout_seconds": 30 + } +] diff --git a/kamidso/native_connectors/providers/didar/golden_reconciliation_template.json b/kamidso/native_connectors/providers/didar/golden_reconciliation_template.json new file mode 100644 index 0000000..2590f94 --- /dev/null +++ b/kamidso/native_connectors/providers/didar/golden_reconciliation_template.json @@ -0,0 +1,43 @@ +{ + "schema_version": "1.0", + "provider": "didar", + "description": "Fill this file with values read from the real Didar panel for one test account. Do not commit real customer identifiers or secrets.", + "test_account_alias": "staging-or-sanitized-account", + "timezone": "Asia/Tehran", + "calendar": "jalali", + "currency_profile": { + "base_currency_id": null, + "base_currency_code": null, + "base_currency_name": null, + "display_unit": null + }, + "periods": [ + { + "id": "mehr_sample", + "period_text": "مهر ۱۴۰۵", + "panel_expected": { + "sales_won_count": null, + "sales_amount": null, + "currency_display": null, + "won_count": null, + "lost_count": null, + "pending_count_snapshot": null, + "sales_by_owner": [] + }, + "kamidso_actual": {}, + "tolerance": { + "count": 0, + "amount_absolute": 0, + "amount_relative": 0 + }, + "result": "not_run" + } + ], + "verification_rules": [ + "Counts must match exactly.", + "Money must match within configured tolerance only after currency semantics are known.", + "For sales reports, compare against Didar 'گزارش کلی فروش' for the same period.", + "Record the date basis used by Kamidso.", + "If a mismatch exists, inspect raw response, resolved UTC boundaries, Status, ChangeToWonTime, CurrencyId, and API-key visibility/access before changing semantics." + ] +} \ No newline at end of file diff --git a/kamidso/native_connectors/providers/didar/guidance.md b/kamidso/native_connectors/providers/didar/guidance.md new file mode 100644 index 0000000..8ddfe89 --- /dev/null +++ b/kamidso/native_connectors/providers/didar/guidance.md @@ -0,0 +1,43 @@ +# Didar Semantic Guidance — compact provider guidance + +این فایل برای تزریق کامل به System Prompt نیست. Runtime باید فقط بخش کوتاه و مرتبط را inject کند. + +## قواعد قطعی + +1. وضعیت فعلی معامله فقط از `Status` خوانده می‌شود: + - `Pending` = جاری + - `Won` = موفق + - `Lost` = ناموفق +2. `IsDeleted` یک بعد جدا است. حذف‌شده را با `Lost` یکی نکن. +3. از `ChangeToWonTime`، `PipelineStageId`، `Probability` یا Label برای استنتاج Won/Pending استفاده نکن. +4. `Response.List` در `search_deals` فقط صفحه فعلی است. برای شمارش کل از aggregateهای Response مانند `WonDealsCount` استفاده کن. +5. «فروش در بازه» یعنی معاملات موفق‌شده در آن بازه، نه معاملات ایجادشده در آن بازه. +6. «معاملات ثبت‌شده/ایجادشده در بازه» بر اساس `RegisterTime` است و وضعیت نمایش‌داده‌شده باید وضعیت فعلی `Status` باشد. +7. «معاملات موفق در بازه» بر اساس `ChangeToWonTime` است. +8. «معاملات ناموفق در بازه» بر اساس `ChangeToLossTime` است. +9. برای نام کاربران `OwnerId` را با `get_users` resolve کن؛ قرارداد اتصال را با Golden Test واقعی کنترل کن. +10. برای نام کاریز/مرحله از `get_deal_pipelines` استفاده کن. + +## تاریخ + +- ورودی فارسی مثل «مهر ۱۴۰۵» را در `Asia/Tehran` و تقویم جلالی resolve کن. +- بازه را `[start, end)` نگه دار و سپس به UTC تبدیل کن. +- در پاسخ مدیریتی مبنای تاریخ را صریح کن: «مبنای بازه: تاریخ موفق‌شدن» یا «مبنای بازه: تاریخ ثبت». + +## پول + +- بدون mapping معتبر `CurrencyId` هرگز اسم واحد پول را حدس نزن. +- مبالغ چند CurrencyId را خام با هم جمع نکن. +- گزارش‌های پنل دیدار از قابلیت چندارزی و ارز مبنا استفاده می‌کنند؛ برابری aggregate API با پنل باید با Golden Reconciliation برای اکانت تأیید شود. +- تا قبل از Reconciliation، `FinalAdjustmentPrice` فقط انتخاب موقت رکوردی است و نباید به‌عنوان تعریف ابدی «فروش خالص دیدار» hard-code شود. + +## Lead + +- Function مستقلی مثل `get_leads` یا `search_leads` وجود ندارد؛ آن را اختراع نکن. +- API فیلدهای lead/contact را در Person/Contact نشان می‌دهد، اما یک فرمول عمومی و قطعی برای «تعداد لیدها» در مستندات عمومی تعریف نشده. +- `lead_count` تا زمانی که با پنل واقعی حساب reconcile نشده disabled بماند. + +## Recovery + +- اگر raw function validation failed شد، همان function را با schema برگشتی اصلاح کن. +- اگر semantic operation accuracy guard داد، به endpoint دیگری نپر تا فقط عددی تولید شود؛ دلیل عدم قطعیت را حفظ کن. diff --git a/kamidso/native_connectors/providers/didar/manifest.json b/kamidso/native_connectors/providers/didar/manifest.json new file mode 100644 index 0000000..ad65e44 --- /dev/null +++ b/kamidso/native_connectors/providers/didar/manifest.json @@ -0,0 +1,26 @@ +{ + "id": "didar", + "version": "1.0.0", + "display_name": "دیدار CRM", + "description": "اتصال مستقیم به مشتریان، معاملات، فعالیت‌ها، کاربران و سایر اطلاعات دیدار.", + "category": "crm", + "base_url": "https://app.didar.me", + "authentication": { + "type": "api_key_query", + "parameter_name": "apikey", + "credential_field": "api_key" + }, + "default_headers": { + "Content-Type": "application/json" + }, + "mode": "read_only", + "health_check": { + "function": "get_users", + "body": {} + }, + "aliases": { + "get_leads": ["search_persons", "search_deals"], + "search_leads": ["search_persons", "search_deals"], + "list_leads": ["search_persons", "search_deals"] + } +} diff --git a/kamidso/native_connectors/providers/didar/money_rules.json b/kamidso/native_connectors/providers/didar/money_rules.json new file mode 100644 index 0000000..47d81ac --- /dev/null +++ b/kamidso/native_connectors/providers/didar/money_rules.json @@ -0,0 +1,48 @@ +{ + "schema_version": "1.0", + "provider": "didar", + "source_fields": { + "deal_amount": "Price", + "deal_final_amount": "FinalAdjustmentPrice", + "currency_id": "CurrencyId", + "search_aggregate_total": [ + "Response.TotalPrice", + "Response.AllDealsTotalPrice", + "Response.PendingDealsTotalPrice", + "Response.WonDealsTotalPrice", + "Response.LostDealsTotalPrice" + ] + }, + "didar_product_behavior": { + "multi_currency_supported": true, + "base_currency_exists": true, + "exchange_rates_are_account_configured": true, + "reports_can_be_viewed_by_currency": true + }, + "safe_reporting_policy": { + "single_currency_known_mapping": "Aggregate and show amount with the known unit/code.", + "single_currency_unknown_mapping": "Aggregate only if business semantics otherwise permit, but label the unit as CurrencyId=; do not say ریال/تومان/etc.", + "multiple_currency_ids": "Never sum raw Price/FinalAdjustmentPrice values across currencies.", + "server_aggregate_with_multiple_currencies": "Do not assume the API aggregate equals the Didar dashboard base-currency conversion until a golden reconciliation proves it for the account.", + "conversion": "Only convert when the connector has a verified base-currency/exchange-rate profile sourced from a trustworthy provider surface or explicit account calibration." + }, + "connection_calibration_fields": { + "base_currency_id": null, + "base_currency_code": null, + "base_currency_name": null, + "display_unit": null, + "currency_map": {}, + "dashboard_aggregate_currency_semantics_verified": false + }, + "amount_field_policy": { + "default_record_amount": "FinalAdjustmentPrice", + "status": "provisional_until_golden_reconciliation", + "fallback": "Price", + "rule": "Do not permanently hard-code which field equals Didar's dashboard sales amount until golden tests against the panel pass." + }, + "user_facing_rules": [ + "When currency semantics are unverified, say so briefly rather than inventing a unit.", + "If more than one CurrencyId is present and no verified conversion exists, report separate totals per CurrencyId.", + "A successful HTTP response is not enough to call a money report accurate." + ] +} \ No newline at end of file diff --git a/kamidso/native_connectors/providers/didar/reports.json b/kamidso/native_connectors/providers/didar/reports.json new file mode 100644 index 0000000..96e8b8a --- /dev/null +++ b/kamidso/native_connectors/providers/didar/reports.json @@ -0,0 +1,260 @@ +{ + "schema_version": "1.0", + "provider": "didar", + "semantic_operations": [ + { + "name": "sales_summary", + "title_fa": "خلاصه فروش", + "enabled": true, + "input_schema": { + "type": "object", + "properties": { + "period": { + "type": "string" + }, + "owner_ids": { + "type": "array", + "items": { + "type": "string" + } + }, + "pipeline_id": { + "type": "string" + } + }, + "required": [ + "period" + ], + "additionalProperties": false + }, + "business_definition": "Won deals whose won time belongs to the requested period.", + "date_basis": "won_at", + "raw_function": "search_deals", + "query_template": { + "Criteria": { + "Status": "Won", + "Sort": 9, + "SearchFromTime": "$period.start_utc", + "SearchToTime": "$period.end_utc_exclusive", + "IsDeleted": false + }, + "From": 0, + "Limit": 100 + }, + "preferred_server_fields": [ + "Response.WonDealsCount", + "Response.WonDealsTotalPrice" + ], + "accuracy": { + "must_verify_row_status": true, + "must_verify_won_time_range": true, + "requires_golden_reconciliation_for_panel_parity": true, + "do_not_count_page_list_for_total": true + } + }, + { + "name": "won_deals", + "title_fa": "معاملات موفق در بازه", + "enabled": true, + "input_schema": { + "type": "object", + "properties": { + "period": { + "type": "string" + }, + "owner_ids": { + "type": "array", + "items": { + "type": "string" + } + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 500 + } + }, + "required": [ + "period" + ], + "additionalProperties": false + }, + "business_definition": "Deals currently marked Won whose won time is in the requested period.", + "date_basis": "won_at", + "raw_function": "search_deals", + "status": "Won", + "materialize_records": true + }, + { + "name": "lost_deals", + "title_fa": "معاملات ناموفق در بازه", + "enabled": true, + "input_schema": { + "type": "object", + "properties": { + "period": { + "type": "string" + }, + "owner_ids": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "period" + ], + "additionalProperties": false + }, + "business_definition": "Deals currently marked Lost whose loss time is in the requested period.", + "date_basis": "lost_at", + "raw_function": "search_deals", + "status": "Lost", + "accuracy": { + "server_side_loss_date_filter_verified": false, + "safe_mode": "post_filter_ChangeToLossTime" + } + }, + { + "name": "deals_created", + "title_fa": "معاملات ثبت‌شده در بازه", + "enabled": true, + "input_schema": { + "type": "object", + "properties": { + "period": { + "type": "string" + }, + "status": { + "type": "string", + "enum": [ + "Pending", + "Won", + "Lost" + ] + }, + "owner_ids": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "period" + ], + "additionalProperties": false + }, + "business_definition": "Deals whose RegisterTime belongs to the requested period; current status must be read from Status.", + "date_basis": "created_at", + "raw_function": "search_deals", + "query_sort": 0, + "response_disclosure": "Period is based on deal registration time." + }, + { + "name": "pending_pipeline_snapshot", + "title_fa": "نمای فعلی معاملات جاری", + "enabled": true, + "input_schema": { + "type": "object", + "properties": { + "owner_ids": { + "type": "array", + "items": { + "type": "string" + } + }, + "pipeline_id": { + "type": "string" + }, + "pipeline_stage_ids": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "additionalProperties": false + }, + "business_definition": "Current snapshot of deals with Status=Pending and IsDeleted=false.", + "date_basis": "current_snapshot", + "raw_function": "search_deals", + "status": "Pending" + }, + { + "name": "sales_by_owner", + "title_fa": "فروش به تفکیک کارشناس", + "enabled": true, + "input_schema": { + "type": "object", + "properties": { + "period": { + "type": "string" + } + }, + "required": [ + "period" + ], + "additionalProperties": false + }, + "business_definition": "Won sales in period grouped by deal OwnerId and resolved via get_users.", + "depends_on": [ + "sales_summary", + "get_users" + ], + "group_by": "OwnerId", + "owner_resolution": "get_users", + "requires_code_interpreter": true, + "money_policy_ref": "money_rules.safe_reporting_policy" + }, + { + "name": "sales_over_time", + "title_fa": "روند فروش در زمان", + "enabled": true, + "input_schema": { + "type": "object", + "properties": { + "period": { + "type": "string" + }, + "grain": { + "type": "string", + "enum": [ + "day", + "week", + "month" + ] + } + }, + "required": [ + "period" + ], + "additionalProperties": false + }, + "business_definition": "Won sales grouped by ChangeToWonTime.", + "requires_records": true, + "group_date_field": "ChangeToWonTime", + "requires_code_interpreter": true + }, + { + "name": "sales_by_product", + "title_fa": "فروش به تفکیک محصول", + "enabled": false, + "business_definition": "Sold quantity and amount by product for won deals in period.", + "reason_disabled": "Search Deals may not contain complete Items for every account/result. Enable only after a verified retrieval recipe and dashboard reconciliation are implemented." + }, + { + "name": "sales_by_product_over_time", + "title_fa": "فروش محصول در زمان", + "enabled": false, + "reason_disabled": "Requires verified product-item retrieval and currency semantics." + }, + { + "name": "lead_count", + "title_fa": "تعداد سرنخ", + "enabled": false, + "reason_disabled": "The public API exposes LeadType/LeadStatus/ContactType examples but does not define one authoritative provider-wide lead counting rule. Calibrate against the account UI before enabling." + } + ] +} \ No newline at end of file diff --git a/kamidso/native_connectors/providers/didar/semantic_contract_tests.json b/kamidso/native_connectors/providers/didar/semantic_contract_tests.json new file mode 100644 index 0000000..ec07b4c --- /dev/null +++ b/kamidso/native_connectors/providers/didar/semantic_contract_tests.json @@ -0,0 +1,94 @@ +{ + "schema_version": "1.0", + "provider": "didar", + "tests": [ + { + "id": "deal_status_authoritative", + "input_record": { + "Status": "Pending", + "ChangeToWonTime": "2023-02-06T14:40:29Z", + "Probability": 100 + }, + "expected": { + "canonical_status": "pending" + }, + "assertion": "ChangeToWonTime or Probability must not override Status." + }, + { + "id": "search_deals_page_count_not_total", + "response": { + "Response": { + "WonDealsCount": 651, + "PendingDealsCount": 0, + "LostDealsCount": 0, + "List": [ + { + "Status": "Won" + } + ] + } + }, + "question": "چند معامله موفق داریم؟", + "expected_count_source": "Response.WonDealsCount", + "expected_count": 651 + }, + { + "id": "sales_period_uses_won_time", + "question": "فروش مهر ۱۴۰۵ چقدر بوده؟", + "expected_operation": "sales_summary", + "expected_date_basis": "won_at", + "expected_status": "Won" + }, + { + "id": "generic_deals_period_uses_register_time", + "question": "معاملات مهر ۱۴۰۵ را بده", + "expected_operation": "deals_created", + "expected_date_basis": "created_at", + "required_disclosure": "مبنای بازه: تاریخ ثبت" + }, + { + "id": "multi_currency_no_raw_sum", + "records": [ + { + "FinalAdjustmentPrice": "100", + "CurrencyId": 1 + }, + { + "FinalAdjustmentPrice": "200", + "CurrencyId": 2 + } + ], + "expected": { + "raw_cross_currency_sum_allowed": false + } + }, + { + "id": "unknown_currency_no_label_guess", + "records": [ + { + "FinalAdjustmentPrice": "100", + "CurrencyId": 96107 + } + ], + "calibration": { + "currency_map": {} + }, + "expected": { + "may_say_toman": false, + "may_say_rial": false + } + }, + { + "id": "lead_function_not_invented", + "question": "تعداد لیدهای مهر را بده", + "expected": { + "must_not_call": [ + "get_leads", + "search_leads", + "list_leads" + ], + "semantic_operation_enabled": false + } + } + ] +} \ No newline at end of file diff --git a/kamidso/native_connectors/providers/didar/semantics.json b/kamidso/native_connectors/providers/didar/semantics.json new file mode 100644 index 0000000..a3336ad --- /dev/null +++ b/kamidso/native_connectors/providers/didar/semantics.json @@ -0,0 +1,112 @@ +{ + "schema_version": "1.0", + "provider": "didar", + "entity_semantics": { + "deal": { + "identity": { + "id": "Id", + "code": "Code", + "title": "Title" + }, + "status": { + "field": "Status", + "enum_ref": "enums.authoritative.deal_status", + "authoritative": true, + "never_infer_from": [ + "PipelineStageId", + "Probability", + "ChangeToWonTime", + "ChangeToLossTime", + "LabelIds" + ] + }, + "deletion": { + "field": "IsDeleted", + "separate_from_status": true + }, + "dates": { + "created_at": "RegisterTime", + "won_at": "ChangeToWonTime", + "lost_at": "ChangeToLossTime", + "last_update_at": "LastUpdateTime", + "next_activity_at": "NextActivityDate", + "expected_close_at": "ExpectedCloseDate" + }, + "money": { + "price": "Price", + "final_adjustment_price": "FinalAdjustmentPrice", + "estimated_price": "EstimatedPrice", + "currency_id": "CurrencyId", + "safe_default_amount_field": "FinalAdjustmentPrice", + "safe_default_is_provisional": true, + "reason": "Exact equivalence to Didar dashboard amount must be reconciled on a real account before declaring a permanent canonical amount." + }, + "relations": { + "owner_id": "OwnerId", + "owner_object": "Owner", + "person_id": "PersonId", + "company_id": "CompanyId", + "pipeline_id": "PipelineId", + "pipeline_stage_id": "PipelineStageId", + "source_id": "SourceId", + "lost_reason_id": "LostReasonId" + } + }, + "person": { + "identity": { + "id": "Id", + "display_name": "DisplayName" + }, + "created_at": "RegisterTime", + "deleted": "IsDeleted", + "owner_id": "OwnerId", + "lead_fields": [ + "LeadType", + "LeadStatus", + "LeadGenerationType", + "ContactType" + ], + "lead_semantics_status": "not_fully_documented", + "rule": "Do not implement a provider-wide lead_count until reconciled against the Didar UI/account because public endpoint examples expose lead-related fields but do not define one authoritative counting rule." + }, + "user": { + "resolver_function": "get_users", + "join_rule": "Deal.OwnerId should be resolved primarily against Response[].UserId. Keep fallback to Response[].Id only if a real-account contract test proves it is required.", + "display_field": "DisplayName" + }, + "pipeline": { + "resolver_function": "get_deal_pipelines", + "pipeline_id_path": "Response[].Id", + "stage_id_path": "Response[].Stages[].Id", + "stage_title_path": "Response[].Stages[].Title" + } + }, + "business_language": { + "فروش": "sales", + "فروش موفق": "sales", + "معاملات موفق": "won_deals", + "معاملات ناموفق": "lost_deals", + "معاملات جاری": "pending_pipeline_snapshot", + "معاملات ثبت شده": "deals_created", + "معاملات ایجاد شده": "deals_created" + }, + "ambiguity_defaults": { + "معاملات ": { + "operation": "deals_created", + "date_basis": "created_at", + "response_disclosure": "Always state that the period basis is deal registration time." + }, + "فروش ": { + "operation": "sales_summary", + "date_basis": "won_at", + "response_disclosure": "Always state that sales period is based on deal won time." + } + }, + "accuracy_guards": [ + "Never count statuses from only Response.List when Search Deals aggregate fields are available.", + "For current deal status, always trust Status over timestamps or stage/probability.", + "Never label a monetary amount as ریال/تومان/USD/etc. unless the currency mapping for the connection is known.", + "Never add raw amounts belonging to different CurrencyId values.", + "If a semantic report has not passed reconciliation for the current connector account, use safe-mode verification or clearly report that exact parity with Didar dashboard is not yet verified." + ] +} \ No newline at end of file diff --git a/kamidso/native_connectors/providers/didar/sources.json b/kamidso/native_connectors/providers/didar/sources.json new file mode 100644 index 0000000..52f805a --- /dev/null +++ b/kamidso/native_connectors/providers/didar/sources.json @@ -0,0 +1,57 @@ +{ + "schema_version": "1.0", + "provider": "didar", + "verified_at": "2026-10-12", + "sources": [ + { + "id": "didar_api_start", + "url": "https://developers.didar.me/start-API", + "purpose": "Base URL, API key authentication, request/response conventions, official endpoint index" + }, + { + "id": "didar_search_deals", + "url": "https://developers.didar.me/Search-Deal", + "purpose": "Search Deals request schema, status enum, sort values, aggregates, deal response fields" + }, + { + "id": "didar_get_deal", + "url": "https://developers.didar.me/Get-Deal-By-Id", + "purpose": "Authoritative deal detail fields and evidence that ChangeToWonTime can be populated while Status is Pending" + }, + { + "id": "didar_search_person", + "url": "https://developers.didar.me/Search-Person", + "purpose": "Person search schema, pagination, response fields including LeadType and ContactType" + }, + { + "id": "didar_users", + "url": "https://developers.didar.me/User-List", + "purpose": "User/owner resolution" + }, + { + "id": "didar_pipelines", + "url": "https://developers.didar.me/List-Deal-Pipelines", + "purpose": "Pipeline and stage resolution" + }, + { + "id": "didar_sales_dashboard", + "url": "https://support.didar.me/docs/%d8%af%d8%a7%d8%b4%d8%a8%d9%88%d8%b1%d8%af-%da%af%d8%b2%d8%a7%d8%b1%d8%b4%d8%a7%d8%aa-%da%af%d8%b2%d8%a7%d8%b1%d8%b4-%da%a9%d9%84%db%8c-%d9%81%d8%b1%d9%88%d8%b4/", + "purpose": "Business meaning of sales reports: won deals, by time, user, product, product-over-time" + }, + { + "id": "didar_won_date", + "url": "https://support.didar.me/docs/%da%86%da%af%d9%88%d9%86%d9%87-%d8%aa%d8%a7%d8%b1%db%8c%d8%ae-%d9%85%d9%88%d9%81%d9%82-%d8%b4%d8%af%d9%86-%d9%85%d8%b9%d8%a7%d9%85%d9%84%d9%87-%d8%b1%d8%a7-%d8%aa%d8%ba%db%8c%db%8c%d8%b1-%d8%af%d9%87/", + "purpose": "Meaning of deal won time" + }, + { + "id": "didar_deal_status_filter", + "url": "https://support.didar.me/docs/%d9%81%db%8c%d9%84%d8%aa%d8%b1-%d9%85%d8%b9%d8%a7%d9%85%d9%84%d8%a7%d8%aa-%d8%a8%d8%b1-%d8%a7%d8%b3%d8%a7%d8%b3-%d9%86%d9%88%d8%b9-%d9%85%d8%b9%d8%a7%d9%85%d9%84%d9%87-%d9%85%d9%88%d9%81%d9%82/", + "purpose": "UI concepts: won, lost, pending/current, deleted" + }, + { + "id": "didar_multi_currency", + "url": "https://support.didar.me/docs/%d9%88%db%8c%da%98%da%af%db%8c-%da%86%d9%86%d8%af-%d8%a7%d8%b1%d8%b2%db%8c-%d8%af%d8%b1-%d8%af%db%8c%d8%af%d8%a7%d8%b1/", + "purpose": "Base currency, exchange rate, decimal precision, report currency behavior" + } + ] +} \ No newline at end of file diff --git a/kamidso/native_connectors/redact.py b/kamidso/native_connectors/redact.py new file mode 100644 index 0000000..e4d4540 --- /dev/null +++ b/kamidso/native_connectors/redact.py @@ -0,0 +1,42 @@ +"""Redact secrets from connector observations and logs.""" + +from __future__ import annotations + +import re +from typing import Any, Dict + +_SECRET_KEYS = re.compile( + r"(apikey|api_key|authorization|set-cookie|cookie|token|password|secret)", + re.IGNORECASE, +) +_QUERY_SECRET = re.compile( + r"(?i)(apikey|api_key|token|password|secret)=([^&\s\"']+)" +) + + +def redact_text(value: str) -> str: + return _QUERY_SECRET.sub(r"\1=***", value) + + +def redact(value: Any) -> Any: + if isinstance(value, dict): + cleaned = {} + for key, item in value.items(): + if _SECRET_KEYS.search(str(key)): + cleaned[key] = "***" + else: + cleaned[key] = redact(item) + return cleaned + if isinstance(value, list): + return [redact(item) for item in value] + if isinstance(value, str): + return redact_text(value) + return value + + +def strip_sensitive_fields(body: Dict[str, Any]) -> Dict[str, Any]: + return { + key: value + for key, value in body.items() + if not _SECRET_KEYS.search(str(key)) + } diff --git a/kamidso/native_connectors/registry.py b/kamidso/native_connectors/registry.py new file mode 100644 index 0000000..bc54c72 --- /dev/null +++ b/kamidso/native_connectors/registry.py @@ -0,0 +1,311 @@ +"""Load and cache predefined native connector catalogs.""" + +from __future__ import annotations + +import copy +import json +import logging +from pathlib import Path +from typing import Dict, List, Optional +from urllib.parse import urlparse + +from kamidso.native_connectors.errors import CatalogError, ProviderNotFound +from kamidso.native_connectors.flags import ( + didar_connector_enabled, + is_production, + native_connectors_v1_enabled, +) +from kamidso.native_connectors.models import ConnectorFunction, NativeProvider +from kamidso.native_connectors.validator import ( + check_input_schema, + clear_validator_cache, +) + +logger = logging.getLogger(__name__) + +_ALLOWED_METHODS = {"GET", "POST", "PUT", "PATCH", "DELETE", "HEAD"} +_ALLOWED_EFFECTS = {"read", "write", "delete"} +_GUIDANCE_LIMIT = 8000 +_REGISTRY: Optional["NativeConnectorRegistry"] = None + + +def _read_json(path: Path) -> object: + try: + return json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise CatalogError(f"Cannot read {path.name}: {exc}") from exc + + +def function_from_raw(raw: Dict[str, object]) -> ConnectorFunction: + transport = raw.get("transport") if isinstance(raw.get("transport"), dict) else {} + method = str(transport.get("method") or raw.get("method") or "").upper() + path = str(transport.get("path") or raw.get("path") or "") + response = raw.get("response") if isinstance(raw.get("response"), dict) else {} + root = str(response.get("root") or raw.get("response_root") or "Response") + records_path = response.get("records_path", raw.get("records_path")) + if records_path is not None: + records_path = str(records_path) or None + summary = response.get("summary_fields") or raw.get("summary_fields") or [] + pagination = raw.get("pagination") if isinstance(raw.get("pagination"), dict) else None + if records_path is None and pagination and pagination.get("type") == "offset_limit": + records_path = f"{root}.List" + hints = raw.get("semantic_hints") or [] + schema = raw.get("input_schema") if isinstance(raw.get("input_schema"), dict) else {} + return ConnectorFunction( + name=str(raw.get("name") or ""), + title=str(raw.get("title") or raw.get("name") or ""), + description=str(raw.get("description") or ""), + domain=str(raw.get("domain") or raw.get("entity") or ""), + effect=str(raw.get("effect") or ""), + enabled=bool(raw.get("enabled")), + method=method, + path=path, + input_schema=schema, + response_root=root, + records_path=records_path, + summary_fields=[str(item) for item in summary] if isinstance(summary, list) else [], + semantic_hints=[str(item) for item in hints] if isinstance(hints, list) else [], + pagination=pagination, + content_type=str(raw.get("content_type") or "application/json"), + timeout_seconds=int(raw.get("timeout_seconds") or 30), + ) + + +def validate_provider(provider: NativeProvider) -> None: + parsed = urlparse(provider.base_url) + if parsed.scheme != "https" or not parsed.hostname: + raise CatalogError(f"Provider '{provider.id}' base_url must be https with a host.") + if provider.mode not in {"read_only", "read_write"}: + raise CatalogError(f"Provider '{provider.id}' has unknown mode '{provider.mode}'.") + if not provider.credential_field or not provider.auth_parameter: + raise CatalogError(f"Provider '{provider.id}' is missing authentication fields.") + seen = set() + for function in provider.functions: + if not function.name or function.name in seen: + raise CatalogError( + f"Provider '{provider.id}' has a duplicate or empty function name " + f"'{function.name}'." + ) + seen.add(function.name) + if function.effect not in _ALLOWED_EFFECTS: + raise CatalogError( + f"Function '{function.name}' has unknown effect '{function.effect}'." + ) + if function.method not in _ALLOWED_METHODS: + raise CatalogError( + f"Function '{function.name}' has invalid method '{function.method}'." + ) + if ( + not function.path.startswith("/") + or "://" in function.path + or function.path.startswith("//") + ): + raise CatalogError(f"Function '{function.name}' path must be relative.") + check_input_schema(function.name, function.input_schema) + if provider.mode == "read_only" and function.enabled and function.effect != "read": + raise CatalogError( + f"Function '{function.name}' is enabled but is not a read." + ) + if provider.health_function not in seen: + raise CatalogError( + f"Provider '{provider.id}' health check function " + f"'{provider.health_function}' is missing." + ) + for alias, targets in provider.aliases.items(): + for target in targets: + if target not in seen: + raise CatalogError( + f"Alias '{alias}' points at unknown function '{target}'." + ) + + +def _merge_object_schema(base: Dict[str, object], incoming: Dict[str, object]) -> None: + """Add accuracy-patch fields without replacing richer existing definitions.""" + properties = base.get("properties") + incoming_properties = incoming.get("properties") + if not isinstance(properties, dict): + properties = {} + base["properties"] = properties + if not isinstance(incoming_properties, dict): + return + for key, spec in incoming_properties.items(): + if not isinstance(spec, dict): + continue + current = properties.get(key) + if not isinstance(current, dict): + properties[key] = copy.deepcopy(spec) + continue + if current.get("type") == "object" and spec.get("type") == "object": + _merge_object_schema(current, spec) + continue + if "enum" in spec and "enum" not in current: + current["enum"] = copy.deepcopy(spec["enum"]) + if "minimum" in spec and "minimum" not in current: + current["minimum"] = spec["minimum"] + if "maximum" in spec and "maximum" not in current: + current["maximum"] = spec["maximum"] + + +def _apply_accuracy_patch(directory: Path, functions_raw: List[object]) -> None: + path = directory / "function_accuracy_patch.json" + if not path.is_file(): + return + document = _read_json(path) + if not isinstance(document, dict): + return + patches = document.get("patches") + if not isinstance(patches, list): + return + by_name = { + str(item.get("name")): item + for item in functions_raw + if isinstance(item, dict) and item.get("name") + } + for patch in patches: + if not isinstance(patch, dict): + continue + current = by_name.get(str(patch.get("name") or "")) + if not isinstance(current, dict): + continue + schema = patch.get("input_schema") + existing_schema = current.get("input_schema") + if isinstance(schema, dict) and isinstance(existing_schema, dict): + _merge_object_schema(existing_schema, schema) + hints = patch.get("semantic_hints") + if isinstance(hints, list): + existing_hints = current.get("semantic_hints") + if not isinstance(existing_hints, list): + existing_hints = [] + current["semantic_hints"] = existing_hints + for hint in hints: + if hint not in existing_hints: + existing_hints.append(hint) + contract = patch.get("response_contract") + if not isinstance(contract, dict): + continue + if contract.get("records_path") and not current.get("records_path"): + current["records_path"] = str(contract["records_path"]) + aggregates = contract.get("aggregates") + if isinstance(aggregates, list): + fields = current.get("summary_fields") + if not isinstance(fields, list): + fields = [] + current["summary_fields"] = fields + for field in aggregates: + if field not in fields: + fields.append(field) + + +def load_provider(directory: Path) -> NativeProvider: + manifest_path = directory / "manifest.json" + functions_path = directory / "functions.json" + manifest = _read_json(manifest_path) + functions_raw = _read_json(functions_path) + if not isinstance(manifest, dict): + raise CatalogError(f"{directory.name} manifest must be an object.") + if not isinstance(functions_raw, list): + raise CatalogError(f"{directory.name} functions.json must be an array.") + _apply_accuracy_patch(directory, functions_raw) + auth = manifest.get("authentication") if isinstance(manifest.get("authentication"), dict) else {} + health = manifest.get("health_check") if isinstance(manifest.get("health_check"), dict) else {} + headers = manifest.get("default_headers") if isinstance(manifest.get("default_headers"), dict) else {} + aliases = manifest.get("aliases") if isinstance(manifest.get("aliases"), dict) else {} + guidance_path = directory / "guidance.md" + guidance = "" + if guidance_path.is_file(): + guidance = guidance_path.read_text(encoding="utf-8").strip() + if len(guidance) > _GUIDANCE_LIMIT: + guidance = guidance[:_GUIDANCE_LIMIT] + functions = [ + function_from_raw(item) + for item in functions_raw + if isinstance(item, dict) + ] + if len(functions) != len(functions_raw): + raise CatalogError(f"{directory.name} functions.json contains a non-object entry.") + normalized_aliases: Dict[str, List[str]] = {} + for key, value in aliases.items(): + if isinstance(value, list): + normalized_aliases[str(key)] = [str(item) for item in value] + return NativeProvider( + id=str(manifest.get("id") or directory.name), + version=str(manifest.get("version") or ""), + display_name=str(manifest.get("display_name") or manifest.get("id") or directory.name), + category=str(manifest.get("category") or ""), + base_url=str(manifest.get("base_url") or "").rstrip("/"), + auth_type=str(auth.get("type") or ""), + auth_parameter=str(auth.get("parameter_name") or ""), + credential_field=str(auth.get("credential_field") or ""), + default_headers={str(key): str(value) for key, value in headers.items()}, + mode=str(manifest.get("mode") or "read_only"), + health_function=str(health.get("function") or ""), + health_body=health.get("body") if isinstance(health.get("body"), dict) else {}, + aliases=normalized_aliases, + guidance=guidance, + functions=functions, + description=str(manifest.get("description") or ""), + ) + + +class NativeConnectorRegistry: + def __init__(self, providers: Dict[str, NativeProvider]) -> None: + self._providers = dict(providers) + + def contains(self, connector_type: Optional[str]) -> bool: + return bool(connector_type) and connector_type in self._providers + + def get(self, connector_type: str) -> NativeProvider: + provider = self._providers.get(connector_type) + if provider is None: + raise ProviderNotFound(f"Native provider '{connector_type}' is not available.") + return provider + + def provider_ids(self) -> List[str]: + return sorted(self._providers) + + @classmethod + def load(cls, root: Optional[Path] = None) -> "NativeConnectorRegistry": + clear_validator_cache() + if not native_connectors_v1_enabled(): + return cls({}) + directory = root or Path(__file__).resolve().parent / "providers" + providers: Dict[str, NativeProvider] = {} + if not directory.is_dir(): + return cls(providers) + for child in sorted(path for path in directory.iterdir() if path.is_dir()): + if not (child / "manifest.json").is_file(): + continue + if child.name == "didar" and not didar_connector_enabled(): + continue + try: + provider = load_provider(child) + if provider.id != child.name: + raise CatalogError( + f"Provider id '{provider.id}' does not match directory '{child.name}'." + ) + validate_provider(provider) + except CatalogError as exc: + if is_production(): + logger.critical( + "Disabling native provider '%s': %s", child.name, exc + ) + continue + raise + providers[provider.id] = provider + return cls(providers) + + +def get_registry() -> NativeConnectorRegistry: + global _REGISTRY + if _REGISTRY is None: + _REGISTRY = NativeConnectorRegistry.load() + return _REGISTRY + + +def reset_registry() -> None: + global _REGISTRY + _REGISTRY = None + clear_validator_cache() + from kamidso.native_connectors.semantic.catalog import reset_semantic_cache + + reset_semantic_cache() diff --git a/kamidso/native_connectors/resolve.py b/kamidso/native_connectors/resolve.py new file mode 100644 index 0000000..2f6b10b --- /dev/null +++ b/kamidso/native_connectors/resolve.py @@ -0,0 +1,71 @@ +"""Resolve attached connector ids into MCP ids and public native instances.""" + +from __future__ import annotations + +import logging +from typing import Any, Dict, List, Optional, Tuple + +from kamidso.native_connectors.executor import default_row_loader +from kamidso.native_connectors.flags import native_connectors_v1_enabled +from kamidso.native_connectors.registry import get_registry + +logger = logging.getLogger(__name__) + + +def load_public_instance(connector_id: str) -> Optional[Dict[str, Any]]: + row = default_row_loader(connector_id) + if row is None: + return None + return { + "connector_id": row.get("connector_id"), + "connector_type": row.get("connector_type"), + "display_name": row.get("display_name") or "", + "status": row.get("status") or "", + "user_name": row.get("user_name"), + "sys_code": row.get("sys_code"), + } + + +def partition_connector_ids( + connector_ids: List[str], +) -> Tuple[List[str], List[Dict[str, Any]]]: + """Split selected ids. Native ids never go to the MCP tool selector.""" + if not connector_ids: + return [], [] + if not native_connectors_v1_enabled(): + return list(connector_ids), [] + registry = get_registry() + mcp_ids: List[str] = [] + native: List[Dict[str, Any]] = [] + for connector_id in connector_ids: + row = load_public_instance(connector_id) + if row is None or not registry.contains(str(row.get("connector_type") or "")): + mcp_ids.append(connector_id) + continue + if str(row.get("status") or "") != "active": + logger.warning( + "Skipping inactive native connector '%s'", connector_id + ) + continue + native.append( + { + "connector_id": row["connector_id"], + "connector_type": row["connector_type"], + "display_name": row["display_name"], + } + ) + return mcp_ids, native + + +def install_native_connectors( + react_state: Dict[str, Any], + instances: List[Dict[str, Any]], +) -> None: + react_state["native_connectors"] = { + str(item["connector_id"]): { + "connector_id": item["connector_id"], + "connector_type": item["connector_type"], + "display_name": item.get("display_name") or item["connector_id"], + } + for item in instances + } diff --git a/kamidso/native_connectors/result_store.py b/kamidso/native_connectors/result_store.py new file mode 100644 index 0000000..b6a1bc8 --- /dev/null +++ b/kamidso/native_connectors/result_store.py @@ -0,0 +1,180 @@ +"""Save raw connector responses where code_interpreter can read them later.""" + +from __future__ import annotations + +import json +import os +from typing import Any, Dict, List, Optional, Sequence + +from kamidso.native_connectors.models import ConnectorFunction, NativeProvider + +_PREVIEW_RECORDS = 10 +_CELL_CHARS = 120 +_MAX_OBS_CHARS = 20_000 + + +def get_nested(payload: Any, path: Optional[str]) -> Any: + if not path or not isinstance(payload, dict): + return None + current: Any = payload + for part in path.split("."): + if not isinstance(current, dict) or part not in current: + return None + current = current[part] + return current + + +def _json_default(value: Any) -> str: + if isinstance(value, (bytes, bytearray)): + return value.decode("utf-8", errors="replace") + return str(value) + + +def _cell(value: Any) -> str: + text = str(value).replace("\n", " ") + if len(text) > _CELL_CHARS: + return text[:_CELL_CHARS] + "…" + return text + + +def materialize_connector_result( + react_state: Dict[str, Any], + response_json: Any, + *, + work_dir: Optional[str] = None, +) -> Dict[str, Any]: + """Write the raw API JSON under ``connector_results/``. + + Returns metadata. The file itself stays the untouched API payload. + """ + from kamidso.engine.app.openapi.api_v1.tools.sql_query import session_work_dir + + folder_root = work_dir or session_work_dir(react_state.get("conv_id")) + out_dir = os.path.join(folder_root, "connector_results") + os.makedirs(out_dir, exist_ok=True) + seq = int(react_state.get("connector_result_seq") or 0) + 1 + react_state["connector_result_seq"] = seq + name = f"c_{seq:03d}.json" + payload = json.dumps(response_json, ensure_ascii=False, default=_json_default) + path = os.path.join(out_dir, name) + with open(path, "w", encoding="utf-8") as handle: + handle.write(payload) + with open(os.path.join(out_dir, "latest.json"), "w", encoding="utf-8") as handle: + handle.write(payload) + relative = f"connector_results/{name}" + react_state["latest_connector_result"] = relative + return { + "result_id": f"connector_{seq:03d}", + "relative_path": relative, + "size_bytes": len(payload.encode("utf-8")), + } + + +def _preview_line(index: int, record: Any) -> str: + if isinstance(record, dict): + preferred = [ + key + for key in ("Title", "Name", "FullName", "Id", "UserId") + if key in record + ] + if not preferred: + preferred = list(record.keys())[:3] + bits = [f"{key}={_cell(record.get(key))}" for key in preferred[:3]] + return f"{index}. " + ", ".join(bits) + return f"{index}. {_cell(record)}" + + +def _pagination_lines( + function: ConnectorFunction, + body: Dict[str, Any], + payload: Any, + records: Optional[Sequence[Any]], +) -> List[str]: + pagination = function.pagination or {} + if pagination.get("type") != "offset_limit": + return [] + offset_field = str(pagination.get("offset_field") or "From") + limit_field = str(pagination.get("limit_field") or "Limit") + returned = len(records) if isinstance(records, list) else 0 + lines = [ + f"Current page: {offset_field}={body.get(offset_field)} {limit_field}={body.get(limit_field)}", + f"Returned: {returned}", + ] + total = get_nested(payload, f"{function.response_root}.TotalCount") + if isinstance(total, int): + lines.append(f"TotalCount: {total}") + offset = body.get(offset_field) + if isinstance(offset, int) and offset + returned < total: + lines.append("More data exists.") + return lines + + +def render_success_observation( + *, + provider: NativeProvider, + function: ConnectorFunction, + body: Dict[str, Any], + status_code: int, + payload: Any, + relative_path: str, +) -> str: + lines = [ + f"{provider.display_name} / {function.name} completed successfully.", + "", + f"HTTP: {status_code}", + "", + "Summary:", + ] + summaries = False + for path in function.summary_fields: + value = get_nested(payload, path) + if value is None: + continue + summaries = True + label = path.split(".")[-1] + lines.append(f"- {label}: {value}") + if not summaries: + lines.append("- (no summary fields)") + if function.records_path and function.records_path.endswith(".List") and function.summary_fields: + lines.append( + "Response.List is only the current page. " + "Use the aggregate counts above for totals." + ) + + records = get_nested(payload, function.records_path) if function.records_path else None + if isinstance(records, list): + lines.extend(["", f"Records in current page: {len(records)}", "", "Preview:"]) + if not records: + lines.append("(empty)") + for index, record in enumerate(records[:_PREVIEW_RECORDS], start=1): + lines.append(_preview_line(index, record)) + if len(records) > _PREVIEW_RECORDS: + lines.append(f"(Showing first {_PREVIEW_RECORDS} of {len(records)} records.)") + else: + root = get_nested(payload, function.response_root) + preview = json.dumps(root if root is not None else payload, ensure_ascii=False, default=_json_default) + if len(preview) > 1000: + preview = preview[:1000] + "…" + lines.extend(["", "Object preview:", preview]) + + page_lines = _pagination_lines(function, body, payload, records if isinstance(records, list) else None) + if page_lines: + lines.extend([""] + page_lines) + + lines.extend( + [ + "", + f"[Saved full result → PLOT_DIR/{relative_path}]", + "", + "Use code_interpreter only if computation, dataframe operations, joining, " + "statistics, charts or export are needed.", + ] + ) + text = "\n".join(lines) + if len(text) > _MAX_OBS_CHARS: + text = ( + text[:_MAX_OBS_CHARS] + + f"\n\n... [Output truncated at {_MAX_OBS_CHARS} chars. " + f"Full result at PLOT_DIR/{relative_path}]" + ) + return text diff --git a/kamidso/native_connectors/runtime/__init__.py b/kamidso/native_connectors/runtime/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/kamidso/native_connectors/runtime/attached.py b/kamidso/native_connectors/runtime/attached.py new file mode 100644 index 0000000..f7211d5 --- /dev/null +++ b/kamidso/native_connectors/runtime/attached.py @@ -0,0 +1,12 @@ +"""Public metadata for native connectors attached to the current turn. + +Credentials are never stored on react_state. The executor decrypts them +only while performing a connector_call. +""" + +from kamidso.native_connectors.resolve import ( + install_native_connectors, + partition_connector_ids, +) + +__all__ = ["install_native_connectors", "partition_connector_ids"] diff --git a/kamidso/native_connectors/semantic/__init__.py b/kamidso/native_connectors/semantic/__init__.py new file mode 100644 index 0000000..64a5a71 --- /dev/null +++ b/kamidso/native_connectors/semantic/__init__.py @@ -0,0 +1 @@ +"""Didar semantic accuracy layer. Same connector_call tool, no second runtime.""" diff --git a/kamidso/native_connectors/semantic/aggregates.py b/kamidso/native_connectors/semantic/aggregates.py new file mode 100644 index 0000000..7bbc0a4 --- /dev/null +++ b/kamidso/native_connectors/semantic/aggregates.py @@ -0,0 +1,35 @@ +"""Search Deal totals come from response aggregates, never from the current page length.""" + +from __future__ import annotations + +from typing import Any, Optional, Tuple + +from kamidso.native_connectors.result_store import get_nested +from kamidso.native_connectors.semantic.text import normalize_text + + +def as_int(value: Any) -> Optional[int]: + if isinstance(value, bool) or value is None: + return None + if isinstance(value, int): + return value + if isinstance(value, str) and value.strip().isdigit(): + return int(value.strip()) + return None + + +def aggregate_count(payload: Any, source: str) -> Optional[int]: + return as_int(get_nested(payload, source)) + + +def count_for_question(payload: Any, question: str) -> Tuple[str, Optional[int]]: + text = normalize_text(question) + if "ناموفق" in text: + source = "Response.LostDealsCount" + elif "جاری" in text: + source = "Response.PendingDealsCount" + elif "موفق" in text: + source = "Response.WonDealsCount" + else: + source = "Response.TotalCount" + return source, aggregate_count(payload, source) diff --git a/kamidso/native_connectors/semantic/catalog.py b/kamidso/native_connectors/semantic/catalog.py new file mode 100644 index 0000000..db1bbca --- /dev/null +++ b/kamidso/native_connectors/semantic/catalog.py @@ -0,0 +1,174 @@ +"""Load Didar semantic contracts shipped beside the native provider.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any, Dict, List, Optional + +from kamidso.native_connectors.errors import CatalogError +from kamidso.native_connectors.flags import ( + didar_connector_enabled, + native_connectors_v1_enabled, +) +from kamidso.native_connectors.semantic.text import normalize_text + +_CACHE: Optional[Dict[str, Any]] = None + +_BASIS_DEFAULTS = { + "sales_by_owner": "won_at", + "sales_over_time": "won_at", +} + +_DISCLOSURE = { + "sales_summary": "مبنای بازه: تاریخ موفق‌شدن", + "won_deals": "مبنای بازه: تاریخ موفق‌شدن", + "sales_by_owner": "مبنای بازه: تاریخ موفق‌شدن", + "sales_over_time": "مبنای بازه: تاریخ موفق‌شدن", + "lost_deals": "مبنای بازه: تاریخ ناموفق‌شدن", + "deals_created": "مبنای بازه: تاریخ ثبت", + "pending_pipeline_snapshot": "مبنای گزارش: وضعیت فعلی", +} + + +def provider_dir() -> Path: + return Path(__file__).resolve().parents[1] / "providers" / "didar" + + +def reset_semantic_cache() -> None: + global _CACHE + _CACHE = None + + +def _read(path: Path) -> Dict[str, Any]: + try: + data = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise CatalogError(f"Cannot read {path.name}: {exc}") from exc + if not isinstance(data, dict): + raise CatalogError(f"{path.name} must be an object.") + return data + + +def load_bundle() -> Dict[str, Any]: + global _CACHE + if _CACHE is not None: + return _CACHE + directory = provider_dir() + loaded = { + "reports": _read(directory / "reports.json"), + "dates": _read(directory / "date_rules.json"), + "money": _read(directory / "money_rules.json"), + "semantics": _read(directory / "semantics.json"), + "enums": _read(directory / "enums.json"), + } + _CACHE = loaded + return loaded + + +def operations() -> List[Dict[str, Any]]: + raw = load_bundle()["reports"].get("semantic_operations") or [] + if not isinstance(raw, list): + return [] + return [item for item in raw if isinstance(item, dict) and item.get("name")] + + +def _didar_ready(connector_type: str) -> bool: + return ( + connector_type == "didar" + and native_connectors_v1_enabled() + and didar_connector_enabled() + ) + + +def find_operation(connector_type: str, name: str) -> Optional[Dict[str, Any]]: + if not _didar_ready(connector_type): + return None + for operation in operations(): + if operation.get("name") == name: + return operation + return None + + +def enabled_operations(connector_type: str) -> List[Dict[str, Any]]: + if not _didar_ready(connector_type): + return [] + return [operation for operation in operations() if operation.get("enabled")] + + +def operation_enabled(name: str) -> bool: + for operation in operations(): + if operation.get("name") == name: + return bool(operation.get("enabled")) + return False + + +def date_basis_for(name: str) -> str: + for operation in operations(): + if operation.get("name") == name and operation.get("date_basis"): + return str(operation["date_basis"]) + return _BASIS_DEFAULTS.get(name, "") + + +def disclosure_for(name: str) -> str: + return _DISCLOSURE.get(name, "") + + +def verification_cap() -> int: + try: + value = load_bundle()["dates"]["didar_search_deals_strategy"]["won_at"]["fallback"][ + "max_records" + ] + return int(value) + except (KeyError, TypeError, ValueError): + return 5000 + + +def _month_names() -> List[str]: + months = load_bundle()["dates"].get("jalali_months") or {} + if not isinstance(months, dict): + return [] + return [str(name) for name in months] + + +def _relative_periods() -> List[str]: + parser = load_bundle()["dates"].get("parser_requirements") or {} + raw = parser.get("accept_relative_periods") if isinstance(parser, dict) else [] + if not isinstance(raw, list): + return [] + return [normalize_text(str(item)) for item in raw] + + +def has_period(text: str) -> bool: + normalized = normalize_text(text) + for phrase in _relative_periods(): + if phrase and phrase in normalized: + return True + tokens = set(normalized.split()) + return any(month in tokens for month in _month_names()) + + +def route_question(text: str) -> Optional[str]: + """Map a management question to a semantic operation name.""" + normalized = normalize_text(text) + if "لید" in normalized or "سرنخ" in normalized: + return "lead_count" + if "محصول" in normalized: + return "sales_by_product" + if "کارشناس" in normalized or "به تفکیک مالک" in normalized: + return "sales_by_owner" + if "روند" in normalized: + return "sales_over_time" + if "جاری" in normalized: + return "pending_pipeline_snapshot" + if "ناموفق" in normalized: + return "lost_deals" + if "موفق" in normalized: + if has_period(normalized): + return "won_deals" + return None + if "فروش" in normalized: + return "sales_summary" + if "معاملات" in normalized or "معامله" in normalized: + return "deals_created" + return None diff --git a/kamidso/native_connectors/semantic/dates.py b/kamidso/native_connectors/semantic/dates.py new file mode 100644 index 0000000..025228d --- /dev/null +++ b/kamidso/native_connectors/semantic/dates.py @@ -0,0 +1,236 @@ +"""Resolve Jalali and relative business periods to a half-open UTC range.""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from typing import Optional +from zoneinfo import ZoneInfo + +import jdatetime + +from kamidso.native_connectors.semantic.catalog import load_bundle +from kamidso.native_connectors.semantic.text import normalize_text + +_TEHRAN = ZoneInfo("Asia/Tehran") + + +class PeriodError(ValueError): + pass + + +@dataclass(frozen=True) +class ResolvedPeriod: + input_text: str + calendar: str + timezone: str + start_local: str + end_local_exclusive: str + start_utc: str + end_utc_exclusive: str + start: datetime + end: datetime + + def contains(self, moment: datetime) -> bool: + return self.start <= moment < self.end + + +def parse_utc(value: object) -> Optional[datetime]: + if not isinstance(value, str) or not value.strip(): + return None + text = value.strip().replace("Z", "+00:00") + try: + parsed = datetime.fromisoformat(text) + except ValueError: + return None + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=timezone.utc) + return parsed.astimezone(timezone.utc) + + +def _months() -> dict: + raw = load_bundle()["dates"].get("jalali_months") or {} + return raw if isinstance(raw, dict) else {} + + +def _relatives() -> list: + parser = load_bundle()["dates"].get("parser_requirements") or {} + raw = parser.get("accept_relative_periods") if isinstance(parser, dict) else [] + if not isinstance(raw, list): + return [] + return [normalize_text(str(item)) for item in raw] + + +def _as_tehran(now: Optional[datetime]) -> datetime: + if now is None: + return datetime.now(_TEHRAN) + if now.tzinfo is None: + return now.replace(tzinfo=_TEHRAN) + return now.astimezone(_TEHRAN) + + +def _jalali_today(now: datetime) -> jdatetime.date: + return jdatetime.date.fromgregorian(date=now.date()) + + +def _add_months(year: int, month: int, delta: int) -> tuple: + index = year * 12 + (month - 1) + delta + return index // 12, index % 12 + 1 + + +def _month_bounds(year: int, month: int) -> tuple: + start = jdatetime.date(year, month, 1) + if month == 12: + end = jdatetime.date(year + 1, 1, 1) + else: + end = jdatetime.date(year, month + 1, 1) + return start, end + + +def _format_offset(moment: datetime) -> str: + text = moment.strftime("%z") + if len(text) == 5: + return text[:3] + ":" + text[3:] + return text + + +def _from_jalali(start: jdatetime.date, end: jdatetime.date, text: str) -> ResolvedPeriod: + start_greg = start.togregorian() + end_greg = end.togregorian() + start_local = datetime(start_greg.year, start_greg.month, start_greg.day, tzinfo=_TEHRAN) + end_local = datetime(end_greg.year, end_greg.month, end_greg.day, tzinfo=_TEHRAN) + start_utc = start_local.astimezone(timezone.utc) + end_utc = end_local.astimezone(timezone.utc) + return ResolvedPeriod( + input_text=text, + calendar="jalali", + timezone="Asia/Tehran", + start_local=( + f"{start.year:04d}-{start.month:02d}-{start.day:02d}T00:00:00" + f"{_format_offset(start_local)}" + ), + end_local_exclusive=( + f"{end.year:04d}-{end.month:02d}-{end.day:02d}T00:00:00" + f"{_format_offset(end_local)}" + ), + start_utc=start_utc.strftime("%Y-%m-%dT%H:%M:%SZ"), + end_utc_exclusive=end_utc.strftime("%Y-%m-%dT%H:%M:%SZ"), + start=start_utc, + end=end_utc, + ) + + +def _from_tehran_datetimes(start_local: datetime, end_local: datetime, text: str) -> ResolvedPeriod: + start_j = jdatetime.date.fromgregorian(date=start_local.date()) + end_j = jdatetime.date.fromgregorian(date=end_local.date()) + start_utc = start_local.astimezone(timezone.utc) + end_utc = end_local.astimezone(timezone.utc) + return ResolvedPeriod( + input_text=text, + calendar="jalali", + timezone="Asia/Tehran", + start_local=( + f"{start_j.year:04d}-{start_j.month:02d}-{start_j.day:02d}" + f"T{start_local.strftime('%H:%M:%S')}{_format_offset(start_local)}" + ), + end_local_exclusive=( + f"{end_j.year:04d}-{end_j.month:02d}-{end_j.day:02d}" + f"T{end_local.strftime('%H:%M:%S')}{_format_offset(end_local)}" + ), + start_utc=start_utc.strftime("%Y-%m-%dT%H:%M:%SZ"), + end_utc_exclusive=end_utc.strftime("%Y-%m-%dT%H:%M:%SZ"), + start=start_utc, + end=end_utc, + ) + + +def _relative(normalized: str, now: datetime, original: str) -> Optional[ResolvedPeriod]: + today = datetime(now.year, now.month, now.day, tzinfo=_TEHRAN) + jalali_today = _jalali_today(now) + if normalized == "امروز": + return _from_tehran_datetimes(today, today + timedelta(days=1), original) + if normalized == "دیروز": + return _from_tehran_datetimes(today - timedelta(days=1), today, original) + if normalized in {"این هفته", "هفته قبل"}: + delta = (today.weekday() - 5) % 7 + this_saturday = today - timedelta(days=delta) + if normalized == "هفته قبل": + start = this_saturday - timedelta(days=7) + end = this_saturday + else: + start = this_saturday + end = this_saturday + timedelta(days=7) + return _from_tehran_datetimes(start, end, original) + if normalized == "این ماه": + start, end = _month_bounds(jalali_today.year, jalali_today.month) + return _from_jalali(start, end, original) + if normalized == "ماه قبل": + year, month = _add_months(jalali_today.year, jalali_today.month, -1) + start, end = _month_bounds(year, month) + return _from_jalali(start, end, original) + if normalized == "امسال": + return _from_jalali( + jdatetime.date(jalali_today.year, 1, 1), + jdatetime.date(jalali_today.year + 1, 1, 1), + original, + ) + if normalized == "سال قبل": + return _from_jalali( + jdatetime.date(jalali_today.year - 1, 1, 1), + jdatetime.date(jalali_today.year, 1, 1), + original, + ) + if normalized == "سه ماه اخیر": + start_year, start_month = _add_months(jalali_today.year, jalali_today.month, -2) + end_year, end_month = _add_months(jalali_today.year, jalali_today.month, 1) + return _from_jalali( + jdatetime.date(start_year, start_month, 1), + jdatetime.date(end_year, end_month, 1), + original, + ) + if normalized == "شش ماه اخیر": + start_year, start_month = _add_months(jalali_today.year, jalali_today.month, -5) + end_year, end_month = _add_months(jalali_today.year, jalali_today.month, 1) + return _from_jalali( + jdatetime.date(start_year, start_month, 1), + jdatetime.date(end_year, end_month, 1), + original, + ) + return None + + +def resolve_period(text: str, *, now: Optional[datetime] = None) -> ResolvedPeriod: + original = str(text or "").strip() + normalized = normalize_text(original) + if not normalized: + raise PeriodError("A period is required.") + local_now = _as_tehran(now) + known = set(_relatives()) + if normalized in known: + resolved = _relative(normalized, local_now, original) + if resolved is not None: + return resolved + months = _months() + month_pattern = "|".join(sorted((str(name) for name in months), key=len, reverse=True)) + if not month_pattern: + raise PeriodError(f"Could not resolve period '{original}'.") + match = re.fullmatch( + rf"(?:ماه\s+)?(?P{month_pattern})(?:\s+ماه)?(?:\s+(?P\d{{4}}))?", + normalized, + ) + if match is None: + raise PeriodError( + f"Could not resolve period '{original}'. " + "Use a Jalali month such as مهر ۱۴۰۵ or a relative period such as این ماه." + ) + month = int(months[match.group("month")]) + year_text = match.group("year") + if year_text: + year = int(year_text) + if year < 1200 or year > 1600: + raise PeriodError(f"'{year_text}' is not a Jalali year.") + else: + year = _jalali_today(local_now).year + start, end = _month_bounds(year, month) + return _from_jalali(start, end, original) diff --git a/kamidso/native_connectors/semantic/executor.py b/kamidso/native_connectors/semantic/executor.py new file mode 100644 index 0000000..dc702c2 --- /dev/null +++ b/kamidso/native_connectors/semantic/executor.py @@ -0,0 +1,782 @@ +"""Run one semantic operation through the existing raw connector caller.""" + +from __future__ import annotations + +import copy +import json +import os +import time +from datetime import datetime, timedelta +from typing import Any, Dict, List, Optional +from zoneinfo import ZoneInfo + +import jdatetime +from jsonschema import Draft202012Validator + +from kamidso.native_connectors.errors import ( + AccuracyGuard, + FunctionDisabled, + ResultMaterializationError, +) +from kamidso.native_connectors.result_store import materialize_connector_result +from kamidso.native_connectors.semantic.aggregates import as_int +from kamidso.native_connectors.semantic.catalog import ( + date_basis_for, + disclosure_for, + verification_cap, +) +from kamidso.native_connectors.semantic.dates import ( + PeriodError, + ResolvedPeriod, + parse_utc, + resolve_period, +) +from kamidso.native_connectors.semantic.money import amount_lines, summarize_amounts +from kamidso.native_connectors.validator import _issue, example_skeleton + +_TEHRAN = ZoneInfo("Asia/Tehran") +_PAGE = 100 +_UNTRUSTED = {"won_at", "lost_at"} +_GUARD = ( + "accuracy_guard: this period cannot be fully verified within the row-check limit. " + "Narrow the period. No exact total is reported. " + "عدد دقیق گزارش نمی‌شود؛ بازه را کوچک‌تر کنید." +) + + +def _validation(name: str, schema: Dict[str, Any], body: Any, connector_name: str) -> Optional[Dict[str, Any]]: + if not isinstance(body, dict): + issues = [{"path": [], "message": "Body must be a JSON object."}] + else: + validator = Draft202012Validator(schema if isinstance(schema, dict) else {"type": "object"}) + issues = [ + _issue(error) + for error in sorted(validator.iter_errors(body), key=lambda item: list(item.path)) + ] + if not issues: + return None + return { + "ok": False, + "error_type": "validation_error", + "connector": connector_name, + "function": name, + "message": "Request body does not match the semantic operation.", + "issues": issues, + "expected_schema": schema, + "example_skeleton": example_skeleton(schema if isinstance(schema, dict) else {}), + "http_sent": False, + "instruction": ( + "Fix the body and retry the SAME semantic operation. " + "Pass a business body such as a period. Do not build Didar Criteria." + ), + } + + +def _period_error(name: str, schema: Dict[str, Any], message: str, connector_name: str) -> Dict[str, Any]: + return { + "ok": False, + "error_type": "validation_error", + "connector": connector_name, + "function": name, + "message": message, + "issues": [{"path": ["period"], "message": message}], + "expected_schema": schema, + "example_skeleton": example_skeleton(schema if isinstance(schema, dict) else {}), + "http_sent": False, + "instruction": "Fix the period and retry the SAME semantic operation.", + } + + +def _basis(operation: Dict[str, Any]) -> str: + if operation.get("date_basis"): + return str(operation["date_basis"]) + return date_basis_for(str(operation.get("name") or "")) + + +def _query(operation: Dict[str, Any], period: Optional[ResolvedPeriod], body: Dict[str, Any], basis: str) -> Dict[str, Any]: + template = operation.get("query_template") + query: Dict[str, Any] + if isinstance(template, dict): + query = copy.deepcopy(template) + else: + query = {"Criteria": {}, "From": 0, "Limit": _PAGE} + criteria = query.get("Criteria") + if not isinstance(criteria, dict): + criteria = {} + query["Criteria"] = criteria + if operation.get("status"): + criteria["Status"] = operation["status"] + elif basis == "won_at": + criteria["Status"] = "Won" + elif basis == "lost_at": + criteria["Status"] = "Lost" + elif basis == "current_snapshot": + criteria["Status"] = "Pending" + if operation.get("name") == "deals_created" and body.get("status"): + criteria["Status"] = body["status"] + if "query_sort" in operation: + criteria["Sort"] = operation["query_sort"] + elif basis == "won_at": + criteria.setdefault("Sort", 9) + elif basis == "created_at": + criteria.setdefault("Sort", 0) + criteria["IsDeleted"] = False + if period is not None and basis not in {"current_snapshot", "lost_at"}: + criteria["SearchFromTime"] = period.start_utc + criteria["SearchToTime"] = period.end_utc_exclusive + else: + criteria.pop("SearchFromTime", None) + criteria.pop("SearchToTime", None) + if basis == "lost_at": + criteria.pop("Sort", None) + if body.get("owner_ids"): + criteria["OwnerIds"] = list(body["owner_ids"]) + if body.get("pipeline_id"): + criteria["PipelineId"] = body["pipeline_id"] + if body.get("pipeline_stage_ids"): + criteria["PipelineStageIds"] = list(body["pipeline_stage_ids"]) + query["From"] = 0 + query["Limit"] = _PAGE + return query + + +def _load_saved(react_state: Dict[str, Any], relative_path: str) -> Any: + from kamidso.engine.app.openapi.api_v1.tools.sql_query import session_work_dir + + folder = session_work_dir(react_state.get("conv_id")) + path = os.path.join(folder, *str(relative_path).split("/")) + try: + with open(path, encoding="utf-8") as handle: + return json.load(handle) + except (OSError, json.JSONDecodeError) as exc: + raise ResultMaterializationError( + "The connector response could not be read back for verification.", + ) from exc + + +def _deal_rows(payload: Any) -> List[Dict[str, Any]]: + response = payload.get("Response") if isinstance(payload, dict) else None + if isinstance(response, dict) and isinstance(response.get("List"), list): + return [item for item in response["List"] if isinstance(item, dict)] + return [] + + +def _metric(payload: Any, key: str) -> Optional[int]: + response = payload.get("Response") if isinstance(payload, dict) else None + if isinstance(response, dict): + return as_int(response.get(key)) + return None + + +def _deleted(record: Dict[str, Any]) -> bool: + return record.get("IsDeleted") in (True, 1, "1", "true", "True") + + +def _matches( + record: Dict[str, Any], + basis: str, + period: Optional[ResolvedPeriod], + status_filter: Optional[str], +) -> bool: + if _deleted(record): + return False + if basis == "current_snapshot": + return record.get("Status") == "Pending" + if basis == "created_at": + if status_filter and record.get("Status") != status_filter: + return False + if period is None: + return False + instant = parse_utc(record.get("RegisterTime")) + return instant is not None and period.contains(instant) + expected = "Won" if basis == "won_at" else "Lost" + if record.get("Status") != expected: + return False + if period is None: + return False + field = "ChangeToWonTime" if basis == "won_at" else "ChangeToLossTime" + instant = parse_utc(record.get(field)) + return instant is not None and period.contains(instant) + + +def _calibration(row: Optional[Dict[str, Any]]) -> Dict[str, Any]: + if not isinstance(row, dict): + return {} + config = row.get("config") + if isinstance(config, str): + try: + config = json.loads(config) + except json.JSONDecodeError: + return {} + if not isinstance(config, dict): + return {} + stored = config.get("semantic_calibration") + return stored if isinstance(stored, dict) else {} + + +def _bucket(moment: datetime, grain: str) -> str: + local = moment.astimezone(_TEHRAN) + if grain == "week": + start = (local - timedelta(days=(local.weekday() - 5) % 7)).date() + stamp = jdatetime.date.fromgregorian(date=start) + return f"{stamp.year:04d}-{stamp.month:02d}-{stamp.day:02d}" + stamp_dt = jdatetime.datetime.fromgregorian(datetime=local.replace(tzinfo=None)) + if grain == "day": + return f"{stamp_dt.year:04d}-{stamp_dt.month:02d}-{stamp_dt.day:02d}" + return f"{stamp_dt.year:04d}-{stamp_dt.month:02d}" + + +class SemanticExecutor: + def __init__(self, runner: Any) -> None: + self._runner = runner + + async def execute( + self, + *, + operation: Dict[str, Any], + connector_id: str, + body: Optional[Dict[str, Any]], + react_state: Dict[str, Any], + display_name: str, + ) -> Dict[str, Any]: + name = str(operation.get("name") or "") + if not operation.get("enabled"): + reason = str(operation.get("reason_disabled") or "This semantic operation is disabled.") + raise FunctionDisabled(f"Semantic operation '{name}' is disabled. {reason}", function=name) + schema = operation.get("input_schema") + if not isinstance(schema, dict): + schema = {"type": "object", "additionalProperties": False} + payload_body = body or {} + invalid = _validation(name, schema, payload_body, display_name) + if invalid: + return invalid + basis = _basis(operation) + period: Optional[ResolvedPeriod] = None + needs_period = basis != "current_snapshot" + if needs_period: + try: + period = resolve_period(str(payload_body.get("period") or "")) + except PeriodError as exc: + return _period_error(name, schema, str(exc), display_name) + query = _query(operation, period, payload_body, basis) + status_filter = payload_body.get("status") if name == "deals_created" else None + if not isinstance(status_filter, str): + status_filter = None + scanned = await self._scan( + query=query, + connector_id=connector_id, + react_state=react_state, + display_name=display_name, + function_name=str(operation.get("raw_function") or "search_deals"), + everything=basis in _UNTRUSTED, + ) + if "rows" not in scanned: + return scanned + if basis not in _UNTRUSTED: + sample = [ + row + for row in scanned["rows"] + if _matches(row, basis, period, status_filter) + ] + if len(sample) != len(scanned["rows"]): + total = scanned.get("total") + if isinstance(total, int) and total > verification_cap(): + raise AccuracyGuard(_GUARD, function=name) + scanned = await self._scan( + query=query, + connector_id=connector_id, + react_state=react_state, + display_name=display_name, + function_name=str(operation.get("raw_function") or "search_deals"), + everything=True, + ) + if "rows" not in scanned: + return scanned + basis_trusted = False + else: + basis_trusted = True + else: + basis_trusted = False + matched = [ + row + for row in scanned["rows"] + if _matches(row, basis, period, status_filter) + ] + count, count_source, money_rows, money_withheld = _totals( + name=name, + basis=basis, + scanned=scanned, + matched=matched, + trusted_sample=basis_trusted, + ) + server_count = _server_count(basis, scanned) + calibration = _calibration(self._runner.load_row(connector_id)) + names: Dict[str, str] = {} + pipeline_title = "" + stage_titles: Dict[str, str] = {} + if name == "sales_by_owner": + names = await self._users(connector_id, react_state, display_name) + if payload_body.get("pipeline_id") or payload_body.get("pipeline_stage_ids"): + pipeline_title, stage_titles = await self._pipelines( + connector_id, + react_state, + display_name, + str(payload_body.get("pipeline_id") or ""), + list(payload_body.get("pipeline_stage_ids") or []), + ) + return self._finish( + name=name, + basis=basis, + period=period, + count=count, + count_source=count_source, + money_rows=money_rows, + money_withheld=money_withheld, + server_count=server_count, + calibration=calibration, + matched=matched, + names=names, + pipeline_title=pipeline_title, + stage_titles=stage_titles, + grain=str(payload_body.get("grain") or "month"), + react_state=react_state, + display_name=display_name, + ) + + async def _scan( + self, + *, + query: Dict[str, Any], + connector_id: str, + react_state: Dict[str, Any], + display_name: str, + function_name: str, + everything: bool, + ) -> Dict[str, Any]: + cap = verification_cap() + rows: List[Dict[str, Any]] = [] + snapshot: Dict[str, Optional[int]] = { + "total": None, + "won": None, + "lost": None, + "pending": None, + } + frm = 0 + while True: + page = copy.deepcopy(query) + page["From"] = frm + page["Limit"] = _PAGE + result = await self._runner.execute( + connector_id=connector_id, + function_name=function_name, + body=page, + react_state=react_state, + display_name=display_name, + ) + if not result.get("ok"): + return result + payload = _load_saved(react_state, str(result.get("relative_path") or "")) + page_rows = _deal_rows(payload) + snapshot = { + "total": _metric(payload, "TotalCount"), + "won": _metric(payload, "WonDealsCount"), + "lost": _metric(payload, "LostDealsCount"), + "pending": _metric(payload, "PendingDealsCount"), + } + if everything and isinstance(snapshot["total"], int) and snapshot["total"] > cap: + raise AccuracyGuard(_GUARD, function=function_name) + rows.extend(page_rows) + if isinstance(snapshot["total"], int): + complete = not page_rows or frm + len(page_rows) >= snapshot["total"] + else: + complete = not page_rows or len(page_rows) < _PAGE + packed = {"rows": rows, "complete": complete, **snapshot} + if not everything or complete: + if everything and not rows and isinstance(snapshot["total"], int) and snapshot["total"] > 0: + raise AccuracyGuard(_GUARD, function=function_name) + if ( + everything + and not rows + and snapshot["total"] is None + and snapshot["won"] is None + and snapshot["lost"] is None + ): + raise AccuracyGuard(_GUARD, function=function_name) + return packed + if len(rows) >= cap: + raise AccuracyGuard(_GUARD, function=function_name) + frm += _PAGE + + async def _users( + self, + connector_id: str, + react_state: Dict[str, Any], + display_name: str, + ) -> Dict[str, str]: + cache = react_state.setdefault("connector_metadata_cache", {}) + key = connector_id + ":users" + now = time.monotonic() + if isinstance(cache, dict): + cached = cache.get(key) + if isinstance(cached, dict) and now - float(cached.get("at") or 0) < 300: + names = cached.get("names") + if isinstance(names, dict): + return {str(item): str(value) for item, value in names.items()} + result = await self._runner.execute( + connector_id=connector_id, + function_name="get_users", + body={}, + react_state=react_state, + display_name=display_name, + ) + names: Dict[str, str] = {} + if result.get("ok"): + payload = _load_saved(react_state, str(result.get("relative_path") or "")) + records = payload.get("Response") if isinstance(payload, dict) else None + if isinstance(records, dict): + records = records.get("List") + if isinstance(records, list): + for record in records: + if not isinstance(record, dict): + continue + user_id = record.get("UserId") + label = record.get("DisplayName") + if user_id and isinstance(label, str) and label.strip(): + names[str(user_id)] = label.strip() + if isinstance(cache, dict): + cache[key] = {"at": now, "names": names} + return names + + async def _pipelines( + self, + connector_id: str, + react_state: Dict[str, Any], + display_name: str, + pipeline_id: str, + stage_ids: List[Any], + ) -> tuple: + result = await self._runner.execute( + connector_id=connector_id, + function_name="get_deal_pipelines", + body={}, + react_state=react_state, + display_name=display_name, + ) + titles: Dict[str, str] = {} + stages: Dict[str, str] = {} + if result.get("ok"): + payload = _load_saved(react_state, str(result.get("relative_path") or "")) + records = payload.get("Response") if isinstance(payload, dict) else None + if isinstance(records, dict): + records = records.get("List") + if isinstance(records, list): + for item in records: + if not isinstance(item, dict): + continue + if item.get("Id") and item.get("Title"): + titles[str(item["Id"])] = str(item["Title"]) + for stage in item.get("Stages") or []: + if isinstance(stage, dict) and stage.get("Id") and stage.get("Title"): + stages[str(stage["Id"])] = str(stage["Title"]) + pipeline_title = titles.get(pipeline_id, "") + chosen = {str(item): stages.get(str(item), "") for item in stage_ids} + return pipeline_title, chosen + + def _finish( + self, + *, + name: str, + basis: str, + period: Optional[ResolvedPeriod], + count: int, + count_source: str, + money_rows: List[Dict[str, Any]], + money_withheld: bool, + server_count: Optional[int], + calibration: Dict[str, Any], + matched: List[Dict[str, Any]], + names: Dict[str, str], + pipeline_title: str, + stage_titles: Dict[str, str], + grain: str, + react_state: Dict[str, Any], + display_name: str, + ) -> Dict[str, Any]: + money = summarize_amounts(money_rows, calibration) + panel_parity = bool(calibration.get("panel_parity_verified")) + disagreement = ( + basis in _UNTRUSTED + and server_count is not None + and server_count != count + ) + mode = "safe" + if ( + panel_parity + and not disagreement + and not money_withheld + and not money.mixed + and (money.currency_verified or not money.groups) + ): + mode = "verified" + warnings = _warnings( + panel_parity=panel_parity, + money=money, + money_withheld=money_withheld, + disagreement=disagreement, + currency_verified=money.currency_verified, + ) + groups = _owner_groups(matched, names) if name == "sales_by_owner" else [] + timeline = _timeline(matched, grain) if name == "sales_over_time" else [] + summary = { + "semantic_operation": name, + "accuracy": { + "mode": mode, + "date_basis": basis, + "period": None + if period is None + else { + "input": period.input_text, + "start_utc": period.start_utc, + "end_utc_exclusive": period.end_utc_exclusive, + }, + "currency_verified": money.currency_verified, + "panel_parity_verified": panel_parity, + "warnings": warnings, + }, + "count": count, + "count_source": count_source, + "amounts": money.groups, + "groups": groups, + "timeline": timeline, + "records": matched, + } + meta = materialize_connector_result(react_state, summary) + observation = _render( + display_name=display_name, + name=name, + mode=mode, + basis=basis, + period=period, + count=count, + count_source=count_source, + money=money, + money_withheld=money_withheld, + warnings=warnings, + groups=groups, + timeline=timeline, + pipeline_title=pipeline_title, + stage_titles=stage_titles, + relative_path=meta["relative_path"], + currency_verified=money.currency_verified, + panel_parity=panel_parity, + ) + return { + "ok": True, + "http_sent": True, + "relative_path": meta["relative_path"], + "observation": observation, + } + + +def _totals( + *, + name: str, + basis: str, + scanned: Dict[str, Any], + matched: List[Dict[str, Any]], + trusted_sample: bool, +) -> tuple: + if basis in _UNTRUSTED or not trusted_sample: + return len(matched), "verified_rows", matched, False + if basis == "current_snapshot": + pending = scanned.get("pending") + if isinstance(pending, int): + return pending, "Response.PendingDealsCount", [], True + if scanned.get("complete"): + return len(matched), "verified_rows", [], True + raise AccuracyGuard(_GUARD, function=name) + total = scanned.get("total") + if isinstance(total, int): + complete = bool(scanned.get("complete")) + return total, "Response.TotalCount", matched if complete else [], not complete + if scanned.get("complete"): + return len(matched), "verified_rows", matched, False + raise AccuracyGuard(_GUARD, function=name) + + +def _server_count(basis: str, scanned: Dict[str, Any]) -> Optional[int]: + if basis == "won_at": + value = scanned.get("won") + elif basis == "lost_at": + value = scanned.get("lost") + elif basis == "current_snapshot": + value = scanned.get("pending") + else: + value = scanned.get("total") + return value if isinstance(value, int) else None + + +def _warnings( + *, + panel_parity: bool, + money: Any, + money_withheld: bool, + disagreement: bool, + currency_verified: bool, +) -> List[str]: + warnings: List[str] = [] + if not panel_parity: + warnings.append("Exact parity with the Didar dashboard is not verified for this connection.") + if money_withheld: + warnings.append( + "Amount totals are withheld because the checked page is not the full period." + ) + elif money.mixed: + warnings.append("Amounts from different CurrencyId values are not summed.") + elif money.groups and not currency_verified: + warnings.append("Currency name is not known; amounts are labeled by CurrencyId.") + if disagreement: + warnings.append( + "The server aggregate count was not used because returned rows did not match the period basis." + ) + if money.groups and not money_withheld and not (panel_parity and currency_verified): + warnings.append( + "FinalAdjustmentPrice is a provisional record amount until golden reconciliation." + ) + return warnings + + +def _owner_groups(records: List[Dict[str, Any]], names: Dict[str, str]) -> List[Dict[str, Any]]: + order: List[str] = [] + buckets: Dict[str, List[Dict[str, Any]]] = {} + for record in records: + owner_id = str(record.get("OwnerId") or "") + if owner_id not in buckets: + order.append(owner_id) + buckets[owner_id] = [] + buckets[owner_id].append(record) + groups = [] + unresolved = 0 + for owner_id in order: + label = names.get(owner_id, "") + if not label or label == owner_id: + unresolved += 1 + label = f"کارشناس شناسایی‌نشده {unresolved}" + summary = summarize_amounts(buckets[owner_id], {}) + groups.append( + { + "owner_id": owner_id, + "owner_name": label, + "count": len(buckets[owner_id]), + "amounts": summary.groups, + } + ) + return groups + + +def _timeline(records: List[Dict[str, Any]], grain: str) -> List[Dict[str, Any]]: + order: List[str] = [] + buckets: Dict[str, List[Dict[str, Any]]] = {} + for record in records: + instant = parse_utc(record.get("ChangeToWonTime")) + if instant is None: + continue + key = _bucket(instant, grain if grain in {"day", "week", "month"} else "month") + if key not in buckets: + order.append(key) + buckets[key] = [] + buckets[key].append(record) + timeline = [] + for key in order: + summary = summarize_amounts(buckets[key], {}) + timeline.append( + { + "bucket": key, + "count": len(buckets[key]), + "amounts": summary.groups, + } + ) + return timeline + + +def _render( + *, + display_name: str, + name: str, + mode: str, + basis: str, + period: Optional[ResolvedPeriod], + count: int, + count_source: str, + money: Any, + money_withheld: bool, + warnings: List[str], + groups: List[Dict[str, Any]], + timeline: List[Dict[str, Any]], + pipeline_title: str, + stage_titles: Dict[str, str], + relative_path: str, + currency_verified: bool, + panel_parity: bool, +) -> str: + lines = [ + f"{display_name} / {name} completed.", + "", + "Accuracy:", + f"- mode: {mode}", + f"- date_basis: {basis}", + f"- currency_verified: {str(currency_verified).lower()}", + f"- panel_parity_verified: {str(panel_parity).lower()}", + ] + disclosure = disclosure_for(name) + if disclosure: + lines.append(f"- {disclosure}") + if period is not None: + lines.append(f"- period: {period.input_text}") + lines.append(f"- period_utc: [{period.start_utc}, {period.end_utc_exclusive})") + if pipeline_title: + lines.append(f"- pipeline: {pipeline_title}") + elif stage_titles: + lines.append("- pipeline: کاریز شناسایی‌نشده") + for title in stage_titles.values(): + lines.append(f"- stage: {title or 'مرحله شناسایی‌نشده'}") + lines.extend(["", f"count: {count}", f"count_source: {count_source}"]) + if money_withheld: + lines.append("Amounts: withheld until the full period can be checked.") + elif name not in {"sales_by_owner", "sales_over_time"}: + lines.extend(amount_lines(money)) + if groups: + lines.extend(["", "By owner:"]) + for group in groups: + parts = [f"{group['owner_name']}: count {group['count']}"] + for amount in group["amounts"]: + parts.append(f"{amount['label']} { _plain_amount(amount['amount']) }") + lines.append("- " + "; ".join(parts)) + if timeline: + lines.extend(["", "Over time:"]) + for item in timeline: + parts = [f"{item['bucket']}: count {item['count']}"] + for amount in item["amounts"]: + parts.append(f"{amount['label']} { _plain_amount(amount['amount']) }") + lines.append("- " + "; ".join(parts)) + if warnings: + lines.extend(["", "warnings:"]) + for warning in warnings: + lines.append(f"- {warning}") + lines.extend( + [ + "", + "Keep these accuracy warnings in the final answer.", + f"[Saved semantic result → PLOT_DIR/{relative_path}]", + "", + "Use code_interpreter only for further computation, joining, charts or export. " + "The count above is already verified for this operation.", + ] + ) + return "\n".join(lines) + + +def _plain_amount(value: float) -> str: + if float(value).is_integer(): + return str(int(value)) + return str(value) diff --git a/kamidso/native_connectors/semantic/golden.py b/kamidso/native_connectors/semantic/golden.py new file mode 100644 index 0000000..4db60d9 --- /dev/null +++ b/kamidso/native_connectors/semantic/golden.py @@ -0,0 +1,29 @@ +"""Golden reconciliation stays unverified until a real panel comparison passes.""" + +from __future__ import annotations + +from typing import Any, Dict + + +def reconciliation_verdict(document: Dict[str, Any]) -> Dict[str, Any]: + periods = document.get("periods") if isinstance(document, dict) else None + if not isinstance(periods, list) or not periods: + return {"verified": False, "panel_parity_verified": False, "result": "not_run"} + for period in periods: + if not isinstance(period, dict) or period.get("result") != "pass": + result = "not_run" + if isinstance(period, dict) and period.get("result"): + result = str(period.get("result")) + return {"verified": False, "panel_parity_verified": False, "result": result} + expected = period.get("panel_expected") + if not isinstance(expected, dict): + return {"verified": False, "panel_parity_verified": False, "result": "not_run"} + for key, value in expected.items(): + if key == "sales_by_owner": + continue + if value is None: + return {"verified": False, "panel_parity_verified": False, "result": "not_run"} + actual = period.get("kamidso_actual") + if not isinstance(actual, dict) or not actual: + return {"verified": False, "panel_parity_verified": False, "result": "not_run"} + return {"verified": True, "panel_parity_verified": True, "result": "pass"} diff --git a/kamidso/native_connectors/semantic/money.py b/kamidso/native_connectors/semantic/money.py new file mode 100644 index 0000000..8643e65 --- /dev/null +++ b/kamidso/native_connectors/semantic/money.py @@ -0,0 +1,122 @@ +"""Currency-safe amount reporting. Raw cross-currency sums are never produced.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Dict, List, Optional + +from kamidso.native_connectors.semantic.catalog import load_bundle + + +@dataclass(frozen=True) +class MoneySummary: + groups: List[Dict[str, Any]] = field(default_factory=list) + mixed: bool = False + combined_total: Optional[float] = None + currency_verified: bool = False + + +def _amount_fields() -> Dict[str, str]: + rules = load_bundle()["money"] + sources = rules.get("source_fields") if isinstance(rules.get("source_fields"), dict) else {} + policy = rules.get("amount_field_policy") if isinstance(rules.get("amount_field_policy"), dict) else {} + return { + "final": str(policy.get("default_record_amount") or "FinalAdjustmentPrice"), + "fallback": str(policy.get("fallback") or "Price"), + "currency": str(sources.get("currency_id") or "CurrencyId"), + } + + +def _as_amount(value: Any) -> Optional[float]: + if isinstance(value, bool) or value is None: + return None + if isinstance(value, (int, float)): + return float(value) + if isinstance(value, str): + text = value.strip().replace(",", "") + if not text: + return None + try: + return float(text) + except ValueError: + return None + return None + + +def _label(currency_id: str, currency_map: Dict[str, Any]) -> tuple: + mapped = currency_map.get(currency_id) + if isinstance(mapped, dict): + name = mapped.get("code") or mapped.get("name") + if name: + return str(name), True + if isinstance(mapped, str) and mapped.strip(): + return mapped.strip(), True + shown = currency_id or "unknown" + return f"CurrencyId={shown}", False + + +def summarize_amounts( + records: List[Any], + calibration: Optional[Dict[str, Any]] = None, +) -> MoneySummary: + calibration = calibration or {} + fields = _amount_fields() + currency_map = calibration.get("currency_map") + if not isinstance(currency_map, dict): + currency_map = {} + buckets: Dict[str, float] = {} + order: List[str] = [] + for record in records: + if not isinstance(record, dict): + continue + amount = _as_amount(record.get(fields["final"])) + if amount is None: + amount = _as_amount(record.get(fields["fallback"])) + if amount is None: + continue + currency = record.get(fields["currency"]) + key = "" if currency is None else str(currency) + if key not in buckets: + order.append(key) + buckets[key] = 0.0 + buckets[key] += amount + groups: List[Dict[str, Any]] = [] + for key in order: + label, known = _label(key, currency_map) + groups.append( + { + "currency_id": key, + "label": label, + "amount": buckets[key], + "known": known, + } + ) + mixed = len(groups) > 1 + known_single = len(groups) == 1 and bool(groups[0]["known"]) + verified = bool(calibration.get("dashboard_aggregate_currency_semantics_verified")) and known_single + combined = None if mixed or not groups else groups[0]["amount"] + return MoneySummary( + groups=groups, + mixed=mixed, + combined_total=combined, + currency_verified=verified, + ) + + +def format_amount(value: float) -> str: + if float(value).is_integer(): + return str(int(value)) + return str(value) + + +def amount_lines(summary: MoneySummary) -> List[str]: + if not summary.groups: + return ["Amounts: none on the verified rows."] + if summary.mixed: + lines = ["Amounts by currency (not summed):"] + for group in summary.groups: + lines.append(f"- {group['label']}: {format_amount(group['amount'])}") + lines.append("These currency groups stay separate.") + return lines + group = summary.groups[0] + return [f"Amount: {group['label']} {format_amount(group['amount'])}"] diff --git a/kamidso/native_connectors/semantic/status.py b/kamidso/native_connectors/semantic/status.py new file mode 100644 index 0000000..626a822 --- /dev/null +++ b/kamidso/native_connectors/semantic/status.py @@ -0,0 +1,20 @@ +"""Deal status comes only from the Status field.""" + +from __future__ import annotations + +from typing import Any, Optional + +_CANONICAL = { + "Pending": "pending", + "Won": "won", + "Lost": "lost", +} + + +def canonical_status(record: Any) -> Optional[str]: + if not isinstance(record, dict): + return None + value = record.get("Status") + if isinstance(value, str) and value in _CANONICAL: + return _CANONICAL[value] + return None diff --git a/kamidso/native_connectors/semantic/text.py b/kamidso/native_connectors/semantic/text.py new file mode 100644 index 0000000..f9955d2 --- /dev/null +++ b/kamidso/native_connectors/semantic/text.py @@ -0,0 +1,12 @@ +"""Shared normalization for Persian business text.""" + +from __future__ import annotations + +_DIGITS = str.maketrans("۰۱۲۳۴۵۶۷۸۹٠١٢٣٤٥٦٧٨٩", "01234567890123456789") + + +def normalize_text(value: str) -> str: + text = value.replace("\u200c", "").replace("\u200d", "") + text = text.replace("ي", "ی").replace("ك", "ک").replace("ة", "ه") + text = text.translate(_DIGITS) + return " ".join(text.strip().split()) diff --git a/kamidso/native_connectors/ux.py b/kamidso/native_connectors/ux.py new file mode 100644 index 0000000..b54ae24 --- /dev/null +++ b/kamidso/native_connectors/ux.py @@ -0,0 +1,68 @@ +"""User-facing connector payloads. Credentials never leave this boundary.""" + +from __future__ import annotations + +from typing import Any, Dict + + +def connector_test_message(result: Dict[str, Any]) -> Dict[str, Any]: + """Map a probe into a short message safe to show in the product UI.""" + if result.get("success"): + return {"success": True, "message": "اتصال برقرار شد."} + status = result.get("status_code") + error_type = str(result.get("error_type") or "") + if status == 401 or error_type == "authentication_error": + message = "کلید API معتبر نیست یا دسترسی آن غیرفعال شده است." + elif status == 429 or error_type == "rate_limit_error": + message = "محدودیت تعداد درخواست‌های API دیدار فعال شده است." + elif (isinstance(status, int) and status >= 500) or error_type == "remote_server_error": + message = "سرویس دیدار در حال حاضر پاسخ صحیح نمی‌دهد." + else: + message = "ارتباط با دیدار برقرار نشد." + payload: Dict[str, Any] = {"success": False, "message": message} + if isinstance(status, int): + payload["status_code"] = status + return payload + + +def public_connector(row: Any) -> Dict[str, Any]: + """Sanitized instance. The stored API key is not a field on this object.""" + config = row.config if isinstance(getattr(row, "config", None), dict) else {} + last_tested = config.get("last_tested_at") + last_ok = config.get("last_test_success") + return { + "id": row.connector_id, + "connector_id": row.connector_id, + "connector_type": row.connector_type, + "connector_kind": "native", + "kind": "native", + "display_name": row.display_name, + "status": row.status, + "has_credentials": True, + "last_tested_at": last_tested if isinstance(last_tested, str) else None, + "last_test_success": last_ok if isinstance(last_ok, bool) else None, + } + + +def public_provider(provider: Any) -> Dict[str, Any]: + description = str(getattr(provider, "description", "") or "").strip() + if not description: + description = ( + "اتصال مستقیم به مشتریان، معاملات، فعالیت‌ها، کاربران و سایر اطلاعات دیدار." + ) + return { + "id": provider.id, + "name": provider.display_name, + "description": description, + "category": provider.category or "crm", + "kind": "native", + "auth_fields": [ + { + "name": provider.credential_field, + "label": "کلید API", + "type": "password", + "required": True, + } + ], + } + diff --git a/kamidso/native_connectors/validator.py b/kamidso/native_connectors/validator.py new file mode 100644 index 0000000..4fd89b9 --- /dev/null +++ b/kamidso/native_connectors/validator.py @@ -0,0 +1,127 @@ +"""JSON Schema checks and actionable validation payloads.""" + +from __future__ import annotations + +from typing import Any, Dict, List, Optional, Tuple + +from jsonschema import Draft202012Validator +from jsonschema.exceptions import SchemaError, ValidationError + +from kamidso.native_connectors.errors import CatalogError +from kamidso.native_connectors.models import ConnectorFunction + +_COMPILED: Dict[str, Draft202012Validator] = {} + + +def check_input_schema(function_name: str, schema: Dict[str, Any]) -> None: + if not isinstance(schema, dict) or schema.get("type") != "object": + raise CatalogError( + f"Function '{function_name}' input_schema must be a JSON object schema." + ) + try: + Draft202012Validator.check_schema(schema) + except SchemaError as exc: + raise CatalogError( + f"Function '{function_name}' has an invalid input schema: {exc.message}" + ) from exc + + +def compiled_validator(function: ConnectorFunction) -> Draft202012Validator: + cached = _COMPILED.get(function.name) + schema = function.input_schema + if cached is not None and cached.schema == schema: + return cached + validator = Draft202012Validator(schema) + _COMPILED[function.name] = validator + return validator + + +def clear_validator_cache() -> None: + _COMPILED.clear() + + +def example_skeleton(schema: Dict[str, Any]) -> Any: + if not isinstance(schema, dict): + return {} + if "default" in schema and schema.get("type") != "object": + return schema["default"] + kind = schema.get("type") + if kind == "object": + properties = schema.get("properties") or {} + required = schema.get("required") or [] + skeleton: Dict[str, Any] = {} + for key in required: + skeleton[key] = example_skeleton(properties.get(key) or {"type": "object"}) + return skeleton + if kind == "array": + return [] + if kind == "integer": + minimum = schema.get("minimum", 0) + maximum = schema.get("maximum") + if isinstance(maximum, int) and maximum <= 100 and isinstance(minimum, int): + return maximum + return minimum if isinstance(minimum, int) else 0 + if kind == "number": + minimum = schema.get("minimum", 0) + return minimum + if kind == "boolean": + return False + if kind == "string": + enum = schema.get("enum") + if isinstance(enum, list) and enum: + return enum[0] + return "" + return {} + + +def _issue(error: ValidationError) -> Dict[str, Any]: + path = list(error.path) + message = error.message + if error.validator == "additionalProperties" and isinstance(error.instance, dict): + schema = error.schema if isinstance(error.schema, dict) else {} + allowed = set((schema.get("properties") or {}).keys()) + extras = sorted(key for key in error.instance.keys() if key not in allowed) + if extras: + message = "Unexpected fields: " + ", ".join(extras) + return {"path": path, "message": message} + + +def validate_body( + function: ConnectorFunction, + body: Any, +) -> Tuple[Optional[Dict[str, Any]], List[Dict[str, Any]]]: + """Return ``(normalized_body, issues)``. Issues empty means the body is valid.""" + if body is None: + body = {} + if isinstance(body, str): + return None, [{"path": [], "message": "Body must be a JSON object, not a string."}] + if not isinstance(body, dict): + return None, [{"path": [], "message": "Body must be a JSON object."}] + validator = compiled_validator(function) + errors = sorted(validator.iter_errors(body), key=lambda item: list(item.path)) + if not errors: + return body, [] + return None, [_issue(error) for error in errors] + + +def validation_payload( + *, + connector_name: str, + function: ConnectorFunction, + issues: List[Dict[str, Any]], +) -> Dict[str, Any]: + return { + "ok": False, + "error_type": "validation_error", + "connector": connector_name, + "function": function.name, + "message": "Request body does not match the predefined schema.", + "issues": issues, + "expected_schema": function.input_schema, + "example_skeleton": example_skeleton(function.input_schema), + "semantic_hints": list(function.semantic_hints), + "http_sent": False, + "instruction": ( + "Fix the body and retry the SAME function. Do not invent another function." + ), + } diff --git a/kamidso/resources/registry.py b/kamidso/resources/registry.py index ad272d6..3d1ca3c 100644 --- a/kamidso/resources/registry.py +++ b/kamidso/resources/registry.py @@ -19,6 +19,8 @@ class ResourceRegistry: doc = await self._store.get_datasource(user.workspace_id, ref.id) elif ref.type == "knowledge": doc = await self._store.get_knowledge_source(user.workspace_id, ref.id) + elif ref.type == "connector": + doc = _authorize_connector(user, ref.id) else: raise KamidsoError(KamidsoErrorCode.RESOURCE_NOT_FOUND) if doc is None: @@ -52,3 +54,25 @@ class ResourceRegistry: "status": "ready", } ) + + +def _authorize_connector(user: CurrentUser, connector_id: str) -> Optional[Dict[str, Any]]: + """Public connector row. Credentials stay in the encrypted connector table.""" + from kamidso.native_connectors.resolve import load_public_instance + + row = load_public_instance(connector_id) + if row is None: + return None + owner = str(row.get("user_name") or "") + if owner and owner != user.id: + return None + workspace = str(row.get("sys_code") or "") + if workspace and user.workspace_id and workspace != user.workspace_id: + return None + status = str(row.get("status") or "") + return { + "id": row.get("connector_id"), + "status": status, + "name": row.get("display_name") or connector_id, + "connector_type": row.get("connector_type"), + } diff --git a/kamidso/storage/layout.py b/kamidso/storage/layout.py index e1cdeae..a8bed5f 100644 --- a/kamidso/storage/layout.py +++ b/kamidso/storage/layout.py @@ -140,7 +140,12 @@ class DataLayout: os.environ["HF_HOME"] = str(huggingface) os.environ["HUGGINGFACE_HUB_CACHE"] = str(huggingface / "hub") os.environ["TRANSFORMERS_CACHE"] = str(huggingface / "transformers") - os.environ["KAMIDSO_SKIP_SQLITE_FILE"] = "1" + # Connector rows live in the engine metadata DB. Keep that file under + # data/meta so a process restart does not drop saved connections. + os.environ.pop("KAMIDSO_SKIP_SQLITE_FILE", None) + os.environ["DBGPT_OBSERVABILITY_SQLITE_PATH"] = str( + self.logs / "observability.db" + ) os.environ["KAMIDSO_OBSERVABILITY_JSONL"] = str( self.logs / "observability.jsonl" ) diff --git a/pyproject.toml b/pyproject.toml index 086dc03..b0b827a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,6 +51,7 @@ dependencies = [ "seaborn", "gTTS==2.3.1", "pymysql", + "jdatetime==5.2.0", "jsonschema", "python-jsonpath", "tokenizers>=0.14", diff --git a/requirements.txt b/requirements.txt index 927ae27..601f39f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -45,6 +45,7 @@ coloredlogs seaborn gTTS==2.3.1 pymysql +jdatetime==5.2.0 jsonschema python-jsonpath tokenizers>=0.14 diff --git a/tests/kamidso/test_connector_product.py b/tests/kamidso/test_connector_product.py new file mode 100644 index 0000000..c938606 --- /dev/null +++ b/tests/kamidso/test_connector_product.py @@ -0,0 +1,74 @@ +"""Product-facing connector payloads stay free of secrets.""" + +from kamidso.contracts.user import CurrentUser +from kamidso.engine.serve.connector.service.service import ConnectorResponse +from kamidso.native_connectors.ux import connector_test_message, public_connector + + +def test_probe_messages_are_persian_and_drop_the_remote_body(): + ok = connector_test_message({"success": True, "message": "Connection successful"}) + assert ok == {"success": True, "message": "اتصال برقرار شد."} + + denied = connector_test_message( + { + "success": False, + "status_code": 401, + "error_type": "authentication_error", + "message": "apikey=super-secret", + } + ) + assert denied["success"] is False + assert denied["status_code"] == 401 + assert "super-secret" not in denied["message"] + assert "کلید API" in denied["message"] + + limited = connector_test_message({"success": False, "status_code": 429}) + assert "محدودیت" in limited["message"] + + remote = connector_test_message({"success": False, "status_code": 503}) + assert "سرویس دیدار" in remote["message"] + + offline = connector_test_message({"success": False, "error_type": "transport_error"}) + assert offline["message"] == "ارتباط با دیدار برقرار نشد." + assert "status_code" not in offline + + +def test_public_connector_hides_config_secrets(): + row = ConnectorResponse( + connector_id="conn_1", + connector_type="didar", + display_name="دیدار فروش", + status="active", + config={"api_key": "super-secret", "last_tested_at": "2026-10-12T09:00:00Z", "last_test_success": True}, + ) + public = public_connector(row) + assert public["has_credentials"] is True + assert public["connector_kind"] == "native" + assert public["last_tested_at"] == "2026-10-12T09:00:00Z" + assert "super-secret" not in str(public) + assert "api_key" not in public + assert "config" not in public + + +def test_authorize_connector_rejects_another_workspace(monkeypatch): + from kamidso.resources.registry import _authorize_connector + + monkeypatch.setattr( + "kamidso.native_connectors.resolve.load_public_instance", + lambda connector_id: { + "connector_id": connector_id, + "connector_type": "didar", + "display_name": "دیدار فروش", + "status": "active", + "user_name": "usr_1", + "sys_code": "ws_a", + }, + ) + user = CurrentUser( + id="usr_1", + email="a@kamidso.test", + name="A", + role="user", + workspace_id="ws_b", + ) + assert _authorize_connector(user, "conn_1") is None diff --git a/tests/kamidso/test_context_builder.py b/tests/kamidso/test_context_builder.py index dfa1587..6e08699 100644 --- a/tests/kamidso/test_context_builder.py +++ b/tests/kamidso/test_context_builder.py @@ -161,3 +161,29 @@ def test_selected_skill_appends_settings_and_rule_pack_instructions(): assert "optional hint" in ctx.behavior assert "IR-1405" in ctx.behavior assert "Article 38" in ctx.behavior + + +def test_connector_only_does_not_claim_nothing_is_attached(): + ctx = KamidsoContextBuilder().build( + _user(), + ext_info={ + "connector_ids": ["didar-a"], + "connector_labels": ["دیدار فروش"], + }, + ) + assert "No file or database is attached this turn" not in ctx.behavior + assert "connector_call" in ctx.behavior + assert "دیدار فروش" in ctx.behavior + assert "validation_error" in ctx.behavior + + +def test_connector_with_file_keeps_both_sources(): + ctx = KamidsoContextBuilder().build( + _user(), + files=["file_1"], + ext_info={"connector_ids": ["didar-a"], "connector_labels": ["دیدار فروش"]}, + ) + assert "File(s) are attached this turn" in ctx.behavior + assert "connector_call" in ctx.behavior + assert "code_interpreter" in ctx.behavior + diff --git a/tests/kamidso/test_data_layout.py b/tests/kamidso/test_data_layout.py index 05d543c..840c227 100644 --- a/tests/kamidso/test_data_layout.py +++ b/tests/kamidso/test_data_layout.py @@ -77,6 +77,7 @@ def test_bind_process_env_sets_engine_home(tmp_path, monkeypatch): "HUGGINGFACE_HUB_CACHE", "TRANSFORMERS_CACHE", "KAMIDSO_SKIP_SQLITE_FILE", + "DBGPT_OBSERVABILITY_SQLITE_PATH", "KAMIDSO_OBSERVABILITY_JSONL", "KAMIDSO_SKILLS_DIR", "DBGPT_SKILLS_DIR", @@ -94,7 +95,11 @@ def test_bind_process_env_sets_engine_home(tmp_path, monkeypatch): assert Path(os.environ["CHROMA_PERSIST_PATH"]) == layout.vectors assert Path(os.environ["TMPDIR"]) == layout.temp assert Path(os.environ["HF_HOME"]) == layout.cache / "huggingface" - assert os.environ["KAMIDSO_SKIP_SQLITE_FILE"] == "1" + assert "KAMIDSO_SKIP_SQLITE_FILE" not in os.environ + assert ( + Path(os.environ["DBGPT_OBSERVABILITY_SQLITE_PATH"]) + == layout.logs / "observability.db" + ) assert Path(engine_paths.LOGDIR) == layout.logs assert Path(engine_paths.PILOT_PATH) == layout.runtime assert Path(os.environ["KAMIDSO_VECTORS_PATH"]) == layout.vectors diff --git a/tests/kamidso/test_didar_semantics.py b/tests/kamidso/test_didar_semantics.py new file mode 100644 index 0000000..111b789 --- /dev/null +++ b/tests/kamidso/test_didar_semantics.py @@ -0,0 +1,585 @@ +"""Didar semantic accuracy: status, dates, money, and report routing.""" + +from __future__ import annotations + +import json +from datetime import datetime, timedelta +from zoneinfo import ZoneInfo + +import httpx +import jdatetime +import pytest + +from kamidso.engine.app.openapi.api_v1.tools.connector_call import make_connector_call +from kamidso.engine.core.agent.resource.connector.credential import CredentialStore +from kamidso.native_connectors.executor import NativeConnectorExecutor +from kamidso.native_connectors.prompt import render_native_prompt +from kamidso.native_connectors.registry import get_registry, reset_registry +from kamidso.native_connectors.semantic.aggregates import count_for_question +from kamidso.native_connectors.semantic.catalog import ( + disclosure_for, + find_operation, + operation_enabled, + provider_dir, + route_question, +) +from kamidso.native_connectors.semantic.dates import resolve_period +from kamidso.native_connectors.semantic.golden import reconciliation_verdict +from kamidso.native_connectors.semantic.money import summarize_amounts +from kamidso.native_connectors.semantic.status import canonical_status + +_TEHRAN = ZoneInfo("Asia/Tehran") + + +def _enable(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("KAMIDSO_NATIVE_CONNECTORS_V1", "1") + monkeypatch.setenv("KAMIDSO_DIDAR_CONNECTOR", "1") + monkeypatch.delenv("KAMIDSO_ENV", raising=False) + reset_registry() + + +def _row(store: CredentialStore) -> dict: + salt = store.generate_salt() + return { + "connector_id": "didar-a", + "connector_type": "didar", + "display_name": "دیدار فروش", + "status": "active", + "encrypted_credentials": store.encrypt({"api_key": "super-secret-key"}, salt), + "encryption_salt": salt, + "user_name": "usr_test", + "config": {}, + } + + +def _client(handler, config=None): + store = CredentialStore() + row = _row(store) + if config is not None: + row["config"] = config + transport = httpx.MockTransport(handler) + client = httpx.AsyncClient(transport=transport) + executor = NativeConnectorExecutor( + client=client, + credential_store=store, + row_loader=lambda connector_id: row if connector_id == "didar-a" else None, + ) + return executor, client + + +def _content(text: str) -> str: + payload = json.loads(text) + return payload["chunks"][0]["content"] + + +def _state(): + return { + "conv_id": "conv-semantic", + "user_name": "usr_test", + "native_connectors": { + "didar-a": { + "connector_id": "didar-a", + "connector_type": "didar", + "display_name": "دیدار فروش", + } + }, + } + + +def _won(when: str, amount: str, currency: int, owner: str = "owner-a") -> dict: + return { + "Status": "Won", + "ChangeToWonTime": when, + "FinalAdjustmentPrice": amount, + "CurrencyId": currency, + "OwnerId": owner, + "IsDeleted": False, + } + + +def test_contract_file_locks_status_counts_and_routing(monkeypatch: pytest.MonkeyPatch): + _enable(monkeypatch) + document = json.loads( + (provider_dir() / "semantic_contract_tests.json").read_text(encoding="utf-8") + ) + for case in document["tests"]: + if case["id"] == "deal_status_authoritative": + assert canonical_status(case["input_record"]) == case["expected"]["canonical_status"] + elif case["id"] == "search_deals_page_count_not_total": + source, count = count_for_question(case["response"], case["question"]) + assert source == case["expected_count_source"] + assert count == case["expected_count"] + assert count != len(case["response"]["Response"]["List"]) + elif case["id"] == "sales_period_uses_won_time": + name = route_question(case["question"]) + assert name == case["expected_operation"] + operation = find_operation("didar", name) + assert operation is not None + assert operation["date_basis"] == case["expected_date_basis"] + assert operation["query_template"]["Criteria"]["Status"] == case["expected_status"] + elif case["id"] == "generic_deals_period_uses_register_time": + name = route_question(case["question"]) + assert name == case["expected_operation"] + assert find_operation("didar", name)["date_basis"] == case["expected_date_basis"] + assert case["required_disclosure"] in disclosure_for(name) + elif case["id"] == "multi_currency_no_raw_sum": + summary = summarize_amounts(case["records"], {}) + assert summary.mixed is True + assert summary.combined_total is None + assert case["expected"]["raw_cross_currency_sum_allowed"] is False + elif case["id"] == "unknown_currency_no_label_guess": + summary = summarize_amounts(case["records"], case["calibration"]) + label = summary.groups[0]["label"] + assert label == "CurrencyId=96107" + assert "تومان" not in label + assert "ریال" not in label + assert case["expected"]["may_say_toman"] is False + elif case["id"] == "lead_function_not_invented": + assert route_question(case["question"]) == "lead_count" + assert operation_enabled("lead_count") is False + enabled = {item.name for item in get_registry().get("didar").enabled_reads()} + for blocked in case["expected"]["must_not_call"]: + assert blocked not in enabled + + +def test_accuracy_patch_adds_sort_without_dropping_the_page_cap(monkeypatch: pytest.MonkeyPatch): + _enable(monkeypatch) + deals = get_registry().get("didar").function("search_deals") + criteria = deals.input_schema["properties"]["Criteria"]["properties"] + assert criteria["Sort"]["maximum"] == 10 + assert criteria["Status"]["enum"] == ["Pending", "Won", "Lost"] + assert deals.input_schema["properties"]["Limit"]["maximum"] == 100 + assert "Response.WonDealsTotalPrice" in deals.summary_fields + assert any("Sort=9" in hint for hint in deals.semantic_hints) + users = get_registry().get("didar").function("get_users") + assert users.records_path == "Response" + + +def test_jalali_month_is_half_open_tehran_and_handles_esfand(): + mehr = resolve_period("مهر ۱۴۰۵") + same = resolve_period("مهر 1405") + assert mehr.start_utc == same.start_utc == "2026-09-22T20:30:00Z" + assert mehr.end_utc_exclusive == "2026-10-22T20:30:00Z" + assert mehr.start_local.startswith("1405-07-01T00:00:00") + assert mehr.end_local_exclusive.startswith("1405-08-01T00:00:00") + assert mehr.contains(mehr.start) is True + assert mehr.contains(mehr.end) is False + assert mehr.contains(mehr.end - timedelta(seconds=1)) is True + + esfand = resolve_period("اسفند ۱۴۰۳") + assert esfand.start_local.startswith("1403-12-01") + assert esfand.end_local_exclusive.startswith("1404-01-01") + jdatetime.date(1403, 12, 30) + with pytest.raises(ValueError): + jdatetime.date(1404, 12, 30) + + now = datetime(2026, 10, 11, 12, 0, tzinfo=_TEHRAN) + omitted = resolve_period("مهر", now=now) + assert omitted.start_utc == mehr.start_utc + month = resolve_period("این ماه", now=now) + today = jdatetime.date.fromgregorian(date=now.date()) + assert month.start_local.startswith(f"{today.year:04d}-{today.month:02d}-01") + week = resolve_period("این هفته", now=now) + start = datetime.fromisoformat(week.start_utc.replace("Z", "+00:00")).astimezone(_TEHRAN) + assert start.weekday() == 5 + assert start.hour == 0 + assert week.end - week.start == timedelta(days=7) + + +def test_shipped_golden_template_is_not_verified(): + document = json.loads( + (provider_dir() / "golden_reconciliation_template.json").read_text(encoding="utf-8") + ) + verdict = reconciliation_verdict(document) + assert verdict["verified"] is False + assert verdict["panel_parity_verified"] is False + passed = reconciliation_verdict( + { + "periods": [ + { + "result": "pass", + "panel_expected": { + "sales_won_count": 1, + "sales_amount": 1, + "currency_display": "IRR", + "won_count": 1, + "lost_count": 0, + "pending_count_snapshot": 0, + "sales_by_owner": [], + }, + "kamidso_actual": {"sales_won_count": 1}, + } + ] + } + ) + assert passed["verified"] is True + + +def test_prompt_prefers_semantic_operations(monkeypatch: pytest.MonkeyPatch): + _enable(monkeypatch) + text = render_native_prompt( + [{"connector_id": "didar-a", "connector_type": "didar", "display_name": "دیدار فروش"}] + ) + assert "sales_summary" in text + assert "deals_created" in text + assert "get_leads" in text + assert "additionalProperties" not in text + assert "/api/" not in text + + +@pytest.mark.asyncio +async def test_sales_summary_verifies_won_time_and_refuses_cross_currency_sum( + monkeypatch: pytest.MonkeyPatch, tmp_path +): + _enable(monkeypatch) + monkeypatch.setattr("kamidso.engine.core.configs.model_config.PILOT_PATH", str(tmp_path)) + period = resolve_period("مهر ۱۴۰۵") + calls = [] + + def handler(request: httpx.Request) -> httpx.Response: + calls.append(json.loads(request.content.decode())) + return httpx.Response( + 200, + json={ + "Response": { + "TotalCount": 2, + "WonDealsCount": 2, + "List": [ + _won(period.start_utc, "100", 1), + _won(period.start_utc, "200", 2), + ], + } + }, + ) + + executor, client = _client(handler) + try: + tool = make_connector_call(_state(), executor=executor) + text = _content(await tool(name="sales_summary", body={"period": "مهر ۱۴۰۵"})) + finally: + await client.aclose() + assert len(calls) == 1 + criteria = calls[0]["Criteria"] + assert criteria["Status"] == "Won" + assert criteria["Sort"] == 9 + assert criteria["SearchFromTime"] == period.start_utc + assert criteria["SearchToTime"] == period.end_utc_exclusive + assert criteria["IsDeleted"] is False + assert "From" not in criteria + assert "\ncount: 2\n" in text + assert "date_basis: won_at" in text + assert "مبنای بازه: تاریخ موفق‌شدن" in text + assert "CurrencyId=1" in text + assert "CurrencyId=2" in text + assert "300" not in text + assert "تومان" not in text + assert "ریال" not in text + assert "panel_parity_verified: false" in text + assert "mode: safe" in text + + +@pytest.mark.asyncio +async def test_pending_timestamp_does_not_count_as_won( + monkeypatch: pytest.MonkeyPatch, tmp_path +): + _enable(monkeypatch) + monkeypatch.setattr("kamidso.engine.core.configs.model_config.PILOT_PATH", str(tmp_path)) + period = resolve_period("مهر 1405") + + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response( + 200, + json={ + "Response": { + "TotalCount": 1, + "WonDealsCount": 1, + "List": [ + { + "Status": "Pending", + "ChangeToWonTime": period.start_utc, + "Probability": 100, + "FinalAdjustmentPrice": "40", + "CurrencyId": 96107, + } + ], + } + }, + ) + + executor, client = _client(handler) + try: + tool = make_connector_call(_state(), executor=executor) + text = _content(await tool(name="sales_summary", body={"period": "مهر 1405"})) + finally: + await client.aclose() + assert "\ncount: 0\n" in text + assert "\ncount: 1\n" not in text + assert "was not used" in text + assert "تومان" not in text + assert "ریال" not in text + + +@pytest.mark.asyncio +async def test_deals_created_uses_registration_total_not_the_page( + monkeypatch: pytest.MonkeyPatch, tmp_path +): + _enable(monkeypatch) + monkeypatch.setattr("kamidso.engine.core.configs.model_config.PILOT_PATH", str(tmp_path)) + period = resolve_period("مهر ۱۴۰۵") + calls = [] + + def handler(request: httpx.Request) -> httpx.Response: + calls.append(json.loads(request.content.decode())) + return httpx.Response( + 200, + json={ + "Response": { + "TotalCount": 12, + "List": [ + { + "Status": "Pending", + "ChangeToWonTime": period.start_utc, + "RegisterTime": period.start_utc, + "CurrencyId": 96107, + "FinalAdjustmentPrice": "10", + } + ], + } + }, + ) + + executor, client = _client(handler) + try: + tool = make_connector_call(_state(), executor=executor) + text = _content(await tool(name="deals_created", body={"period": "مهر ۱۴۰۵"})) + finally: + await client.aclose() + assert len(calls) == 1 + assert calls[0]["Criteria"]["Sort"] == 0 + assert "Status" not in calls[0]["Criteria"] + assert "\ncount: 12\n" in text + assert "count_source: Response.TotalCount" in text + assert "مبنای بازه: تاریخ ثبت" in text + assert "date_basis: created_at" in text + assert "Amounts: withheld" in text + + +@pytest.mark.asyncio +async def test_sales_summary_pages_past_the_first_hundred( + monkeypatch: pytest.MonkeyPatch, tmp_path +): + _enable(monkeypatch) + monkeypatch.setattr("kamidso.engine.core.configs.model_config.PILOT_PATH", str(tmp_path)) + period = resolve_period("مهر 1405") + offsets = [] + + def handler(request: httpx.Request) -> httpx.Response: + body = json.loads(request.content.decode()) + offsets.append(body["From"]) + if body["From"] == 0: + rows = [_won(period.start_utc, "1", 1) for _ in range(100)] + else: + rows = [_won(period.start_utc, "1", 1)] + return httpx.Response( + 200, + json={"Response": {"TotalCount": 101, "WonDealsCount": 101, "List": rows}}, + ) + + executor, client = _client(handler) + try: + tool = make_connector_call(_state(), executor=executor) + text = _content(await tool(name="sales_summary", body={"period": "مهر 1405"})) + finally: + await client.aclose() + assert offsets == [0, 100] + assert "\ncount: 101\n" in text + assert "\ncount: 100\n" not in text + + +@pytest.mark.asyncio +async def test_over_cap_period_returns_accuracy_guard_without_a_total( + monkeypatch: pytest.MonkeyPatch, tmp_path +): + _enable(monkeypatch) + monkeypatch.setattr("kamidso.engine.core.configs.model_config.PILOT_PATH", str(tmp_path)) + calls = {"n": 0} + + def handler(request: httpx.Request) -> httpx.Response: + calls["n"] += 1 + return httpx.Response( + 200, + json={ + "Response": { + "TotalCount": 5001, + "WonDealsCount": 5001, + "List": [_won("2026-09-23T00:00:00Z", "9", 1)], + } + }, + ) + + executor, client = _client(handler) + try: + tool = make_connector_call(_state(), executor=executor) + text = await tool(name="sales_summary", body={"period": "مهر ۱۴۰۵"}) + finally: + await client.aclose() + assert calls["n"] == 1 + assert "accuracy_guard" in text + assert "5001" not in text + assert "count:" not in text + + +@pytest.mark.asyncio +async def test_disabled_lead_count_and_malformed_body_send_no_http( + monkeypatch: pytest.MonkeyPatch, tmp_path +): + _enable(monkeypatch) + monkeypatch.setattr("kamidso.engine.core.configs.model_config.PILOT_PATH", str(tmp_path)) + calls = {"n": 0} + + def handler(request: httpx.Request) -> httpx.Response: + calls["n"] += 1 + return httpx.Response(200, json={"Response": []}) + + executor, client = _client(handler) + try: + tool = make_connector_call(_state(), executor=executor) + disabled = await tool(name="lead_count", body={"period": "مهر ۱۴۰۵"}) + invalid = await tool(name="sales_summary", body={"Criteria": {"Status": "Won"}}) + unknown = await tool(name="get_leads", body={}) + finally: + await client.aclose() + assert calls["n"] == 0 + assert "function_disabled" in disabled + assert "validation_error" in invalid + assert "http_sent" in invalid + assert "unknown_function" in unknown + assert "search_persons" in unknown + assert "search_deals" in unknown + + +@pytest.mark.asyncio +async def test_sales_by_owner_resolves_names_and_hides_raw_ids( + monkeypatch: pytest.MonkeyPatch, tmp_path +): + _enable(monkeypatch) + monkeypatch.setattr("kamidso.engine.core.configs.model_config.PILOT_PATH", str(tmp_path)) + period = resolve_period("مهر 1405") + owner_id = "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee" + + def handler(request: httpx.Request) -> httpx.Response: + if request.url.path.endswith("/User/List"): + return httpx.Response( + 200, + json={"Response": [{"UserId": owner_id, "DisplayName": "سارا"}]}, + ) + return httpx.Response( + 200, + json={ + "Response": { + "TotalCount": 1, + "WonDealsCount": 1, + "List": [_won(period.start_utc, "80", 96107, owner_id)], + } + }, + ) + + executor, client = _client(handler) + try: + tool = make_connector_call(_state(), executor=executor) + text = _content(await tool(name="sales_by_owner", body={"period": "مهر 1405"})) + finally: + await client.aclose() + assert "سارا" in text + assert owner_id not in text + assert "CurrencyId=96107" in text + assert "تومان" not in text + assert "ریال" not in text + + +@pytest.mark.asyncio +async def test_calibrated_account_can_be_marked_verified( + monkeypatch: pytest.MonkeyPatch, tmp_path +): + _enable(monkeypatch) + monkeypatch.setattr("kamidso.engine.core.configs.model_config.PILOT_PATH", str(tmp_path)) + period = resolve_period("مهر 1405") + pipeline_id = "11111111-2222-3333-4444-555555555555" + + def handler(request: httpx.Request) -> httpx.Response: + if request.url.path.endswith("/pipeline/list/0"): + return httpx.Response( + 200, + json={"Response": [{"Id": pipeline_id, "Title": "فروش سازمانی", "Stages": []}]}, + ) + return httpx.Response( + 200, + json={ + "Response": { + "TotalCount": 1, + "WonDealsCount": 1, + "List": [_won(period.start_utc, "80", 1)], + } + }, + ) + + executor, client = _client( + handler, + { + "semantic_calibration": { + "panel_parity_verified": True, + "dashboard_aggregate_currency_semantics_verified": True, + "currency_map": {"1": {"code": "IRR"}}, + } + }, + ) + try: + tool = make_connector_call(_state(), executor=executor) + text = _content( + await tool( + name="sales_summary", + body={"period": "مهر 1405", "pipeline_id": pipeline_id}, + ) + ) + finally: + await client.aclose() + assert "mode: verified" in text + assert "currency_verified: true" in text + assert "panel_parity_verified: true" in text + assert "Amount: IRR 80" in text + assert "pipeline: فروش سازمانی" in text + assert pipeline_id not in text + + +@pytest.mark.asyncio +async def test_raw_search_tells_the_model_not_to_count_the_page( + monkeypatch: pytest.MonkeyPatch, tmp_path +): + _enable(monkeypatch) + monkeypatch.setattr("kamidso.engine.core.configs.model_config.PILOT_PATH", str(tmp_path)) + + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response( + 200, + json={ + "Response": { + "TotalCount": 651, + "WonDealsCount": 651, + "List": [{"Title": "Deal", "Status": "Won"}], + } + }, + ) + + executor, client = _client(handler) + try: + result = await executor.execute( + connector_id="didar-a", + function_name="search_deals", + body={"Criteria": {}, "From": 0, "Limit": 100}, + react_state={"conv_id": "conv-semantic"}, + display_name="دیدار فروش", + ) + finally: + await client.aclose() + assert "WonDealsCount: 651" in result["observation"] + assert "only the current page" in result["observation"] diff --git a/tests/kamidso/test_memory_lifecycle.py b/tests/kamidso/test_memory_lifecycle.py new file mode 100644 index 0000000..31e9598 --- /dev/null +++ b/tests/kamidso/test_memory_lifecycle.py @@ -0,0 +1,647 @@ +"""Memory lifecycle: queues, ReAct scratch, adapter cache, DuckDB caps.""" + +from __future__ import annotations + +import asyncio +import gc +import json +import tracemalloc +from types import SimpleNamespace + +import psutil +import pytest + +from kamidso.engine.app.openapi.api_v1.react_runtime import ( + REACT_AGENT_MEMORY_CACHE, + REACT_CONVERSATION_LOCKS, + acquire_conversation_run, + conversation_lock_refcount, + end_react_turn, + release_conversation_run, +) +from kamidso.engine.core.agent.core.memory.gpts.base import GptsMessage +from kamidso.engine.core.agent.core.memory.gpts.gpts_memory import GptsMemory +from kamidso.engine.core.datasource import duckdb_memory as duck_mem +from kamidso.integrations.engine import events as adapter_events +from kamidso.integrations.engine.engine import KamidsoEngine +from kamidso.integrations.engine.events import ( + adapter_cache_size, + adapter_run_is_active, + begin_adapter_run, + map_engine_event, + release_adapter_run, + sweep_adapter_cache, +) + + +def _message( + conv_id: str, + content: str, + *, + rounds: int = 0, + sender: str = "ReAct", + receiver: str = "Human", + action_report: str | None = None, +) -> GptsMessage: + return GptsMessage( + conv_id=conv_id, + sender=sender, + receiver=receiver, + role="ai" if sender != "Human" else "human", + content=content, + rounds=rounds, + app_code="react_agent", + app_name="ReAct", + action_report=action_report, + ) + + +class _Store: + def __init__(self) -> None: + self.rows: list[GptsMessage] = [] + self.deleted: list[str] = [] + + def append(self, message: GptsMessage) -> None: + self.rows.append(message) + + def get_by_conv_id(self, conv_id: str) -> list[GptsMessage]: + return [row for row in self.rows if row.conv_id == conv_id] + + def get_by_agent(self, conv_id: str, agent: str) -> list[GptsMessage]: + return [ + row + for row in self.rows + if row.conv_id == conv_id and (row.sender == agent or row.receiver == agent) + ] + + def get_between_agents(self, conv_id: str, agent1: str, agent2: str, current_goal: str | None = None): + del conv_id, agent1, agent2, current_goal + return [] + + def get_last_message(self, conv_id: str) -> GptsMessage | None: + rows = self.get_by_conv_id(conv_id) + return rows[-1] if rows else None + + def delete_by_conv_id(self, conv_id: str) -> None: + self.deleted.append(conv_id) + + +class _Plans: + def __init__(self) -> None: + self.removed: list[str] = [] + + def remove_by_conv_id(self, conv_id: str) -> None: + self.removed.append(conv_id) + + +class _LatestStore(_Store): + """Keeps one row so a repeated-load test does not retain every payload.""" + + def append(self, message: GptsMessage) -> None: + self.rows = [message] + + +def _memory(store: _Store | None = None) -> tuple[GptsMemory, _Store, _Plans]: + store = store or _Store() + plans = _Plans() + return GptsMemory(plans_memory=plans, message_memory=store), store, plans + + +def _reset_runtime() -> None: + REACT_AGENT_MEMORY_CACHE.clear() + REACT_CONVERSATION_LOCKS.clear() + from kamidso.engine.app.openapi.api_v1 import react_runtime + + react_runtime._LOCK_REFS.clear() + + +def test_vis_off_without_reader_does_not_queue_history_snapshot(): + async def _run() -> None: + memory, store, plans = _memory() + conv = "conv-quiet" + + async def _forbidden(*_args, **_kwargs): + raise AssertionError("full history snapshot was built") + + memory.simple_message = _forbidden # type: ignore[method-assign] + memory.init(conv, enable_vis_message=False) + payload = "x" * 50_000 + for index in range(6): + await memory.append_message( + conv, + _message( + conv, + payload, + rounds=index, + action_report=json.dumps({"content": f"step-{index}-" + payload[:32]}), + ), + ) + queue = memory.queue(conv) + assert queue is not None and queue.empty() + assert len(store.rows) == 6 + assert store.deleted == [] + assert plans.removed == [] + memory.dispose_runtime(conv) + assert memory.queue(conv) is None + assert store.deleted == [] + assert plans.removed == [] + assert len(store.rows) == 6 + + asyncio.run(_run()) + + +def test_vis_off_reader_receives_only_the_latest_message(): + async def _run() -> None: + memory, _store, _plans = _memory() + conv = "conv-reader" + memory.init(conv, enable_vis_message=False) + memory.attach_queue_reader(conv) + for index in range(4): + await memory.append_message( + conv, + _message( + conv, + f"body-{index}", + rounds=index, + action_report=json.dumps({"content": f"view-{index}"}), + ), + ) + queued = [] + queue = memory.queue(conv) + assert queue is not None + while not queue.empty(): + queued.append(queue.get_nowait()) + assert [item[0]["markdown"] for item in queued] == [f"view-{i}" for i in range(4)] + assert all(len(item) == 1 for item in queued) + memory.detach_queue_reader(conv) + await memory.complete(conv) + assert queue.get_nowait() == "[DONE]" + memory.dispose_runtime(conv) + + asyncio.run(_run()) + + +def test_vis_on_still_publishes_the_legacy_view(): + async def _run() -> None: + memory, _store, _plans = _memory() + conv = "conv-vis" + + async def _link(_conv_id: str) -> str: + return "VIS-SNAPSHOT" + + memory.app_link_chat_message = _link # type: ignore[method-assign] + memory.init(conv, enable_vis_message=True) + await memory.append_message(conv, _message(conv, "hello", rounds=0)) + assert memory.queue(conv).get_nowait() == "VIS-SNAPSHOT" + memory.clear(conv) + assert memory._executor is not None + memory.shutdown_executor() + + asyncio.run(_run()) + + +def test_clear_does_not_delete_persisted_rows_or_owned_executor(): + async def _run() -> None: + memory, store, plans = _memory() + conv = "conv-clear" + memory.init(conv, enable_vis_message=False) + await memory.append_message(conv, _message(conv, "kept", rounds=0)) + executor = memory._executor + memory.clear(conv) + assert store.get_by_conv_id(conv)[0].content == "kept" + assert store.deleted == [] + assert plans.removed == [] + assert memory._executor is executor + memory.shutdown_executor() + + asyncio.run(_run()) + + +def test_next_turn_reloads_history_after_dispose(): + async def _run() -> None: + memory, store, plans = _memory() + conv = "conv-reload" + report = json.dumps({"content": "persisted-answer", "is_exe_success": True}) + memory.init(conv, enable_vis_message=False) + await memory.append_message( + conv, + _message(conv, "what is revenue?", rounds=0, sender="Human", receiver="ReAct"), + ) + await memory.append_message( + conv, + _message(conv, "persisted-answer", rounds=1, sender="ReAct", action_report=report), + ) + memory.dispose_runtime(conv) + assert memory._executor is None + assert store.deleted == [] + assert plans.removed == [] + + reloaded, _store, plans2 = _memory(store) + reloaded.init(conv, enable_vis_message=False) + rows = await reloaded.get_messages(conv) + assert [row.content for row in rows] == ["what is revenue?", "persisted-answer"] + history = await reloaded.get_agent_history_memory(conv, "ReAct") + assert history and history[0].content == "persisted-answer" + assert plans2.removed == [] + reloaded.dispose_runtime(conv) + + asyncio.run(_run()) + + +def test_metadb_rows_survive_dispose(tmp_path): + from sqlalchemy.pool import NullPool + + from kamidso.engine.core.storage.metadata.db_manager import db + from kamidso.engine.serve.agent.agents.db_gpts_memory import ( + MetaDbGptsMessageMemory, + MetaDbGptsPlansMemory, + ) + + db_file = tmp_path / "lifecycle.db" + db.init_db( + f"sqlite:///{db_file.as_posix()}", + engine_args={"connect_args": {"check_same_thread": False}, "poolclass": NullPool}, + ) + db.create_all() + + async def _run() -> None: + conv = "conv-metadb-lifecycle" + memory = GptsMemory( + plans_memory=MetaDbGptsPlansMemory(), + message_memory=MetaDbGptsMessageMemory(), + ) + memory.init(conv, enable_vis_message=False) + await memory.append_message(conv, _message(conv, "durable", rounds=1, sender="Human")) + memory.dispose_runtime(conv) + rows = MetaDbGptsMessageMemory().get_by_conv_id(conv) + assert rows and rows[0].content == "durable" + follow = GptsMemory( + plans_memory=MetaDbGptsPlansMemory(), + message_memory=MetaDbGptsMessageMemory(), + ) + follow.init(conv, enable_vis_message=False) + loaded = await follow.get_messages(conv) + assert loaded[0].content == "durable" + follow.dispose_runtime(conv) + + asyncio.run(_run()) + + +def test_same_conversation_is_serialized_and_lock_survives_a_waiter(): + async def _run() -> None: + _reset_runtime() + conv = "conv-same" + order: list[str] = [] + locks: list[asyncio.Lock] = [] + + class _Scratch: + def __init__(self) -> None: + self.disposed = False + + def dispose_runtime(self, conv_id: str | None = None) -> None: + del conv_id + self.disposed = True + + async def _turn(name: str, hold: float) -> None: + lock = await acquire_conversation_run(conv) + locks.append(lock) + order.append(f"enter:{name}") + try: + if conv not in REACT_AGENT_MEMORY_CACHE: + REACT_AGENT_MEMORY_CACHE[conv] = _Scratch() + await asyncio.sleep(hold) + order.append(f"exit:{name}") + finally: + await end_react_turn(conv, lock) + + first = asyncio.create_task(_turn("a", 0.08)) + await asyncio.sleep(0.02) + second = asyncio.create_task(_turn("b", 0.01)) + await asyncio.sleep(0.02) + assert conversation_lock_refcount(conv) == 2 + assert REACT_CONVERSATION_LOCKS[conv].locked() + await first + await second + assert order == ["enter:a", "exit:a", "enter:b", "exit:b"] + assert locks[0] is locks[1] + assert conv not in REACT_CONVERSATION_LOCKS + assert conv not in REACT_AGENT_MEMORY_CACHE + assert conversation_lock_refcount(conv) == 0 + + asyncio.run(_run()) + + +def test_cancelled_waiter_does_not_pin_the_conversation_lock(): + async def _run() -> None: + _reset_runtime() + conv = "conv-cancel-wait" + holder = await acquire_conversation_run(conv) + + async def _wait() -> None: + await acquire_conversation_run(conv) + + waiter = asyncio.create_task(_wait()) + await asyncio.sleep(0.05) + assert conversation_lock_refcount(conv) == 2 + waiter.cancel() + with pytest.raises(asyncio.CancelledError): + await waiter + assert conversation_lock_refcount(conv) == 1 + assert REACT_CONVERSATION_LOCKS[conv] is holder + await release_conversation_run(conv, holder) + assert conv not in REACT_CONVERSATION_LOCKS + + asyncio.run(_run()) + + +def test_twenty_different_conversations_run_together(): + async def _run() -> None: + _reset_runtime() + active = 0 + peak = 0 + + async def _one(index: int) -> None: + nonlocal active, peak + conv = f"user-{index}" + lock = await acquire_conversation_run(conv) + active += 1 + peak = max(peak, active) + await asyncio.sleep(0.05) + active -= 1 + await release_conversation_run(conv, lock) + + await asyncio.gather(*(_one(index) for index in range(20))) + assert peak == 20 + assert REACT_CONVERSATION_LOCKS == {} + + asyncio.run(_run()) + + +def test_repeated_turns_release_scratch_and_allocated_memory_stabilizes(): + async def _cycle(store: _LatestStore, index: int) -> None: + memory, _store, _plans = _memory(store) + conv = "conv-repeat" + memory.init(conv, enable_vis_message=False) + payload = ("row-" + str(index) + "-") * 4_000 + for step in range(6): + await memory.append_message( + conv, + _message( + conv, + payload, + rounds=step, + action_report=json.dumps({"content": payload[:64]}), + ), + ) + assert memory.queue(conv).empty() + executor = memory._executor + memory.dispose_runtime(conv) + assert executor is not None and getattr(executor, "_shutdown", False) + assert memory._executor is None + assert memory.queue(conv) is None + assert conv not in memory.messages_cache + + async def _run() -> None: + store = _LatestStore() + tracemalloc.start() + alloc: list[int] = [] + rss: list[int] = [] + process = psutil.Process() + try: + for index in range(100): + await _cycle(store, index) + if index % 10 == 9: + gc.collect() + current, _peak = tracemalloc.get_traced_memory() + alloc.append(current) + rss.append(process.memory_info().rss) + finally: + tracemalloc.stop() + growth = alloc[-1] - alloc[2] + print( + "LIFECYCLE_MEASURE", + f"alloc_bytes={alloc}", + f"rss_bytes={rss}", + f"alloc_growth={growth}", + f"rss_growth={rss[-1] - rss[2]}", + ) + assert store.deleted == [] + assert len(store.rows) == 1 + # Warmup is the first samples. Later allocated memory must not climb + # with the number of turns. RSS may stay above the process baseline. + assert growth < 8 * 1024 * 1024 + + asyncio.run(_run()) + + +def test_adapter_release_covers_done_error_and_abandoned_runs(): + done_id = "run-done" + error_id = "run-error" + quiet_id = "run-quiet" + active_id = "run-active" + try: + begin_adapter_run(done_id) + mapped = map_engine_event(done_id, {"type": "step.start", "id": "s", "title": "sql"}) + assert mapped and mapped[0]["type"] == "run.started" + assert adapter_run_is_active(done_id) + release_adapter_run(done_id) + assert not adapter_run_is_active(done_id) + + begin_adapter_run(error_id) + try: + map_engine_event(error_id, {"type": "step.start", "id": "s", "title": "code"}) + raise RuntimeError("stream failed") + except RuntimeError: + release_adapter_run(error_id) + assert not adapter_run_is_active(error_id) + + map_engine_event(quiet_id, {"type": "step.start", "id": "s", "title": "sql"}) + begin_adapter_run(active_id) + map_engine_event(active_id, {"type": "step.start", "id": "s", "title": "sql"}) + with adapter_events._LOCK: + adapter_events._TOUCHED[quiet_id] = 0.0 + adapter_events._TOUCHED[active_id] = 0.0 + removed = sweep_adapter_cache(now=10_000.0) + assert quiet_id in removed + assert active_id not in removed + assert adapter_run_is_active(active_id) + finally: + release_adapter_run(done_id) + release_adapter_run(error_id) + release_adapter_run(quiet_id) + release_adapter_run(active_id) + + +def test_release_does_not_drop_another_active_run(): + first = "run-keep" + second = "run-drop" + try: + begin_adapter_run(first) + begin_adapter_run(second) + map_engine_event(first, {"type": "step.start", "id": "a", "title": "sql"}) + map_engine_event(second, {"type": "step.start", "id": "b", "title": "sql"}) + release_adapter_run(second) + assert adapter_run_is_active(first) + assert not adapter_run_is_active(second) + assert adapter_cache_size() >= 1 + finally: + release_adapter_run(first) + release_adapter_run(second) + + +def test_engine_releases_adapter_on_success_error_and_disconnect(): + async def _collect(engine: KamidsoEngine, run_id: str): + run = SimpleNamespace(id=run_id) + events = [] + async for event in engine.execute( + user=None, # type: ignore[arg-type] + run=run, # type: ignore[arg-type] + request=None, # type: ignore[arg-type] + materialized=[], + ): + events.append(event) + return events + + async def _run() -> None: + engine = KamidsoEngine.__new__(KamidsoEngine) + engine._runtime = SimpleNamespace(ready=False) + failed = await _collect(engine, "run-unavailable") + assert failed[0]["error"]["code"] == "ENGINE_UNAVAILABLE" + assert not adapter_run_is_active("run-unavailable") + + engine._runtime = SimpleNamespace(ready=True) + + async def _ok(**_kwargs): + yield {"type": "run.delta"} + + engine._execute_ready = _ok # type: ignore[method-assign] + events = await _collect(engine, "run-ok") + assert events == [{"type": "run.delta"}] + assert not adapter_run_is_active("run-ok") + + async def _boom(**_kwargs): + yield {"type": "partial"} + raise RuntimeError("execution failed") + + engine._execute_ready = _boom # type: ignore[method-assign] + with pytest.raises(RuntimeError): + await _collect(engine, "run-boom") + assert not adapter_run_is_active("run-boom") + + async def _slow(**_kwargs): + yield {"type": "partial"} + await asyncio.sleep(30) + + engine._execute_ready = _slow # type: ignore[method-assign] + generator = engine.execute( + user=None, # type: ignore[arg-type] + run=SimpleNamespace(id="run-disconnect"), # type: ignore[arg-type] + request=None, # type: ignore[arg-type] + materialized=[], + ) + first = await generator.__anext__() + assert first["type"] == "partial" + await generator.aclose() + assert not adapter_run_is_active("run-disconnect") + + asyncio.run(_run()) + + +def _limit_bytes(text: str) -> float: + number, unit = str(text).split() + scales = { + "b": 1, + "byte": 1, + "bytes": 1, + "kib": 1024, + "mib": 1024**2, + "gib": 1024**3, + "tib": 1024**4, + "kb": 1000, + "mb": 1000**2, + "gb": 1000**3, + "tb": 1000**4, + } + return float(number) * scales[unit.lower()] + + +def test_duckdb_invalid_limit_falls_back_to_two_gigabytes(monkeypatch): + monkeypatch.setenv("KAMIDSO_DUCKDB_MEMORY_LIMIT", "not-a-size") + monkeypatch.delenv("KAMIDSO_DUCKDB_AGGREGATE_MEMORY_LIMIT", raising=False) + assert duck_mem.configured_memory_limit() == "2GB" + assert duck_mem.configured_aggregate_limit() == "2GB" + + +def test_duckdb_tool_connections_share_one_aggregate_budget(monkeypatch): + monkeypatch.setenv("KAMIDSO_DUCKDB_MEMORY_LIMIT", "2GB") + monkeypatch.delenv("KAMIDSO_DUCKDB_AGGREGATE_MEMORY_LIMIT", raising=False) + duck_mem.reset_duckdb_memory_state_for_tests() + duck_mem.cap_duckdb_default_connection() + first = duck_mem.open_tool_duckdb(":memory:") + second = None + try: + alone = first.execute("SELECT current_setting('memory_limit')").fetchone()[0] + assert _limit_bytes(alone) == pytest.approx(2_000_000_000, rel=0.15) + assert len(duck_mem._LIVE) == 1 + second = duck_mem.open_tool_duckdb(":memory:") + limits = [ + _limit_bytes(conn.execute("SELECT current_setting('memory_limit')").fetchone()[0]) + for conn in (first, second) + ] + assert all(700 * 1024**2 < limit < 1.2 * 1024**3 for limit in limits) + duck_mem.release_tool_duckdb(second) + second.close() + second = None + restored = first.execute("SELECT current_setting('memory_limit')").fetchone()[0] + assert _limit_bytes(restored) == pytest.approx(2_000_000_000, rel=0.15) + finally: + duck_mem.release_tool_duckdb(first) + first.close() + if second is not None: + duck_mem.release_tool_duckdb(second) + second.close() + duck_mem.reset_duckdb_memory_state_for_tests() + + +def test_excel_reader_registers_and_releases_duckdb(tmp_path, monkeypatch): + monkeypatch.setenv("KAMIDSO_DUCKDB_MEMORY_LIMIT", "2GB") + monkeypatch.delenv("KAMIDSO_DUCKDB_AGGREGATE_MEMORY_LIMIT", raising=False) + duck_mem.reset_duckdb_memory_state_for_tests() + from kamidso.engine.app.scene.chat_data.chat_excel.excel_reader import ExcelReader + + csv_path = tmp_path / "tiny.csv" + csv_path.write_text("city,amount\nShiraz,10\nTehran,20\n", encoding="utf-8") + monkeypatch.setattr( + "kamidso.engine.core.util.file_client.knowledge_upload_root", + lambda: str(tmp_path), + ) + reader = ExcelReader("conv-excel", str(csv_path), read_type="df") + live_after_close = None + try: + limit = reader.db.execute("SELECT current_setting('memory_limit')").fetchone()[0] + assert _limit_bytes(limit) == pytest.approx(2_000_000_000, rel=0.15) + assert len(duck_mem._LIVE) == 1 + rows = reader.db.execute("SELECT COUNT(*) FROM temp_table").fetchone()[0] + assert rows == 2 + finally: + reader.close() + live_after_close = dict(duck_mem._LIVE) + duck_mem.reset_duckdb_memory_state_for_tests() + assert reader.db is None + assert live_after_close == {} + + +def test_duckdb_connector_binds_sqlalchemy_connections(tmp_path, monkeypatch): + monkeypatch.setenv("KAMIDSO_DUCKDB_MEMORY_LIMIT", "2GB") + monkeypatch.delenv("KAMIDSO_DUCKDB_AGGREGATE_MEMORY_LIMIT", raising=False) + duck_mem.reset_duckdb_memory_state_for_tests() + from kamidso.engine.ext.datasource.rdbms.conn_duckdb import DuckDbConnector + + connector = DuckDbConnector.from_file_path(str(tmp_path / "analytics.duckdb")) + try: + with connector._engine.connect() as connection: + limit = connection.exec_driver_sql("SELECT current_setting('memory_limit')").scalar() + assert _limit_bytes(str(limit)) == pytest.approx(2_000_000_000, rel=0.15) + finally: + connector._engine.dispose() + duck_mem.reset_duckdb_memory_state_for_tests() diff --git a/tests/kamidso/test_native_connector.py b/tests/kamidso/test_native_connector.py new file mode 100644 index 0000000..3920dee --- /dev/null +++ b/tests/kamidso/test_native_connector.py @@ -0,0 +1,482 @@ +"""Native Didar connector: catalog, validation, HTTP boundary, and handoff.""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys + +import httpx +import pytest + +from kamidso.contracts.chat import KamidsoChatRequest +from kamidso.engine.app.openapi.api_v1.tools.code_interpreter import make_code_interpreter +from kamidso.engine.app.openapi.api_v1.tools.connector_call import make_connector_call +from kamidso.engine.core.agent.resource.connector.credential import CredentialStore +from kamidso.integrations.engine.engine import connector_selection +from kamidso.native_connectors.errors import CatalogError +from kamidso.native_connectors.executor import NativeConnectorExecutor +from kamidso.native_connectors.prompt import render_native_prompt +from kamidso.native_connectors.registry import ( + NativeConnectorRegistry, + get_registry, + reset_registry, +) +from kamidso.native_connectors.resolve import partition_connector_ids +from kamidso.native_connectors.result_store import materialize_connector_result + + +def _enable(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("KAMIDSO_NATIVE_CONNECTORS_V1", "1") + monkeypatch.setenv("KAMIDSO_DIDAR_CONNECTOR", "1") + monkeypatch.delenv("KAMIDSO_ENV", raising=False) + reset_registry() + + +def _provider(): + return get_registry().get("didar") + + +def test_didar_catalog_is_fixed_and_read_only(monkeypatch: pytest.MonkeyPatch): + _enable(monkeypatch) + provider = _provider() + names = [item.name for item in provider.functions] + assert len(names) == 49 + assert len(set(names)) == 49 + assert provider.base_url == "https://app.didar.me" + for function in provider.functions: + assert function.path.startswith("/") + assert "://" not in function.path + assert function.method in {"GET", "POST", "PUT", "PATCH", "DELETE"} + if function.enabled: + assert function.effect == "read" + deals = provider.function("search_deals") + assert deals is not None + assert deals.input_schema["required"] == ["Criteria", "From", "Limit"] + assert deals.records_path == "Response.List" + + +def test_invalid_catalog_fails_fast_outside_production(tmp_path, monkeypatch): + monkeypatch.setenv("KAMIDSO_NATIVE_CONNECTORS_V1", "1") + monkeypatch.delenv("KAMIDSO_ENV", raising=False) + _write_bad_provider(tmp_path) + with pytest.raises(CatalogError): + NativeConnectorRegistry.load(tmp_path) + + +def test_invalid_catalog_is_disabled_in_production(tmp_path, monkeypatch): + monkeypatch.setenv("KAMIDSO_NATIVE_CONNECTORS_V1", "1") + monkeypatch.setenv("KAMIDSO_ENV", "production") + _write_bad_provider(tmp_path) + registry = NativeConnectorRegistry.load(tmp_path) + assert registry.provider_ids() == [] + + +def _write_bad_provider(root): + directory = root / "bad" + directory.mkdir() + (directory / "manifest.json").write_text( + json.dumps( + { + "id": "bad", + "version": "0", + "display_name": "Bad", + "base_url": "https://example.com", + "authentication": { + "type": "api_key_query", + "parameter_name": "apikey", + "credential_field": "api_key", + }, + "mode": "read_only", + "health_check": {"function": "ping", "body": {}}, + } + ), + encoding="utf-8", + ) + (directory / "functions.json").write_text( + json.dumps( + [ + { + "name": "ping", + "description": "ping", + "entity": "ping", + "effect": "read", + "enabled": True, + "method": "GET", + "path": "/ping", + "input_schema": {"type": "object"}, + }, + { + "name": "ping", + "description": "duplicate", + "entity": "ping", + "effect": "read", + "enabled": True, + "method": "GET", + "path": "/ping", + "input_schema": {"type": "object"}, + }, + ] + ), + encoding="utf-8", + ) + + +def _row(store: CredentialStore, secret: str = "super-secret-key") -> dict: + salt = store.generate_salt() + return { + "connector_id": "didar-a", + "connector_type": "didar", + "display_name": "دیدار فروش", + "status": "active", + "encrypted_credentials": store.encrypt({"api_key": secret}, salt), + "encryption_salt": salt, + "user_name": "usr_test", + } + + +def _executor(handler, row, store: CredentialStore): + transport = httpx.MockTransport(handler) + client = httpx.AsyncClient(transport=transport) + executor = NativeConnectorExecutor( + client=client, + credential_store=store, + row_loader=lambda connector_id: row if connector_id == row["connector_id"] else None, + ) + return executor, client + + +def _state(): + return { + "conv_id": "conv-native", + "user_name": "usr_test", + "native_connectors": { + "didar-a": { + "connector_id": "didar-a", + "connector_type": "didar", + "display_name": "دیدار فروش", + }, + "didar-b": { + "connector_id": "didar-b", + "connector_type": "didar", + "display_name": "دیدار شیراز", + }, + }, + } + + +@pytest.mark.asyncio +async def test_search_deals_validation_and_one_http_call( + monkeypatch: pytest.MonkeyPatch, tmp_path +): + _enable(monkeypatch) + monkeypatch.setattr( + "kamidso.engine.core.configs.model_config.PILOT_PATH", str(tmp_path) + ) + calls = {"n": 0} + secret = "super-secret-key" + + def handler(request: httpx.Request) -> httpx.Response: + calls["n"] += 1 + assert request.url.host == "app.didar.me" + assert request.url.path == "/api/deal/search_v2" + assert request.url.params["apikey"] == secret + assert secret not in (request.content.decode() or "") + return httpx.Response( + 200, + json={ + "Response": { + "TotalCount": 651, + "TotalPrice": 10, + "WonDealsCount": 651, + "LostDealsCount": 0, + "PendingDealsCount": 0, + "List": [{"Title": "Deal"}], + } + }, + ) + + store = CredentialStore() + row = _row(store, secret) + executor, client = _executor(handler, row, store) + try: + missing = await executor.execute( + connector_id="didar-a", + function_name="search_deals", + body={"From": 0, "Limit": 100}, + react_state={"conv_id": "conv-native"}, + display_name="دیدار فروش", + ) + assert missing["error_type"] == "validation_error" + assert missing["http_sent"] is False + assert calls["n"] == 0 + nested = await executor.execute( + connector_id="didar-a", + function_name="search_deals", + body={"Criteria": {"From": 0, "Limit": 50}}, + react_state={"conv_id": "conv-native"}, + display_name="دیدار فروش", + ) + assert nested["http_sent"] is False + assert any(issue["message"].startswith("Unexpected fields") for issue in nested["issues"]) + assert calls["n"] == 0 + unknown = None + try: + await executor.execute( + connector_id="didar-a", + function_name="get_leads", + body={}, + react_state={"conv_id": "conv-native"}, + ) + except Exception as exc: # noqa: BLE001 + unknown = exc + assert unknown is not None + payload = unknown.to_payload() + assert payload["error_type"] == "unknown_function" + assert payload["http_sent"] is False + assert "search_persons" in payload["suggestions"] + assert "search_deals" in payload["suggestions"] + assert calls["n"] == 0 + try: + await executor.execute( + connector_id="didar-a", + function_name="create_deal", + body={"Deal": {}}, + react_state={"conv_id": "conv-native"}, + ) + except Exception as exc: # noqa: BLE001 + blocked = exc + else: + blocked = None + assert blocked is not None + blocked_payload = blocked.to_payload() + assert blocked_payload["error_type"] == "operation_not_allowed" + assert blocked_payload["http_sent"] is False + assert calls["n"] == 0 + ok = await executor.execute( + connector_id="didar-a", + function_name="search_deals", + body={"Criteria": {}, "From": 0, "Limit": 100}, + react_state={"conv_id": "conv-native"}, + display_name="دیدار فروش", + ) + assert ok["ok"] is True + assert calls["n"] == 1 + saved = tmp_path / "tmp" / "conv-native" / ok["relative_path"].replace("/", os.sep) + raw = json.loads(saved.read_text(encoding="utf-8")) + assert raw["Response"]["TotalCount"] == 651 + assert secret not in saved.read_text(encoding="utf-8") + assert secret not in ok["observation"] + finally: + await client.aclose() + + +@pytest.mark.asyncio +async def test_duplicate_connector_call_does_not_repeat_http( + monkeypatch: pytest.MonkeyPatch, tmp_path +): + _enable(monkeypatch) + monkeypatch.setattr( + "kamidso.engine.core.configs.model_config.PILOT_PATH", str(tmp_path) + ) + calls = {"n": 0} + + def handler(request: httpx.Request) -> httpx.Response: + calls["n"] += 1 + return httpx.Response(200, json={"Response": {"List": [], "TotalCount": 0}}) + + store = CredentialStore() + row = _row(store) + executor, client = _executor(handler, row, store) + state = { + "conv_id": "conv-native", + "native_connectors": { + "didar-a": { + "connector_id": "didar-a", + "connector_type": "didar", + "display_name": "دیدار فروش", + } + }, + } + tool = make_connector_call(state, executor=executor) + body = {"Criteria": {}, "From": 0, "Limit": 1} + try: + first = await tool(name="get_users", body={}) + second = await tool(name="get_users", body={}) + third = await tool(name="get_users", body={}) + changed = await tool(name="search_deals", body=body) + finally: + await client.aclose() + assert calls["n"] == 2 + assert "Reuse previous result" in first or "Reuse previous result" in second + assert "already executed" in third + assert "super-secret-key" not in first + second + third + changed + assert "api_key" not in json.dumps(state["native_connectors"]) + + +@pytest.mark.asyncio +async def test_validation_failure_loop_stops_repeating_the_schema( + monkeypatch: pytest.MonkeyPatch, +): + _enable(monkeypatch) + calls = {"n": 0} + + def handler(request: httpx.Request) -> httpx.Response: + calls["n"] += 1 + return httpx.Response(200, json={}) + + store = CredentialStore() + executor, client = _executor(handler, _row(store), store) + state = { + "conv_id": "conv-native", + "native_connectors": { + "didar-a": { + "connector_id": "didar-a", + "connector_type": "didar", + "display_name": "دیدار فروش", + } + }, + } + tool = make_connector_call(state, executor=executor) + try: + first = await tool(name="search_deals", body={"From": 0}) + second = await tool(name="search_deals", body={"From": 0}) + third = await tool(name="search_deals", body={"From": 0}) + finally: + await client.aclose() + assert calls["n"] == 0 + assert "expected_schema" in first + assert "expected_schema" in second + assert "failed repeatedly" in third + assert "expected_schema" not in third + + +@pytest.mark.asyncio +async def test_unattached_connector_is_rejected(monkeypatch: pytest.MonkeyPatch): + _enable(monkeypatch) + state = _state() + del state["native_connectors"]["didar-b"] + tool = make_connector_call(state, executor=NativeConnectorExecutor()) + text = await tool(name="get_users", connection="دیدار شیراز") + assert "connector_not_attached" in text + assert "super-secret" not in text + + +@pytest.mark.asyncio +async def test_several_connectors_require_a_connection(monkeypatch: pytest.MonkeyPatch): + _enable(monkeypatch) + tool = make_connector_call(_state(), executor=NativeConnectorExecutor()) + text = await tool(name="get_users") + assert "connector_ambiguous" in text + assert "دیدار فروش" in text + assert "دیدار شیراز" in text + + +def test_prompt_lists_names_without_schema_or_secrets(monkeypatch: pytest.MonkeyPatch): + _enable(monkeypatch) + text = render_native_prompt( + [ + { + "connector_id": "didar-a", + "connector_type": "didar", + "display_name": "دیدار فروش", + } + ] + ) + assert "search_deals" in text + assert "get_leads" in text + assert "additionalProperties" not in text + assert "/api/" not in text + assert "api_key" not in text + assert "apikey" not in text + + +def test_partition_keeps_native_ids_away_from_mcp(monkeypatch: pytest.MonkeyPatch): + _enable(monkeypatch) + rows = { + "didar-a": { + "connector_id": "didar-a", + "connector_type": "didar", + "display_name": "دیدار فروش", + "status": "active", + "user_name": "usr_test", + "api_key": "must-not-leak", + }, + "mcp-1": { + "connector_id": "mcp-1", + "connector_type": "github", + "display_name": "GitHub", + "status": "active", + "user_name": "usr_test", + }, + } + monkeypatch.setattr( + "kamidso.native_connectors.resolve.default_row_loader", + lambda connector_id: rows.get(connector_id), + ) + mcp_ids, native = partition_connector_ids(["didar-a", "mcp-1", "missing"]) + assert mcp_ids == ["mcp-1", "missing"] + assert native == [ + { + "connector_id": "didar-a", + "connector_type": "didar", + "display_name": "دیدار فروش", + } + ] + assert "api_key" not in json.dumps(native) + + +def test_result_file_survives_a_fresh_python_process( + monkeypatch: pytest.MonkeyPatch, tmp_path +): + monkeypatch.setattr( + "kamidso.engine.core.configs.model_config.PILOT_PATH", str(tmp_path) + ) + meta = materialize_connector_result( + {"conv_id": "conv-native"}, + {"Response": {"TotalCount": 651, "List": [{"Title": "Deal"}]}}, + ) + work = tmp_path / "tmp" / "conv-native" + env = os.environ.copy() + env["PLOT_DIR"] = str(work) + script = ( + "import json, os\n" + "path = os.path.join(os.environ['PLOT_DIR'], 'connector_results', 'c_001.json')\n" + "print(json.load(open(path, encoding='utf-8'))['Response']['TotalCount'])\n" + ) + completed = subprocess.run( + [sys.executable, "-c", script], + env=env, + capture_output=True, + text=True, + check=False, + ) + assert completed.returncode == 0, completed.stderr + assert completed.stdout.strip() == "651" + assert meta["relative_path"] == "connector_results/c_001.json" + + +def test_code_interpreter_mentions_connector_results_when_attached(): + attached = make_code_interpreter({}, sql_enabled=False, native_connector=True) + plain = make_code_interpreter({}, sql_enabled=False, native_connector=False) + assert "connector_results" in attached._tool.description + assert "requests/httpx" in attached._tool.description + assert "connector_results" not in plain._tool.description + + +def test_chat_resources_map_onto_connector_ids(): + request = KamidsoChatRequest.from_body( + { + "message": "چند معامله داریم؟", + "resources": [ + {"id": "didar-a", "type": "connector", "name": "دیدار فروش"}, + {"id": "file-1", "type": "file", "name": "budget.xlsx"}, + ], + } + ) + selected = connector_selection(request.resources) + assert selected == { + "connector_ids": ["didar-a"], + "connector_labels": ["دیدار فروش"], + } diff --git a/tests/kamidso/test_sse_keepalive.py b/tests/kamidso/test_sse_keepalive.py index 0482440..ab983c0 100644 --- a/tests/kamidso/test_sse_keepalive.py +++ b/tests/kamidso/test_sse_keepalive.py @@ -5,7 +5,12 @@ from collections.abc import AsyncIterator import pytest -from kamidso.api.sse import SSE_PING, with_sse_keepalive +from kamidso.api.sse import SSE_HEADERS, SSE_PING, with_sse_keepalive + + +def test_sse_headers_do_not_ask_proxies_to_reuse_the_socket(): + assert "Connection" not in SSE_HEADERS + assert SSE_HEADERS["X-Accel-Buffering"] == "no" @pytest.mark.asyncio