Files
kamidso-back/kamidso/native_connectors/semantic/executor.py
T
javadandCursor edbaa2acec Add native Didar connectors so chats can query CRM data reliably.
Connector calls, semantic rules, and DuckDB memory keep long runs stable, and stream retry stops chats from stalling.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-10-12 19:26:47 +03:30

783 lines
28 KiB
Python

"""Run one semantic operation through the existing raw connector caller."""
from __future__ import annotations
import copy
import json
import os
import time
from datetime import datetime, timedelta
from typing import Any, Dict, List, Optional
from zoneinfo import ZoneInfo
import jdatetime
from jsonschema import Draft202012Validator
from kamidso.native_connectors.errors import (
AccuracyGuard,
FunctionDisabled,
ResultMaterializationError,
)
from kamidso.native_connectors.result_store import materialize_connector_result
from kamidso.native_connectors.semantic.aggregates import as_int
from kamidso.native_connectors.semantic.catalog import (
date_basis_for,
disclosure_for,
verification_cap,
)
from kamidso.native_connectors.semantic.dates import (
PeriodError,
ResolvedPeriod,
parse_utc,
resolve_period,
)
from kamidso.native_connectors.semantic.money import amount_lines, summarize_amounts
from kamidso.native_connectors.validator import _issue, example_skeleton
_TEHRAN = ZoneInfo("Asia/Tehran")
_PAGE = 100
_UNTRUSTED = {"won_at", "lost_at"}
_GUARD = (
"accuracy_guard: this period cannot be fully verified within the row-check limit. "
"Narrow the period. No exact total is reported. "
"عدد دقیق گزارش نمی‌شود؛ بازه را کوچک‌تر کنید."
)
def _validation(name: str, schema: Dict[str, Any], body: Any, connector_name: str) -> Optional[Dict[str, Any]]:
if not isinstance(body, dict):
issues = [{"path": [], "message": "Body must be a JSON object."}]
else:
validator = Draft202012Validator(schema if isinstance(schema, dict) else {"type": "object"})
issues = [
_issue(error)
for error in sorted(validator.iter_errors(body), key=lambda item: list(item.path))
]
if not issues:
return None
return {
"ok": False,
"error_type": "validation_error",
"connector": connector_name,
"function": name,
"message": "Request body does not match the semantic operation.",
"issues": issues,
"expected_schema": schema,
"example_skeleton": example_skeleton(schema if isinstance(schema, dict) else {}),
"http_sent": False,
"instruction": (
"Fix the body and retry the SAME semantic operation. "
"Pass a business body such as a period. Do not build Didar Criteria."
),
}
def _period_error(name: str, schema: Dict[str, Any], message: str, connector_name: str) -> Dict[str, Any]:
return {
"ok": False,
"error_type": "validation_error",
"connector": connector_name,
"function": name,
"message": message,
"issues": [{"path": ["period"], "message": message}],
"expected_schema": schema,
"example_skeleton": example_skeleton(schema if isinstance(schema, dict) else {}),
"http_sent": False,
"instruction": "Fix the period and retry the SAME semantic operation.",
}
def _basis(operation: Dict[str, Any]) -> str:
if operation.get("date_basis"):
return str(operation["date_basis"])
return date_basis_for(str(operation.get("name") or ""))
def _query(operation: Dict[str, Any], period: Optional[ResolvedPeriod], body: Dict[str, Any], basis: str) -> Dict[str, Any]:
template = operation.get("query_template")
query: Dict[str, Any]
if isinstance(template, dict):
query = copy.deepcopy(template)
else:
query = {"Criteria": {}, "From": 0, "Limit": _PAGE}
criteria = query.get("Criteria")
if not isinstance(criteria, dict):
criteria = {}
query["Criteria"] = criteria
if operation.get("status"):
criteria["Status"] = operation["status"]
elif basis == "won_at":
criteria["Status"] = "Won"
elif basis == "lost_at":
criteria["Status"] = "Lost"
elif basis == "current_snapshot":
criteria["Status"] = "Pending"
if operation.get("name") == "deals_created" and body.get("status"):
criteria["Status"] = body["status"]
if "query_sort" in operation:
criteria["Sort"] = operation["query_sort"]
elif basis == "won_at":
criteria.setdefault("Sort", 9)
elif basis == "created_at":
criteria.setdefault("Sort", 0)
criteria["IsDeleted"] = False
if period is not None and basis not in {"current_snapshot", "lost_at"}:
criteria["SearchFromTime"] = period.start_utc
criteria["SearchToTime"] = period.end_utc_exclusive
else:
criteria.pop("SearchFromTime", None)
criteria.pop("SearchToTime", None)
if basis == "lost_at":
criteria.pop("Sort", None)
if body.get("owner_ids"):
criteria["OwnerIds"] = list(body["owner_ids"])
if body.get("pipeline_id"):
criteria["PipelineId"] = body["pipeline_id"]
if body.get("pipeline_stage_ids"):
criteria["PipelineStageIds"] = list(body["pipeline_stage_ids"])
query["From"] = 0
query["Limit"] = _PAGE
return query
def _load_saved(react_state: Dict[str, Any], relative_path: str) -> Any:
from kamidso.engine.app.openapi.api_v1.tools.sql_query import session_work_dir
folder = session_work_dir(react_state.get("conv_id"))
path = os.path.join(folder, *str(relative_path).split("/"))
try:
with open(path, encoding="utf-8") as handle:
return json.load(handle)
except (OSError, json.JSONDecodeError) as exc:
raise ResultMaterializationError(
"The connector response could not be read back for verification.",
) from exc
def _deal_rows(payload: Any) -> List[Dict[str, Any]]:
response = payload.get("Response") if isinstance(payload, dict) else None
if isinstance(response, dict) and isinstance(response.get("List"), list):
return [item for item in response["List"] if isinstance(item, dict)]
return []
def _metric(payload: Any, key: str) -> Optional[int]:
response = payload.get("Response") if isinstance(payload, dict) else None
if isinstance(response, dict):
return as_int(response.get(key))
return None
def _deleted(record: Dict[str, Any]) -> bool:
return record.get("IsDeleted") in (True, 1, "1", "true", "True")
def _matches(
record: Dict[str, Any],
basis: str,
period: Optional[ResolvedPeriod],
status_filter: Optional[str],
) -> bool:
if _deleted(record):
return False
if basis == "current_snapshot":
return record.get("Status") == "Pending"
if basis == "created_at":
if status_filter and record.get("Status") != status_filter:
return False
if period is None:
return False
instant = parse_utc(record.get("RegisterTime"))
return instant is not None and period.contains(instant)
expected = "Won" if basis == "won_at" else "Lost"
if record.get("Status") != expected:
return False
if period is None:
return False
field = "ChangeToWonTime" if basis == "won_at" else "ChangeToLossTime"
instant = parse_utc(record.get(field))
return instant is not None and period.contains(instant)
def _calibration(row: Optional[Dict[str, Any]]) -> Dict[str, Any]:
if not isinstance(row, dict):
return {}
config = row.get("config")
if isinstance(config, str):
try:
config = json.loads(config)
except json.JSONDecodeError:
return {}
if not isinstance(config, dict):
return {}
stored = config.get("semantic_calibration")
return stored if isinstance(stored, dict) else {}
def _bucket(moment: datetime, grain: str) -> str:
local = moment.astimezone(_TEHRAN)
if grain == "week":
start = (local - timedelta(days=(local.weekday() - 5) % 7)).date()
stamp = jdatetime.date.fromgregorian(date=start)
return f"{stamp.year:04d}-{stamp.month:02d}-{stamp.day:02d}"
stamp_dt = jdatetime.datetime.fromgregorian(datetime=local.replace(tzinfo=None))
if grain == "day":
return f"{stamp_dt.year:04d}-{stamp_dt.month:02d}-{stamp_dt.day:02d}"
return f"{stamp_dt.year:04d}-{stamp_dt.month:02d}"
class SemanticExecutor:
def __init__(self, runner: Any) -> None:
self._runner = runner
async def execute(
self,
*,
operation: Dict[str, Any],
connector_id: str,
body: Optional[Dict[str, Any]],
react_state: Dict[str, Any],
display_name: str,
) -> Dict[str, Any]:
name = str(operation.get("name") or "")
if not operation.get("enabled"):
reason = str(operation.get("reason_disabled") or "This semantic operation is disabled.")
raise FunctionDisabled(f"Semantic operation '{name}' is disabled. {reason}", function=name)
schema = operation.get("input_schema")
if not isinstance(schema, dict):
schema = {"type": "object", "additionalProperties": False}
payload_body = body or {}
invalid = _validation(name, schema, payload_body, display_name)
if invalid:
return invalid
basis = _basis(operation)
period: Optional[ResolvedPeriod] = None
needs_period = basis != "current_snapshot"
if needs_period:
try:
period = resolve_period(str(payload_body.get("period") or ""))
except PeriodError as exc:
return _period_error(name, schema, str(exc), display_name)
query = _query(operation, period, payload_body, basis)
status_filter = payload_body.get("status") if name == "deals_created" else None
if not isinstance(status_filter, str):
status_filter = None
scanned = await self._scan(
query=query,
connector_id=connector_id,
react_state=react_state,
display_name=display_name,
function_name=str(operation.get("raw_function") or "search_deals"),
everything=basis in _UNTRUSTED,
)
if "rows" not in scanned:
return scanned
if basis not in _UNTRUSTED:
sample = [
row
for row in scanned["rows"]
if _matches(row, basis, period, status_filter)
]
if len(sample) != len(scanned["rows"]):
total = scanned.get("total")
if isinstance(total, int) and total > verification_cap():
raise AccuracyGuard(_GUARD, function=name)
scanned = await self._scan(
query=query,
connector_id=connector_id,
react_state=react_state,
display_name=display_name,
function_name=str(operation.get("raw_function") or "search_deals"),
everything=True,
)
if "rows" not in scanned:
return scanned
basis_trusted = False
else:
basis_trusted = True
else:
basis_trusted = False
matched = [
row
for row in scanned["rows"]
if _matches(row, basis, period, status_filter)
]
count, count_source, money_rows, money_withheld = _totals(
name=name,
basis=basis,
scanned=scanned,
matched=matched,
trusted_sample=basis_trusted,
)
server_count = _server_count(basis, scanned)
calibration = _calibration(self._runner.load_row(connector_id))
names: Dict[str, str] = {}
pipeline_title = ""
stage_titles: Dict[str, str] = {}
if name == "sales_by_owner":
names = await self._users(connector_id, react_state, display_name)
if payload_body.get("pipeline_id") or payload_body.get("pipeline_stage_ids"):
pipeline_title, stage_titles = await self._pipelines(
connector_id,
react_state,
display_name,
str(payload_body.get("pipeline_id") or ""),
list(payload_body.get("pipeline_stage_ids") or []),
)
return self._finish(
name=name,
basis=basis,
period=period,
count=count,
count_source=count_source,
money_rows=money_rows,
money_withheld=money_withheld,
server_count=server_count,
calibration=calibration,
matched=matched,
names=names,
pipeline_title=pipeline_title,
stage_titles=stage_titles,
grain=str(payload_body.get("grain") or "month"),
react_state=react_state,
display_name=display_name,
)
async def _scan(
self,
*,
query: Dict[str, Any],
connector_id: str,
react_state: Dict[str, Any],
display_name: str,
function_name: str,
everything: bool,
) -> Dict[str, Any]:
cap = verification_cap()
rows: List[Dict[str, Any]] = []
snapshot: Dict[str, Optional[int]] = {
"total": None,
"won": None,
"lost": None,
"pending": None,
}
frm = 0
while True:
page = copy.deepcopy(query)
page["From"] = frm
page["Limit"] = _PAGE
result = await self._runner.execute(
connector_id=connector_id,
function_name=function_name,
body=page,
react_state=react_state,
display_name=display_name,
)
if not result.get("ok"):
return result
payload = _load_saved(react_state, str(result.get("relative_path") or ""))
page_rows = _deal_rows(payload)
snapshot = {
"total": _metric(payload, "TotalCount"),
"won": _metric(payload, "WonDealsCount"),
"lost": _metric(payload, "LostDealsCount"),
"pending": _metric(payload, "PendingDealsCount"),
}
if everything and isinstance(snapshot["total"], int) and snapshot["total"] > cap:
raise AccuracyGuard(_GUARD, function=function_name)
rows.extend(page_rows)
if isinstance(snapshot["total"], int):
complete = not page_rows or frm + len(page_rows) >= snapshot["total"]
else:
complete = not page_rows or len(page_rows) < _PAGE
packed = {"rows": rows, "complete": complete, **snapshot}
if not everything or complete:
if everything and not rows and isinstance(snapshot["total"], int) and snapshot["total"] > 0:
raise AccuracyGuard(_GUARD, function=function_name)
if (
everything
and not rows
and snapshot["total"] is None
and snapshot["won"] is None
and snapshot["lost"] is None
):
raise AccuracyGuard(_GUARD, function=function_name)
return packed
if len(rows) >= cap:
raise AccuracyGuard(_GUARD, function=function_name)
frm += _PAGE
async def _users(
self,
connector_id: str,
react_state: Dict[str, Any],
display_name: str,
) -> Dict[str, str]:
cache = react_state.setdefault("connector_metadata_cache", {})
key = connector_id + ":users"
now = time.monotonic()
if isinstance(cache, dict):
cached = cache.get(key)
if isinstance(cached, dict) and now - float(cached.get("at") or 0) < 300:
names = cached.get("names")
if isinstance(names, dict):
return {str(item): str(value) for item, value in names.items()}
result = await self._runner.execute(
connector_id=connector_id,
function_name="get_users",
body={},
react_state=react_state,
display_name=display_name,
)
names: Dict[str, str] = {}
if result.get("ok"):
payload = _load_saved(react_state, str(result.get("relative_path") or ""))
records = payload.get("Response") if isinstance(payload, dict) else None
if isinstance(records, dict):
records = records.get("List")
if isinstance(records, list):
for record in records:
if not isinstance(record, dict):
continue
user_id = record.get("UserId")
label = record.get("DisplayName")
if user_id and isinstance(label, str) and label.strip():
names[str(user_id)] = label.strip()
if isinstance(cache, dict):
cache[key] = {"at": now, "names": names}
return names
async def _pipelines(
self,
connector_id: str,
react_state: Dict[str, Any],
display_name: str,
pipeline_id: str,
stage_ids: List[Any],
) -> tuple:
result = await self._runner.execute(
connector_id=connector_id,
function_name="get_deal_pipelines",
body={},
react_state=react_state,
display_name=display_name,
)
titles: Dict[str, str] = {}
stages: Dict[str, str] = {}
if result.get("ok"):
payload = _load_saved(react_state, str(result.get("relative_path") or ""))
records = payload.get("Response") if isinstance(payload, dict) else None
if isinstance(records, dict):
records = records.get("List")
if isinstance(records, list):
for item in records:
if not isinstance(item, dict):
continue
if item.get("Id") and item.get("Title"):
titles[str(item["Id"])] = str(item["Title"])
for stage in item.get("Stages") or []:
if isinstance(stage, dict) and stage.get("Id") and stage.get("Title"):
stages[str(stage["Id"])] = str(stage["Title"])
pipeline_title = titles.get(pipeline_id, "")
chosen = {str(item): stages.get(str(item), "") for item in stage_ids}
return pipeline_title, chosen
def _finish(
self,
*,
name: str,
basis: str,
period: Optional[ResolvedPeriod],
count: int,
count_source: str,
money_rows: List[Dict[str, Any]],
money_withheld: bool,
server_count: Optional[int],
calibration: Dict[str, Any],
matched: List[Dict[str, Any]],
names: Dict[str, str],
pipeline_title: str,
stage_titles: Dict[str, str],
grain: str,
react_state: Dict[str, Any],
display_name: str,
) -> Dict[str, Any]:
money = summarize_amounts(money_rows, calibration)
panel_parity = bool(calibration.get("panel_parity_verified"))
disagreement = (
basis in _UNTRUSTED
and server_count is not None
and server_count != count
)
mode = "safe"
if (
panel_parity
and not disagreement
and not money_withheld
and not money.mixed
and (money.currency_verified or not money.groups)
):
mode = "verified"
warnings = _warnings(
panel_parity=panel_parity,
money=money,
money_withheld=money_withheld,
disagreement=disagreement,
currency_verified=money.currency_verified,
)
groups = _owner_groups(matched, names) if name == "sales_by_owner" else []
timeline = _timeline(matched, grain) if name == "sales_over_time" else []
summary = {
"semantic_operation": name,
"accuracy": {
"mode": mode,
"date_basis": basis,
"period": None
if period is None
else {
"input": period.input_text,
"start_utc": period.start_utc,
"end_utc_exclusive": period.end_utc_exclusive,
},
"currency_verified": money.currency_verified,
"panel_parity_verified": panel_parity,
"warnings": warnings,
},
"count": count,
"count_source": count_source,
"amounts": money.groups,
"groups": groups,
"timeline": timeline,
"records": matched,
}
meta = materialize_connector_result(react_state, summary)
observation = _render(
display_name=display_name,
name=name,
mode=mode,
basis=basis,
period=period,
count=count,
count_source=count_source,
money=money,
money_withheld=money_withheld,
warnings=warnings,
groups=groups,
timeline=timeline,
pipeline_title=pipeline_title,
stage_titles=stage_titles,
relative_path=meta["relative_path"],
currency_verified=money.currency_verified,
panel_parity=panel_parity,
)
return {
"ok": True,
"http_sent": True,
"relative_path": meta["relative_path"],
"observation": observation,
}
def _totals(
*,
name: str,
basis: str,
scanned: Dict[str, Any],
matched: List[Dict[str, Any]],
trusted_sample: bool,
) -> tuple:
if basis in _UNTRUSTED or not trusted_sample:
return len(matched), "verified_rows", matched, False
if basis == "current_snapshot":
pending = scanned.get("pending")
if isinstance(pending, int):
return pending, "Response.PendingDealsCount", [], True
if scanned.get("complete"):
return len(matched), "verified_rows", [], True
raise AccuracyGuard(_GUARD, function=name)
total = scanned.get("total")
if isinstance(total, int):
complete = bool(scanned.get("complete"))
return total, "Response.TotalCount", matched if complete else [], not complete
if scanned.get("complete"):
return len(matched), "verified_rows", matched, False
raise AccuracyGuard(_GUARD, function=name)
def _server_count(basis: str, scanned: Dict[str, Any]) -> Optional[int]:
if basis == "won_at":
value = scanned.get("won")
elif basis == "lost_at":
value = scanned.get("lost")
elif basis == "current_snapshot":
value = scanned.get("pending")
else:
value = scanned.get("total")
return value if isinstance(value, int) else None
def _warnings(
*,
panel_parity: bool,
money: Any,
money_withheld: bool,
disagreement: bool,
currency_verified: bool,
) -> List[str]:
warnings: List[str] = []
if not panel_parity:
warnings.append("Exact parity with the Didar dashboard is not verified for this connection.")
if money_withheld:
warnings.append(
"Amount totals are withheld because the checked page is not the full period."
)
elif money.mixed:
warnings.append("Amounts from different CurrencyId values are not summed.")
elif money.groups and not currency_verified:
warnings.append("Currency name is not known; amounts are labeled by CurrencyId.")
if disagreement:
warnings.append(
"The server aggregate count was not used because returned rows did not match the period basis."
)
if money.groups and not money_withheld and not (panel_parity and currency_verified):
warnings.append(
"FinalAdjustmentPrice is a provisional record amount until golden reconciliation."
)
return warnings
def _owner_groups(records: List[Dict[str, Any]], names: Dict[str, str]) -> List[Dict[str, Any]]:
order: List[str] = []
buckets: Dict[str, List[Dict[str, Any]]] = {}
for record in records:
owner_id = str(record.get("OwnerId") or "")
if owner_id not in buckets:
order.append(owner_id)
buckets[owner_id] = []
buckets[owner_id].append(record)
groups = []
unresolved = 0
for owner_id in order:
label = names.get(owner_id, "")
if not label or label == owner_id:
unresolved += 1
label = f"کارشناس شناسایی‌نشده {unresolved}"
summary = summarize_amounts(buckets[owner_id], {})
groups.append(
{
"owner_id": owner_id,
"owner_name": label,
"count": len(buckets[owner_id]),
"amounts": summary.groups,
}
)
return groups
def _timeline(records: List[Dict[str, Any]], grain: str) -> List[Dict[str, Any]]:
order: List[str] = []
buckets: Dict[str, List[Dict[str, Any]]] = {}
for record in records:
instant = parse_utc(record.get("ChangeToWonTime"))
if instant is None:
continue
key = _bucket(instant, grain if grain in {"day", "week", "month"} else "month")
if key not in buckets:
order.append(key)
buckets[key] = []
buckets[key].append(record)
timeline = []
for key in order:
summary = summarize_amounts(buckets[key], {})
timeline.append(
{
"bucket": key,
"count": len(buckets[key]),
"amounts": summary.groups,
}
)
return timeline
def _render(
*,
display_name: str,
name: str,
mode: str,
basis: str,
period: Optional[ResolvedPeriod],
count: int,
count_source: str,
money: Any,
money_withheld: bool,
warnings: List[str],
groups: List[Dict[str, Any]],
timeline: List[Dict[str, Any]],
pipeline_title: str,
stage_titles: Dict[str, str],
relative_path: str,
currency_verified: bool,
panel_parity: bool,
) -> str:
lines = [
f"{display_name} / {name} completed.",
"",
"Accuracy:",
f"- mode: {mode}",
f"- date_basis: {basis}",
f"- currency_verified: {str(currency_verified).lower()}",
f"- panel_parity_verified: {str(panel_parity).lower()}",
]
disclosure = disclosure_for(name)
if disclosure:
lines.append(f"- {disclosure}")
if period is not None:
lines.append(f"- period: {period.input_text}")
lines.append(f"- period_utc: [{period.start_utc}, {period.end_utc_exclusive})")
if pipeline_title:
lines.append(f"- pipeline: {pipeline_title}")
elif stage_titles:
lines.append("- pipeline: کاریز شناسایی‌نشده")
for title in stage_titles.values():
lines.append(f"- stage: {title or 'مرحله شناسایی‌نشده'}")
lines.extend(["", f"count: {count}", f"count_source: {count_source}"])
if money_withheld:
lines.append("Amounts: withheld until the full period can be checked.")
elif name not in {"sales_by_owner", "sales_over_time"}:
lines.extend(amount_lines(money))
if groups:
lines.extend(["", "By owner:"])
for group in groups:
parts = [f"{group['owner_name']}: count {group['count']}"]
for amount in group["amounts"]:
parts.append(f"{amount['label']} { _plain_amount(amount['amount']) }")
lines.append("- " + "; ".join(parts))
if timeline:
lines.extend(["", "Over time:"])
for item in timeline:
parts = [f"{item['bucket']}: count {item['count']}"]
for amount in item["amounts"]:
parts.append(f"{amount['label']} { _plain_amount(amount['amount']) }")
lines.append("- " + "; ".join(parts))
if warnings:
lines.extend(["", "warnings:"])
for warning in warnings:
lines.append(f"- {warning}")
lines.extend(
[
"",
"Keep these accuracy warnings in the final answer.",
f"[Saved semantic result → PLOT_DIR/{relative_path}]",
"",
"Use code_interpreter only for further computation, joining, charts or export. "
"The count above is already verified for this operation.",
]
)
return "\n".join(lines)
def _plain_amount(value: float) -> str:
if float(value).is_integer():
return str(int(value))
return str(value)