kamidso-back/tests/kamidso/test_data_layout.py

import os
from pathlib import Path

from kamidso.engine.core.configs.model_config import interpreter_work_dir, knowledge_upload_root
from kamidso.engine.core.observability.span_store import resolve_sqlite_path
from kamidso.engine.core.util.tracer.tracer_impl import resolve_tracer_filename
from kamidso.settings.paths import repo_root
from kamidso.storage.layout import DataLayout, candidate_storage_keys, upload_storage_key


def test_relative_data_path_uses_backend_app_root(monkeypatch):
    monkeypatch.chdir(repo_root().parent)
    layout = DataLayout.from_base_path("./data")
    assert layout.root == (repo_root() / "data").resolve()
    assert layout.root.name == "data"
    assert layout.root.parent == repo_root()


def test_data_layout_creates_canonical_dirs(tmp_path, monkeypatch):
    monkeypatch.chdir(tmp_path)
    layout = DataLayout.from_base_path(tmp_path / "data")
    layout.ensure()
    for name in (
        "uploads",
        "runtime",
        "generated",
        "temp",
        "knowledge",
        "vectors",
        "cache",
        "engine-home",
        "logs",
        "meta",
    ):
        assert (layout.root / name).is_dir()
    assert layout.vectors == layout.root / "vectors" / "chromadb"
    assert layout.vectors.is_dir()
    assert layout.meta == layout.root / "meta"
    assert (layout.root / "runtime" / "op_snapshots").is_dir()


def test_bind_process_env_sets_engine_home(tmp_path, monkeypatch):
    import kamidso.engine.core.configs.model_config as engine_paths

    monkeypatch.chdir(tmp_path)
    saved_paths = {
        name: getattr(engine_paths, name)
        for name in (
            "PILOT_PATH",
            "LOGDIR",
            "STATIC_MESSAGE_IMG_PATH",
            "STATIC_MESSAGE_FILE_PATH",
            "DATASETS_DIR",
            "DATA_DIR",
            "MODEL_DISK_CACHE_DIR",
            "FILE_SERVER_LOCAL_STORAGE_PATH",
            "KNOWLEDGE_UPLOAD_ROOT_PATH",
            "KNOWLEDGE_CACHE_ROOT_PATH",
            "BENCHMARK_DATA_ROOT_PATH",
        )
    }
    keys = (
        "KAMIDSO_DATA_ROOT",
        "KAMIDSO_STORAGE_PATH",
        "KAMIDSO_VECTORS_PATH",
        "KAMIDSO_ENGINE_HOME",
        "KAMIDSO_ENGINE_LOG_DIR",
        "DBGPT_HOME",
        "FILE_SERVER_LOCAL_STORAGE_PATH",
        "DBGPT_LOG_DIR",
        "CHROMA_PERSIST_PATH",
        "TMPDIR",
        "TEMP",
        "TMP",
        "XDG_CACHE_HOME",
        "HF_HOME",
        "HUGGINGFACE_HUB_CACHE",
        "TRANSFORMERS_CACHE",
        "KAMIDSO_SKIP_SQLITE_FILE",
        "DBGPT_OBSERVABILITY_SQLITE_PATH",
        "KAMIDSO_OBSERVABILITY_JSONL",
        "KAMIDSO_SKILLS_DIR",
        "DBGPT_SKILLS_DIR",
    )
    for key in keys:
        monkeypatch.setenv(key, os.environ.get(key, ""))
    layout = DataLayout.from_base_path(tmp_path / "data")
    layout.ensure()
    try:
        layout.bind_process_env(skills_dir=tmp_path / "skills")
        assert Path(os.environ["KAMIDSO_DATA_ROOT"]) == layout.root
        assert Path(os.environ["KAMIDSO_ENGINE_HOME"]) == layout.engine_home
        assert Path(os.environ["DBGPT_HOME"]) == layout.engine_home
        assert Path(os.environ["FILE_SERVER_LOCAL_STORAGE_PATH"]) == layout.cache / "files"
        assert Path(os.environ["CHROMA_PERSIST_PATH"]) == layout.vectors
        assert Path(os.environ["TMPDIR"]) == layout.temp
        assert Path(os.environ["HF_HOME"]) == layout.cache / "huggingface"
        assert "KAMIDSO_SKIP_SQLITE_FILE" not in os.environ
        assert (
            Path(os.environ["DBGPT_OBSERVABILITY_SQLITE_PATH"])
            == layout.logs / "observability.db"
        )
        assert Path(engine_paths.LOGDIR) == layout.logs
        assert Path(engine_paths.PILOT_PATH) == layout.runtime
        assert Path(os.environ["KAMIDSO_VECTORS_PATH"]) == layout.vectors
        assert (
            Path(os.environ["KAMIDSO_OBSERVABILITY_JSONL"])
            == layout.logs / "observability.jsonl"
        )
    finally:
        for name, value in saved_paths.items():
            setattr(engine_paths, name, value)


def test_upload_storage_key_shape():
    assert (
        upload_storage_key("ws_1", "file_abc", "Book.xlsx")
        == "uploads/ws_1/file_abc/Book.xlsx"
    )


def test_candidate_storage_keys_include_legacy_layout():
    keys = candidate_storage_keys("ws_1/files/file_abc/Book.xlsx")
    assert keys[0] == "ws_1/files/file_abc/Book.xlsx"
    assert "uploads/ws_1/file_abc/Book.xlsx" in keys
    assert "storage/ws_1/files/file_abc/Book.xlsx" in keys


def test_interpreter_work_dir_uses_data_root(tmp_path, monkeypatch):
    monkeypatch.setenv("KAMIDSO_DATA_ROOT", str(tmp_path / "data"))
    path = interpreter_work_dir("conv_1")
    assert Path(path) == (tmp_path / "data" / "temp" / "conv_1")
    assert Path(path).is_dir()


def test_knowledge_upload_root_uses_data_tree(tmp_path, monkeypatch):
    monkeypatch.setenv("KAMIDSO_DATA_ROOT", str(tmp_path / "data"))
    path = knowledge_upload_root()
    assert Path(path) == tmp_path / "data" / "knowledge"
    assert Path(path).is_dir()


def test_observability_sqlite_is_memory_when_skip_flag(monkeypatch):
    monkeypatch.setenv("KAMIDSO_SKIP_SQLITE_FILE", "1")
    assert resolve_sqlite_path("/tmp/observability.db") == ":memory:"


def test_tracer_filename_uses_data_logs(tmp_path, monkeypatch):
    monkeypatch.delenv("KAMIDSO_OBSERVABILITY_JSONL", raising=False)
    monkeypatch.setenv("KAMIDSO_DATA_ROOT", str(tmp_path / "data"))
    path = resolve_tracer_filename("logs/dbgpt_webserver_tracer.jsonl")
    assert Path(path) == tmp_path / "data" / "logs" / "dbgpt_webserver_tracer.jsonl"
    assert Path(path).parent.is_dir()