Evaluation: Use Config Management

backend/app/alembic/versions/041_add_config_in_evals_run_table.py

@@ -0,0 +1,60 @@
+"""add config in evals run table
+
+Revision ID: 041
+Revises: 040
+Create Date: 2025-12-15 14:03:22.082746
+
+"""
+from alembic import op
+import sqlalchemy as sa
+import sqlmodel.sql.sqltypes
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "041"
+down_revision = "040"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column(
+        "evaluation_run",
+        sa.Column(
+            "config_id",
+            sa.Uuid(),
+            nullable=True,
+            comment="Reference to the stored config used",
+        ),
+    )
+    op.add_column(
+        "evaluation_run",
+        sa.Column(
+            "config_version",
+            sa.Integer(),
+            nullable=True,
+            comment="Version of the config used",
+        ),
+    )
+    op.create_foreign_key(None, "evaluation_run", "config", ["config_id"], ["id"])
+    op.drop_column("evaluation_run", "config")
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column(
+        "evaluation_run",
+        sa.Column(
+            "config",
+            postgresql.JSONB(astext_type=sa.Text()),
+            autoincrement=False,
+            nullable=False,
+            comment="Evaluation configuration (model, instructions, etc.)",
+        ),
+    )
+    op.drop_constraint(None, "evaluation_run", type_="foreignkey")
+    op.drop_column("evaluation_run", "config_version")
+    op.drop_column("evaluation_run", "config_id")
+    # ### end Alembic commands ###

backend/app/api/routes/evaluations/dataset.py

@@ -48,7 +48,7 @@ def _dataset_to_response(dataset: EvaluationDataset) -> DatasetUploadResponse:
 
 
 @router.post(
-    "/",
+    "",
     description=load_description("evaluation/upload_dataset.md"),
     response_model=APIResponse[DatasetUploadResponse],
     dependencies=[Depends(require_permission(Permission.REQUIRE_PROJECT))],
@@ -87,7 +87,7 @@ async def upload_dataset(
 
 
 @router.get(
-    "/",
+    "",
     description=load_description("evaluation/list_datasets.md"),
     response_model=APIResponse[list[DatasetUploadResponse]],
     dependencies=[Depends(require_permission(Permission.REQUIRE_PROJECT))],

backend/app/api/routes/evaluations/evaluation.py

@@ -1,6 +1,7 @@
 """Evaluation run API routes."""
 
 import logging
+from uuid import UUID
 
 from fastapi import (
     APIRouter,
@@ -29,7 +30,7 @@
 
 
 @router.post(
-    "/",
+    "",
     description=load_description("evaluation/create_evaluation.md"),
     response_model=APIResponse[EvaluationRunPublic],
     dependencies=[Depends(require_permission(Permission.REQUIRE_PROJECT))],
@@ -41,19 +42,16 @@ def evaluate(
     experiment_name: str = Body(
         ..., description="Name for this evaluation experiment/run"
     ),
-    config: dict = Body(default_factory=dict, description="Evaluation configuration"),
-    assistant_id: str
-    | None = Body(
-        None, description="Optional assistant ID to fetch configuration from"
-    ),
+    config_id: UUID = Body(..., description="Stored config ID"),
+    config_version: int = Body(..., ge=1, description="Stored config version"),
 ) -> APIResponse[EvaluationRunPublic]:
     """Start an evaluation run."""
     eval_run = start_evaluation(
         session=_session,
         dataset_id=dataset_id,
         experiment_name=experiment_name,
-        config=config,
-        assistant_id=assistant_id,
+        config_id=config_id,
+        config_version=config_version,
         organization_id=auth_context.organization_.id,
         project_id=auth_context.project_.id,
     )
@@ -68,7 +66,7 @@ def evaluate(
 
 
 @router.get(
-    "/",
+    "",
     description=load_description("evaluation/list_evaluations.md"),
     response_model=APIResponse[list[EvaluationRunPublic]],
     dependencies=[Depends(require_permission(Permission.REQUIRE_PROJECT))],

backend/app/crud/evaluations/__init__.py

@@ -5,6 +5,7 @@
     create_evaluation_run,
     get_evaluation_run_by_id,
     list_evaluation_runs,
+    resolve_model_from_config,
     save_score,
 )
 from app.crud.evaluations.cron import (
@@ -43,3 +44,44 @@
     TraceData,
     TraceScore,
 )
+
+__all__ = [
+    # Core
+    "create_evaluation_run",
+    "get_evaluation_run_by_id",
+    "list_evaluation_runs",
+    "resolve_model_from_config",
+    "save_score",
+    # Cron
+    "process_all_pending_evaluations",
+    "process_all_pending_evaluations_sync",
+    # Dataset
+    "create_evaluation_dataset",
+    "delete_dataset",
+    "get_dataset_by_id",
+    "list_datasets",
+    "upload_csv_to_object_store",
+    # Batch
+    "start_evaluation_batch",
+    # Processing
+    "check_and_process_evaluation",
+    "poll_all_pending_evaluations",
+    "process_completed_embedding_batch",
+    "process_completed_evaluation",
+    # Embeddings
+    "calculate_average_similarity",
+    "calculate_cosine_similarity",
+    "start_embedding_batch",
+    # Langfuse
+    "create_langfuse_dataset_run",
+    "fetch_trace_scores_from_langfuse",
+    "update_traces_with_cosine_scores",
+    "upload_dataset_to_langfuse",
+    # Score types
+    "CategoricalSummaryScore",
+    "EvaluationScore",
+    "NumericSummaryScore",
+    "SummaryScore",
+    "TraceData",
+    "TraceScore",
+]

backend/app/crud/evaluations/batch.py

@@ -16,6 +16,7 @@
 
 from app.core.batch import OpenAIBatchProvider, start_batch_job
 from app.models import EvaluationRun
+from app.models.llm.request import KaapiLLMParams
 
 logger = logging.getLogger(__name__)
 
@@ -59,7 +60,7 @@ def fetch_dataset_items(langfuse: Langfuse, dataset_name: str) -> list[dict[str,
 
 
 def build_evaluation_jsonl(
-    dataset_items: list[dict[str, Any]], config: dict[str, Any]
+    dataset_items: list[dict[str, Any]], config: KaapiLLMParams
 ) -> list[dict[str, Any]]:
     """
     Build JSONL data for evaluation batch using OpenAI Responses API.
@@ -88,7 +89,6 @@ def build_evaluation_jsonl(
         List of dictionaries (JSONL data)
     """
     jsonl_data = []
-
     for item in dataset_items:
         # Extract question from input
         question = item["input"].get("question", "")
@@ -105,7 +105,18 @@ def build_evaluation_jsonl(
             "method": "POST",
             "url": "/v1/responses",
             "body": {
-                **config,  # Use config as-is
+                # Use config as-is
+                "model": config.model,
+                "instructions": config.instructions,
+                "temperature": config.temperature,
+                "reasoning": {"effort": config.reasoning} if config.reasoning else None,
+                "tools": [
+                    {
+                        "type": "file_search",
+                        "vector_store_ids": config.knowledge_base_ids,
+                        "max_num_results": config.max_num_results or 20,
+                    }
+                ],
                 "input": question,  # Add input from dataset
             },
         }
@@ -119,7 +130,7 @@ def start_evaluation_batch(
     openai_client: OpenAI,
     session: Session,
     eval_run: EvaluationRun,
-    config: dict[str, Any],
+    config: KaapiLLMParams,
 ) -> EvaluationRun:
     """
     Fetch data, build JSONL, and start evaluation batch.
@@ -166,7 +177,7 @@ def start_evaluation_batch(
             "description": f"Evaluation: {eval_run.run_name}",
             "completion_window": "24h",
             # Store complete config for reference
-            "evaluation_config": config,
+            "evaluation_config": config.model_dump(exclude_none=True),
         }
 
         # Step 5: Start batch job using generic infrastructure

backend/app/crud/evaluations/core.py

@@ -1,12 +1,16 @@
 import logging
+from uuid import UUID
 
 from langfuse import Langfuse
 from sqlmodel import Session, select
 
 from app.core.util import now
+from app.crud.config.version import ConfigVersionCrud
 from app.crud.evaluations.langfuse import fetch_trace_scores_from_langfuse
 from app.crud.evaluations.score import EvaluationScore
 from app.models import EvaluationRun
+from app.models.llm.request import LLMCallConfig
+from app.services.llm.jobs import resolve_config_blob
 
 logger = logging.getLogger(__name__)
 
@@ -16,7 +20,8 @@ def create_evaluation_run(
     run_name: str,
     dataset_name: str,
     dataset_id: int,
-    config: dict,
+    config_id: UUID,
+    config_version: int,
     organization_id: int,
     project_id: int,
 ) -> EvaluationRun:
@@ -28,7 +33,8 @@ def create_evaluation_run(
         run_name: Name of the evaluation run/experiment
         dataset_name: Name of the dataset being used
         dataset_id: ID of the dataset
-        config: Configuration dict for the evaluation
+        config_id: UUID of the stored config
+        config_version: Version number of the config
         organization_id: Organization ID
         project_id: Project ID
 
@@ -39,7 +45,8 @@ def create_evaluation_run(
         run_name=run_name,
         dataset_name=dataset_name,
         dataset_id=dataset_id,
-        config=config,
+        config_id=config_id,
+        config_version=config_version,
         status="pending",
         organization_id=organization_id,
         project_id=project_id,
@@ -56,7 +63,10 @@ def create_evaluation_run(
         logger.error(f"Failed to create EvaluationRun: {e}", exc_info=True)
         raise
 
-    logger.info(f"Created EvaluationRun record: id={eval_run.id}, run_name={run_name}")
+    logger.info(
+        f"Created EvaluationRun record: id={eval_run.id}, run_name={run_name}, "
+        f"config_id={config_id}, config_version={config_version}"
+    )
 
     return eval_run
 
@@ -311,3 +321,47 @@ def save_score(
                 f"traces={len(score.get('traces', []))}"
             )
         return eval_run
+
+
+def resolve_model_from_config(
+    session: Session,
+    eval_run: EvaluationRun,
+) -> str:
+    """
+    Resolve the model name from the evaluation run's config.
+
+    Args:
+        session: Database session
+        eval_run: EvaluationRun instance
+
+    Returns:
+        Model name from config
+
+    Raises:
+        ValueError: If config is missing, invalid, or has no model
+    """
+    if not eval_run.config_id or not eval_run.config_version:
+        raise ValueError(
+            f"Evaluation run {eval_run.id} has no config reference "
+            f"(config_id={eval_run.config_id}, config_version={eval_run.config_version})"
+        )
+
+    config_version_crud = ConfigVersionCrud(
+        session=session,
+        config_id=eval_run.config_id,
+        project_id=eval_run.project_id,
+    )
+
+    config, error = resolve_config_blob(
+        config_crud=config_version_crud,
+        config=LLMCallConfig(id=eval_run.config_id, version=eval_run.config_version),
+    )
+
+    if error or config is None:
+        raise ValueError(
+            f"Config resolution failed for evaluation {eval_run.id} "
+            f"(config_id={eval_run.config_id}, version={eval_run.config_version}): {error}"
+        )
+
+    model = config.completion.params.model
+    return model

backend/app/crud/evaluations/embeddings.py

@@ -363,19 +363,7 @@ def start_embedding_batch(
         logger.info(f"Starting embedding batch for evaluation run {eval_run.id}")
 
         # Get embedding model from config (default: text-embedding-3-large)
-        embedding_model = eval_run.config.get(
-            "embedding_model", "text-embedding-3-large"
-        )
-
-        # Validate and fallback to default if invalid
-        try:
-            validate_embedding_model(embedding_model)
-        except ValueError as e:
-            logger.warning(
-                f"Invalid embedding model '{embedding_model}' in config: {e}. "
-                f"Falling back to text-embedding-3-large"
-            )
-            embedding_model = "text-embedding-3-large"
+        embedding_model = "text-embedding-3-large"
 
         # Step 1: Build embedding JSONL with trace_ids
         jsonl_data = build_embedding_jsonl(