Merge branch 'main' into chore/type-safety-llm-calls

Author: Ayush8923

.claude/agents/senior-engineer.md

@@ -79,6 +79,19 @@ to what the task needs.
 - Naming: `list_*` plural fetch, `get_*` singleton; `Enum` suffix on enum classes.
 - Timestamps are `inserted_at` / `updated_at`, never `created_at`.
 
+## Before finishing: type-check
+
+After editing any file — **always when you touched `app/models/`** — type-check the files you changed:
+
+```bash
+cd backend && bash scripts/pyright.sh <changed files>   # e.g. app/models/foo.py
+```
+
+Runs pyright via `uvx` (no `uv.lock` churn; config in `[tool.pyright]`). Fix real type errors you
+introduced before emitting the summary; a model change ripples into crud/service typing, so re-check
+those files too if you edited them. Distinguish real bugs from pre-existing stub noise — don't fix
+what you didn't touch. Same as the `/typecheck` command.
+
 ## After building
 
 Emit ONE summary (not one per layer):

.claude/commands/typecheck.md

@@ -0,0 +1,24 @@
+---
+description: Run pyright type checking on Python files and triage the errors.
+---
+
+Run pyright over the backend and report type errors in this FastAPI + SQLModel service.
+
+## Run
+
+From the repo root:
+
+```bash
+cd backend && bash scripts/pyright.sh $ARGUMENTS
+```
+
+- No `$ARGUMENTS` → checks all of `app/` (config in `backend/pyproject.toml` `[tool.pyright]`).
+- `$ARGUMENTS` → narrow to those paths/files, e.g. `/typecheck app/services/response`.
+- Pyright runs via `uvx` (ephemeral) so it never mutates `uv.lock`.
+
+## Report
+
+1. Group errors by file, most-affected first.
+2. For each: `file:line` — the error, then the concrete fix (narrow the type, add an annotation, guard the `None`, etc.).
+3. Separate real type bugs from noise (missing third-party stubs, `# type: ignore` candidates). Flag which is which.
+4. Do NOT auto-edit code — surface findings and let the user pick what to fix. If asked to fix, follow the layer conventions in `.claude/conventions/`.

backend/app/api/routes/documents.py

@@ -33,7 +33,7 @@
 from app.services.documents.helpers import (
     calculate_file_size,
     schedule_transformation,
-    pre_transform_validation,
+    validate_upload,
     build_document_schema,
     build_document_schemas,
 )
@@ -126,8 +126,8 @@ async def upload_doc(
     if callback_url:
         validate_callback_url(callback_url)
 
-    source_format, actual_transformer = pre_transform_validation(
-        src_filename=src.filename,
+    source_format, actual_transformer = validate_upload(
+        src=src,
         target_format=target_format,
         transformer=transformer,
     )

backend/app/crud/rag/open_ai.py

@@ -1,17 +1,47 @@
 import json
 import logging
+import time
 import functools as ft
 
 import openai
 from openai import OpenAI, OpenAIError
+from openai.types import VectorStore
+from openai.types.vector_stores import VectorStoreFileBatch
 from pydantic import BaseModel
+from tenacity import (
+    RetryCallState,
+    retry,
+    retry_if_exception_type,
+    stop_after_attempt,
+    wait_exponential,
+)
 
 from app.models import Document, ProviderType
 
 logger = logging.getLogger(__name__)
 
 OPENAI_PROVIDER = ProviderType.openai.value
 
+# Under the Celery soft time limit so a hung call can't eat the whole task window.
+# SDK-level retries are off (registry.py: max_retries=0); tenacity is the sole
+# retry layer, wrapping batch create+index.
+OPENAI_TIMEOUT_SECONDS = 30
+
+BATCH_POLL_INTERVAL_SECONDS = 2
+
+# Retry batch create+index on any OpenAI/indexing failure, exponential backoff
+# (~2s, 4s, 8s), all inside one Celery soft-time-limit window.
+BATCH_INDEX_MAX_ATTEMPTS = 4
+BATCH_RETRY_BACKOFF_BASE_SECONDS = 2
+
+
+def _log_batch_retry(retry_state: RetryCallState) -> None:
+    logger.warning(
+        f"[OpenAIVectorStoreCrud._create_and_index_batch] Batch attempt failed, retrying | "
+        f"attempt={retry_state.attempt_number}, "
+        f"error={retry_state.outcome.exception() if retry_state.outcome else None}"
+    )
+
 
 def vs_ls(client: OpenAI, vector_store_id: str):
     kwargs = {}
@@ -85,7 +115,7 @@ def __init__(self, client):
 
 
 class OpenAIVectorStoreCrud(OpenAICrud):
-    def create(self):
+    def create(self) -> VectorStore:
         logger.info(
             f"[OpenAIVectorStoreCrud.create] Creating vector store | {{'action': 'create'}}"
         )
@@ -101,6 +131,100 @@ def read(self, vector_store_id: str):
         )
         yield from vs_ls(self.client, vector_store_id)
 
+    def _create_file_batch(self, vector_store_id: str, file_ids: list[str]) -> str:
+        """Returns the vsfb_ id. poll()'s return deserializes a vector-store body,
+        so its .id is the vs_ id - take the batch id from create()."""
+        created = self.client.vector_stores.file_batches.create(
+            vector_store_id=vector_store_id,
+            file_ids=file_ids,
+        )
+        return created.id
+
+    def _retrieve_file_batch(
+        self, batch_id: str, vector_store_id: str
+    ) -> VectorStoreFileBatch:
+        return self.client.vector_stores.file_batches.retrieve(
+            batch_id, vector_store_id=vector_store_id
+        )
+
+    def _poll_file_batch(
+        self, batch_id: str, vector_store_id: str
+    ) -> VectorStoreFileBatch:
+        """Poll until indexing finishes; the Celery soft time limit is the deadline."""
+        while True:
+            batch = self._retrieve_file_batch(batch_id, vector_store_id)
+            if batch.status != "in_progress":
+                return batch
+            time.sleep(BATCH_POLL_INTERVAL_SECONDS)
+
+    def _raise_if_batch_incomplete(
+        self,
+        batch: VectorStoreFileBatch,
+        batch_id: str,
+        vector_store_id: str,
+        docs: list[Document],
+    ) -> None:
+        """Raise on any indexing failure so the batch attempt is retried."""
+        if batch.file_counts.failed > 0:
+            try:
+                failed_files = self.client.vector_stores.file_batches.list_files(
+                    vector_store_id=vector_store_id,
+                    batch_id=batch_id,
+                    filter="failed",
+                )
+                doc_by_file_id = {d.file_id[OPENAI_PROVIDER]: d for d in docs}
+                parts = []
+                for f in failed_files:
+                    d = doc_by_file_id.get(f.id)
+                    label = d.fname if d else f.id
+                    msg = f.last_error.message if f.last_error else "no error detail"
+                    parts.append(f"{label}: {msg}")
+                logger.error(
+                    f"[OpenAIVectorStoreCrud._raise_if_batch_incomplete] Files failed to index | "
+                    f"{{'batch_id': '{batch_id}', 'failed_files': '{', '.join(parts)}'}}"
+                )
+                raise RuntimeError("; ".join(parts))
+            except OpenAIError as err:
+                logger.warning(
+                    f"[OpenAIVectorStoreCrud._raise_if_batch_incomplete] Could not fetch per-file errors | "
+                    f"{{'batch_id': '{batch_id}', 'error': '{str(err)}'}}"
+                )
+                raise
+
+        # Only 'completed' is success; a 'cancelled'/'failed' batch with no per-file
+        # failures slips past the failed-count check above.
+        if batch.status != "completed":
+            error_message = (
+                f"[OPENAI] Vector store indexing did not complete "
+                f"(status: {batch.status}). Retry the collection."
+            )
+            logger.error(
+                f"[OpenAIVectorStoreCrud._raise_if_batch_incomplete] {error_message} | "
+                f"vector_store_id={vector_store_id}, batch_id={batch_id}, "
+                f"status={batch.status}"
+            )
+            raise RuntimeError(error_message)
+
+    @retry(
+        reraise=True,
+        stop=stop_after_attempt(BATCH_INDEX_MAX_ATTEMPTS),
+        wait=wait_exponential(multiplier=BATCH_RETRY_BACKOFF_BASE_SECONDS),
+        retry=retry_if_exception_type((OpenAIError, RuntimeError)),
+        before_sleep=_log_batch_retry,
+    )
+    def _create_and_index_batch(
+        self, vector_store_id: str, docs: list[Document]
+    ) -> tuple[VectorStoreFileBatch, str]:
+        """Create the file batch, wait for indexing, verify it completed. Retried as
+        a unit on any OpenAI/indexing failure; SoftTimeLimitExceeded is not retried
+        (not an OpenAIError/RuntimeError) so it aborts the task inside the window."""
+        batch_id = self._create_file_batch(
+            vector_store_id, [doc.file_id[OPENAI_PROVIDER] for doc in docs]
+        )
+        batch = self._poll_file_batch(batch_id, vector_store_id)
+        self._raise_if_batch_incomplete(batch, batch_id, vector_store_id, docs)
+        return batch, batch_id
+
     def update(
         self,
         vector_store_id: str,
@@ -115,11 +239,7 @@ def update(
         )
 
         try:
-            batch = self.client.vector_stores.file_batches.upload_and_poll(
-                vector_store_id=vector_store_id,
-                files=[],
-                file_ids=[doc.file_id[OPENAI_PROVIDER] for doc in docs],
-            )
+            batch, batch_id = self._create_and_index_batch(vector_store_id, docs)
         except openai.RateLimitError as e:
             error_message = (
                 f"[OPENAI] Rate limit exceeded (code: {e.status_code}): "
@@ -215,30 +335,9 @@ def update(
 
         logger.info(
             f"[OpenAIVectorStoreCrud.update] Batch complete | "
-            f"{{'vector_store_id': '{vector_store_id}', "
+            f"{{'vector_store_id': '{vector_store_id}', 'batch_id': '{batch_id}', "
             f"'completed': {batch.file_counts.completed}, 'failed': {batch.file_counts.failed}}}"
         )
-        if batch.file_counts.failed > 0:
-            try:
-                failed_files = self.client.vector_stores.file_batches.list_files(
-                    vector_store_id=vector_store_id,
-                    batch_id=batch.id,
-                    filter="failed",
-                )
-                doc_by_file_id = {d.file_id[OPENAI_PROVIDER]: d for d in docs}
-                parts = []
-                for f in failed_files:
-                    d = doc_by_file_id.get(f.id)
-                    label = d.fname if d else f.id
-                    msg = f.last_error.message if f.last_error else "no error detail"
-                    parts.append(f"{label}: {msg}")
-                raise RuntimeError("; ".join(parts))
-            except OpenAIError as err:
-                logger.warning(
-                    f"[OpenAIVectorStoreCrud.update] Could not fetch per-file errors | "
-                    f"{{'batch_id': '{batch.id}', 'error': '{str(err)}'}}"
-                )
-                raise
 
     def delete(self, vector_store_id: str, retries: int = 3):
         if retries < 1:

backend/app/services/collections/create_collection.py

@@ -403,9 +403,11 @@ def execute_batch_job(
                 collection_job_crud = CollectionJobCrud(session, project_id)
                 collection_job = collection_job_crud.read_one(job_uuid)
                 already_uploaded = collection_job.documents_uploaded or []
-                now_uploaded = already_uploaded + [
-                    str(d) for d in all_doc_ids_this_batch
-                ]
+                now_uploaded = list(
+                    dict.fromkeys(
+                        already_uploaded + [str(d) for d in all_doc_ids_this_batch]
+                    )
+                )
 
                 collection_job = collection_job_crud.update(
                     job_uuid,
@@ -507,7 +509,10 @@ def execute_batch_job(
                     webhook_secret=webhook_secret,
                 )
 
-        except (Timeout, SoftTimeLimitExceeded) as err:
+        except (Timeout, SoftTimeLimitExceeded):
+            # Batch-level retries happen in-task (tenacity in OpenAIVectorStoreCrud),
+            # so hitting the soft time limit means the window is spent — fail, don't
+            # re-queue.
             timeout_err = TimeoutError("Task exceeded soft time limit")
             logger.warning(
                 "[create_collection.execute_batch_job] Collection Creation Timed Out | {'collection_job_id': '%s', 'error': '%s'}",

backend/app/services/collections/providers/registry.py

@@ -5,6 +5,7 @@
 from openai import OpenAI
 
 from app.crud import get_provider_credential
+from app.crud.rag.open_ai import OPENAI_TIMEOUT_SECONDS
 from app.services.collections.providers.base import BaseProvider
 from app.services.collections.providers.gemini import GeminiAIStudioProvider
 from app.services.collections.providers.openai import OpenAIProvider
@@ -63,7 +64,11 @@ def get_llm_provider(
     if provider == LLMProvider.OPENAI:
         if "api_key" not in credentials:
             raise ValueError("OpenAI credentials not configured for this project.")
-        client = OpenAI(api_key=credentials["api_key"])
+        client = OpenAI(
+            api_key=credentials["api_key"],
+            max_retries=0,
+            timeout=OPENAI_TIMEOUT_SECONDS,
+        )
     elif provider == LLMProvider.GOOGLE_AISTUDIO:
         if "api_key" not in credentials:
             raise ValueError(

backend/app/services/doctransform/registry.py

@@ -39,6 +39,8 @@ class TransformationError(Exception):
     ".markdown": "markdown",
     ".csv": "csv",
     ".json": "json",
+    ".xlsx": "xlsx",
+    ".xls": "xls",
 }
 
 # Map format names to file extensions
@@ -51,6 +53,8 @@ class TransformationError(Exception):
     "markdown": ".md",
     "csv": ".csv",
     "json": ".json",
+    "xlsx": ".xlsx",
+    "xls": ".xls",
 }
 
 

backend/app/services/documents/helpers.py

@@ -1,3 +1,4 @@
+import logging
 from typing import Optional, Tuple, Iterable, Union
 from uuid import UUID
 
@@ -11,6 +12,10 @@
 )
 from app.crud import DocTransformationJobCrud, DocumentCrud
 from app.services.doctransform import job as transformation_job
+from app.services.documents.validator import (
+    DocumentValidationError,
+    validate_document_content,
+)
 from app.models import (
     DocTransformJobCreate,
     TransformationStatus,
@@ -23,6 +28,39 @@
 )
 
 
+logger = logging.getLogger(__name__)
+
+
+def validate_upload(
+    *,
+    src: UploadFile,
+    target_format: str | None,
+    transformer: str | None,
+) -> Tuple[str, str | None]:
+    """
+    Full pre-storage gate: extension and transformer validation plus a content
+    sanity check. Returns (source_format, actual_transformer_or_none).
+
+    Raises: HTTPException(400) on client errors.
+    """
+    source_format, actual_transformer = pre_transform_validation(
+        src_filename=src.filename,
+        target_format=target_format,
+        transformer=transformer,
+    )
+
+    try:
+        validate_document_content(file=src, source_format=source_format)
+    except DocumentValidationError as e:
+        logger.warning(
+            f"[validate_upload] Document failed sanity check | "
+            f"filename: {e.filename} | format: {source_format} | reason: {e.reason}"
+        )
+        raise HTTPException(status_code=400, detail=e.client_message)
+
+    return source_format, actual_transformer
+
+
 def calculate_file_size(file: UploadFile) -> float:
     """
     Calculate the size of an uploaded file in kilobytes.