From f24922f962e4899f794bf3017549a76bfe035d4a Mon Sep 17 00:00:00 2001 From: Jordan Dubrick Date: Wed, 5 Aug 2026 16:08:21 -0400 Subject: [PATCH 1/2] Resolve question_validity shield prompts from customization profile. When model_prompt or invalid_question_response are omitted, fill them at configuration load from the profile (then LCORE defaults) so GET /v1/shields and agent QV see effective text without duplicating YAML. Co-authored-by: Cursor --- docs/devel_doc/openapi.json | 57 +++- docs/models/successful_responses.json | 37 +- docs/models/successful_responses.md | 6 +- docs/user_doc/config.md | 14 +- docs/user_doc/shields_guide.md | 75 ++++- examples/lightspeed-stack-shields.yaml | 8 +- .../api/responses/successful/catalog.py | 2 +- src/models/config.py | 117 ++++++- .../question_validity/_capability.py | 50 ++- tests/profiles/empty_qv_strings/profile.py | 9 + tests/profiles/no_qv_keys/profile.py | 7 + tests/profiles/test/profile.py | 4 + tests/unit/app/endpoints/test_shields.py | 2 +- .../responses/test_successful_responses.py | 2 +- .../question_validity/test_capability.py | 81 +++-- tests/unit/qv_config.py | 23 ++ tests/unit/test_configuration.py | 317 ++++++++++++++++-- tests/unit/utils/test_prompts.py | 19 ++ tests/unit/utils/test_pydantic_ai.py | 10 +- 19 files changed, 709 insertions(+), 131 deletions(-) create mode 100644 tests/profiles/empty_qv_strings/profile.py create mode 100644 tests/profiles/no_qv_keys/profile.py create mode 100644 tests/unit/qv_config.py diff --git a/docs/devel_doc/openapi.json b/docs/devel_doc/openapi.json index 020105046..9cf355734 100644 --- a/docs/devel_doc/openapi.json +++ b/docs/devel_doc/openapi.json @@ -1440,7 +1440,7 @@ "config": { "invalid_question_response": "I can only answer questions about the product.", "model_id": "openai/gpt-4o-mini", - "model_prompt": "Is this question valid?" + "model_prompt": "Is this question valid? ${message}" }, "name": "question-validity", "provider_id": "question_validity", @@ -13887,7 +13887,34 @@ "type": "object", "title": "System prompts", "description": "Dictionary containing map of system prompts", - "default": {} + "default": {}, + "readOnly": true + }, + "validation": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Question validity classifier prompt", + "description": "Read-only. Loaded from the custom profile Python module (PROFILE_CONFIG['system_prompts']['validation']), not from lightspeed-stack.yaml. Used when a question_validity shield omits model_prompt.", + "readOnly": true + }, + "invalid_resp": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Invalid question response", + "description": "Read-only. Loaded from the custom profile Python module (PROFILE_CONFIG['query_responses']['invalid_resp']), not from lightspeed-stack.yaml. Used when a question_validity shield omits invalid_question_response.", + "readOnly": true } }, "type": "object", @@ -18804,16 +18831,28 @@ "description": "The model_id to use for the guard" }, "model_prompt": { - "type": "string", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "Model prompt", - "description": "The default prompt sent to the LLM used to validate the Users' question.", - "default": "\nInstructions:\n- You are a question classifying tool\n- You are an expert in kubernetes and openshift\n- Your job is to determine where or a user's question is related to kubernetes and/or openshift technologies and to provide a one-word response.\n- If a question appears to be related to kubernetes or openshift technologies, answer with the word ${allowed}, otherwise answer with the word ${rejected}.\n- Do not explain your answer, just provide the one-word response. Do not give any other response.\n- If the given question is an empty string, answer with the word ${rejected}\n\n\nExample Question:\nWhy is the sky blue?\nExample Response:\n${rejected}\n\nExample Question:\nWhy is the grass green?\nExample Response:\n${rejected}\n\nExample Question:\nWhy is sand yellow?\nExample Response:\n${rejected}\n\nExample Question:\nCan you help configure my cluster to automatically scale?\nExample Response:\n${allowed}\n\nQuestion:\n${message}\nResponse:\n" + "description": "Classifier prompt for the agent / wrap_run path. Null/omitted at load is filled from the profile module's system_prompts.validation, then the LCORE default; an explicit empty string is kept. Include $message or ${message} so the user question is substituted. Not used by responses-path run(), which sends raw user input." }, "invalid_question_response": { - "type": "string", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "Invalid question response", - "description": "The default response when the Users' question is determined to be invalid.", - "default": "\nHi, I'm the OpenShift Lightspeed assistant, I can help you with questions about OpenShift, \nplease ask me a question related to OpenShift.\n" + "description": "Refusal text used on both agent and responses / run() paths. Null/omitted at load is filled from the profile module's query_responses.invalid_resp, then the LCORE default; an explicit empty string is kept." } }, "additionalProperties": false, @@ -21233,7 +21272,7 @@ "config": { "invalid_question_response": "I can only answer questions about the product.", "model_id": "openai/gpt-4o-mini", - "model_prompt": "Is this question valid?" + "model_prompt": "Is this question valid? ${message}" }, "name": "question-validity", "provider_id": "question_validity", diff --git a/docs/models/successful_responses.json b/docs/models/successful_responses.json index f1dcd79bf..58f0af2a6 100644 --- a/docs/models/successful_responses.json +++ b/docs/models/successful_responses.json @@ -1524,8 +1524,25 @@ }, "default": {}, "description": "Dictionary containing map of system prompts", + "readOnly": true, "title": "System prompts", "type": "object" + }, + "validation": { + "type": "string", + "nullable": true, + "default": null, + "description": "Read-only. Loaded from the custom profile Python module (PROFILE_CONFIG['system_prompts']['validation']), not from lightspeed-stack.yaml. Used when a question_validity shield omits model_prompt.", + "readOnly": true, + "title": "Question validity classifier prompt" + }, + "invalid_resp": { + "type": "string", + "nullable": true, + "default": null, + "description": "Read-only. Loaded from the custom profile Python module (PROFILE_CONFIG['query_responses']['invalid_resp']), not from lightspeed-stack.yaml. Used when a question_validity shield omits invalid_question_response.", + "readOnly": true, + "title": "Invalid question response" } }, "required": [ @@ -4473,16 +4490,18 @@ "type": "string" }, "model_prompt": { - "default": "\nInstructions:\n- You are a question classifying tool\n- You are an expert in kubernetes and openshift\n- Your job is to determine where or a user's question is related to kubernetes and/or openshift technologies and to provide a one-word response.\n- If a question appears to be related to kubernetes or openshift technologies, answer with the word ${allowed}, otherwise answer with the word ${rejected}.\n- Do not explain your answer, just provide the one-word response. Do not give any other response.\n- If the given question is an empty string, answer with the word ${rejected}\n\n\nExample Question:\nWhy is the sky blue?\nExample Response:\n${rejected}\n\nExample Question:\nWhy is the grass green?\nExample Response:\n${rejected}\n\nExample Question:\nWhy is sand yellow?\nExample Response:\n${rejected}\n\nExample Question:\nCan you help configure my cluster to automatically scale?\nExample Response:\n${allowed}\n\nQuestion:\n${message}\nResponse:\n", - "description": "The default prompt sent to the LLM used to validate the Users' question.", - "title": "Model prompt", - "type": "string" + "type": "string", + "nullable": true, + "default": null, + "description": "Classifier prompt for the agent / wrap_run path. Null/omitted at load is filled from the profile module's system_prompts.validation, then the LCORE default; an explicit empty string is kept. Include $message or ${message} so the user question is substituted. Not used by responses-path run(), which sends raw user input.", + "title": "Model prompt" }, "invalid_question_response": { - "default": "\nHi, I'm the OpenShift Lightspeed assistant, I can help you with questions about OpenShift, \nplease ask me a question related to OpenShift.\n", - "description": "The default response when the Users' question is determined to be invalid.", - "title": "Invalid question response", - "type": "string" + "type": "string", + "nullable": true, + "default": null, + "description": "Refusal text used on both agent and responses / run() paths. Null/omitted at load is filled from the profile module's query_responses.invalid_resp, then the LCORE default; an explicit empty string is kept.", + "title": "Invalid question response" } }, "required": [ @@ -5838,7 +5857,7 @@ "config": { "invalid_question_response": "I can only answer questions about the product.", "model_id": "openai/gpt-4o-mini", - "model_prompt": "Is this question valid?" + "model_prompt": "Is this question valid? ${message}" }, "name": "question-validity", "provider_id": "question_validity", diff --git a/docs/models/successful_responses.md b/docs/models/successful_responses.md index 3da2c6735..42e6b426a 100644 --- a/docs/models/successful_responses.md +++ b/docs/models/successful_responses.md @@ -573,6 +573,8 @@ Custom profile customization for prompts and validation. |-------|------|-------------| | path | string | Path to Python modules containing custom profile. | | prompts | object | Dictionary containing map of system prompts | +| validation | string | Read-only. Loaded from the custom profile Python module (PROFILE_CONFIG['system_prompts']['validation']), not from lightspeed-stack.yaml. Used when a question_validity shield omits model_prompt. | +| invalid_resp | string | Read-only. Loaded from the custom profile Python module (PROFILE_CONFIG['query_responses']['invalid_resp']), not from lightspeed-stack.yaml. Used when a question_validity shield omits invalid_question_response. | ## Customization @@ -1951,8 +1953,8 @@ Configuration for the question validity guardrail. | Field | Type | Description | |-------|------|-------------| | model_id | string | The model_id to use for the guard | -| model_prompt | string | The default prompt sent to the LLM used to validate the Users' question. | -| invalid_question_response | string | The default response when the Users' question is determined to be invalid. | +| model_prompt | string | Classifier prompt for the agent / wrap_run path. Null/omitted at load is filled from the profile module's system_prompts.validation, then the LCORE default; an explicit empty string is kept. Include $message or ${message} so the user question is substituted. Not used by responses-path run(), which sends raw user input. | +| invalid_question_response | string | Refusal text used on both agent and responses / run() paths. Null/omitted at load is filled from the profile module's query_responses.invalid_resp, then the LCORE default; an explicit empty string is kept. | ## QuestionValidityShieldConfiguration diff --git a/docs/user_doc/config.md b/docs/user_doc/config.md index bf8d52d2b..accc44710 100644 --- a/docs/user_doc/config.md +++ b/docs/user_doc/config.md @@ -281,10 +281,12 @@ Conversation history configuration. Custom profile customization for prompts and validation. -| Field | Type | Description | -|---------|--------|---------------------------------------------------| -| path | string | Path to Python modules containing custom profile. | -| prompts | object | Dictionary containing map of system prompts | +| Field | Type | Description | +|--------------|--------|---------------------------------------------------| +| path | string | Path to Python modules containing custom profile. | +| prompts | object | Read-only map of system prompts loaded from the profile module | +| validation | string | Read-only. Loaded from profile module `PROFILE_CONFIG["system_prompts"]["validation"]` (not a YAML `customization` field). Used when a question_validity shield omits `model_prompt`. | +| invalid_resp | string | Read-only. Loaded from profile module `PROFILE_CONFIG["query_responses"]["invalid_resp"]` (not a YAML `customization` field). Used when a question_validity shield omits `invalid_question_response`. | ## Customization @@ -767,8 +769,8 @@ Configuration for the question validity guardrail. | Field | Type | Description | |---------------------------|--------|---------------------------------------------------------------| | model_id | string | The model_id to use for the guard | -| model_prompt | string | Prompt sent to the LLM used to validate the user's question | -| invalid_question_response | string | Response when the user's question is determined to be invalid | +| model_prompt | string | Optional classifier prompt (`null` when omitted in YAML). Filled at load from the profile module's `system_prompts.validation`, then the LCORE default. Explicit YAML (including `""`) is kept. Used on the agent / `wrap_run` path (with `$message` / `${message}` substitution). Not applied by responses-path `run()`. After load / on `GET /v1/shields`, the effective string is returned. | +| invalid_question_response | string | Optional refusal text (`null` when omitted in YAML). Filled at load from the profile module's `query_responses.invalid_resp`, then the LCORE default. Explicit YAML (including `""`) is kept. Used on both agent and responses / `run()` paths. After load / on `GET /v1/shields`, the effective string is returned. | ## QuestionValidityShieldConfiguration diff --git a/docs/user_doc/shields_guide.md b/docs/user_doc/shields_guide.md index 5e1bdeeca..0c1f4af59 100644 --- a/docs/user_doc/shields_guide.md +++ b/docs/user_doc/shields_guide.md @@ -81,8 +81,60 @@ for a complete example. | Config field | Required | Description | |--------------|----------|-------------| | `model_id` | Yes | Model used for the validity check (for example `openai/gpt-4o-mini`) | -| `model_prompt` | No | Classifier prompt (has a built-in default) | -| `invalid_question_response` | No | Reply returned when the question is rejected | +| `model_prompt` | No | Classifier prompt. When omitted: customization profile `system_prompts.validation`, then the LCORE default | +| `invalid_question_response` | No | Reply when rejected. When omitted: profile `query_responses.invalid_resp`, then the LCORE default | + +### Profile fallback (resolved at startup) + +When LCORE loads configuration, omitted `model_prompt` / +`invalid_question_response` on each `question_validity` shield are filled +once from the customization profile module (when `customization.profile_path` +points at a Python profile) or from LCORE defaults: + +| Shield field | Profile module key | Runtime use | +|--------------|--------------------|-------------| +| `model_prompt` | `PROFILE_CONFIG["system_prompts"]["validation"]` | **Classifier template** — agent / `wrap_run` only | +| `invalid_question_response` | `PROFILE_CONFIG["query_responses"]["invalid_resp"]` | **Refusal text** — agent and responses / `run()` paths | + +Those profile keys are read from the profile **Python module** +(`PROFILE_CONFIG`), not from fields under `customization:` in +`lightspeed-stack.yaml`. + +Explicit values in `lightspeed-stack.yaml` always win. Missing profile keys +fall through silently to LCORE defaults. After load, `GET /v1/shields` returns +the **effective** prompt and refusal text. + +**Omit vs empty string:** Only a missing / `null` field is filled. An explicit +empty string (`""`) in YAML **or** in the profile (`validation` / +`invalid_resp`) is kept as-is and is not replaced by LCORE defaults. + +**Upgrade note:** If a deployment already uses a customization profile that +defines `validation` / `invalid_resp`, and a QV shield omits those YAML fields, +the effective values change after upgrade (no YAML edit required): classifier +text on the agent path, and refusal text on both agent and responses paths. +Set the fields explicitly in YAML to keep the previous LCORE defaults. + +**Schema / OpenAPI note:** In the config model these fields are optional +(`null` when omitted in YAML). After configuration load they are always filled +with the effective string. Clients that read `GET /v1/shields` see the resolved +values, not `null`. + +### `model_prompt` placeholders (agent / `wrap_run` only) + +On the agent capability path, the classifier prompt is rendered with Python +`string.Template` before the validity model runs. Include `$message` or +`${message}` so the user question is substituted into the prompt. Without it, +the classifier still runs but does not see the user's question (same behavior +as before). The responses moderation path (`run()`) does not use this template; +it sends raw user input (see below). + +Optional placeholders: + +| Placeholder | Substituted value | +|-------------|-------------------| +| `${message}` / `$message` | User question text | +| `${allowed}` / `$allowed` | `ALLOWED` | +| `${rejected}` / `$rejected` | `REJECTED` | ## redaction @@ -95,8 +147,9 @@ Invalid regex patterns are rejected at configuration load time. # How shields apply at runtime -The same shield logic (`question_validity` and `redaction`) is used on both -agent-based and responses-based endpoints; only the integration point differs. +Both agent-based and responses-based endpoints use the configured +`question_validity` and `redaction` shields; the integration point differs +(see below for how `question_validity` prompts are applied on each path). ## Agent-based endpoints @@ -108,10 +161,16 @@ questions or redacting PII from model messages — using the configured shields. ## Responses-based endpoints On pure responses-based endpoints (for example `/v1/responses` and `/v1/infer`), -there is no agent capability layer. Instead, LCORE runs the **same core shield -functionality directly** through a custom API (`run_shield_moderation`) before -each request. When moderation blocks the input, the endpoint returns a refusal -(and may persist the blocked turn) without calling the model. +there is no agent capability layer. Instead, LCORE runs shield moderation +directly through `run_shield_moderation_v2` before each request. When +moderation blocks the input, the endpoint returns a refusal (and may persist +the blocked turn) without calling the model. + +For `question_validity`, that moderation path sends the **raw user input** to +the validity model. It does **not** render `model_prompt` with +`$message` / `${message}` the way the agent capability path (`wrap_run`) does. +Configured / profile-resolved `invalid_question_response` is still used for the +refusal text on this path. This matches pre-existing moderation behavior. ## Per-endpoint behavior diff --git a/examples/lightspeed-stack-shields.yaml b/examples/lightspeed-stack-shields.yaml index 2400f6ab7..5679f071b 100644 --- a/examples/lightspeed-stack-shields.yaml +++ b/examples/lightspeed-stack-shields.yaml @@ -19,14 +19,14 @@ authentication: # LCORE-owned safety shields (not Llama Stack / OGX Safety API resources). # Listed via GET /v1/shields; selected per request with optional shield_ids. shields: - - identifier: topic-guard + - name: topic-guard provider_id: question_validity config: model_id: openai/gpt-4o-mini - # Optional; omit to use built-in defaults: - model_prompt: "Classify whether the question is about OpenShift. Reply ALLOWED or REJECTED." + # Optional; omit to use profile/LCORE defaults: + model_prompt: "Classify whether the question is about OpenShift. Reply ALLOWED or REJECTED. Question: ${message}" invalid_question_response: "I can only answer questions about OpenShift." - - identifier: pii-redaction + - name: pii-redaction provider_id: redaction config: rules: diff --git a/src/models/api/responses/successful/catalog.py b/src/models/api/responses/successful/catalog.py index 54ade5d84..5c83f2bb6 100644 --- a/src/models/api/responses/successful/catalog.py +++ b/src/models/api/responses/successful/catalog.py @@ -95,7 +95,7 @@ class ShieldsResponse(AbstractSuccessfulResponse): "type": "shield", "config": { "model_id": "openai/gpt-4o-mini", - "model_prompt": "Is this question valid?", + "model_prompt": "Is this question valid? ${message}", "invalid_question_response": ( "I can only answer questions about the product." ), diff --git a/src/models/config.py b/src/models/config.py index 8e4dfd1cb..2d6b2d7a9 100644 --- a/src/models/config.py +++ b/src/models/config.py @@ -1636,6 +1636,33 @@ class CustomProfile: init=False, title="System prompts", description="Dictionary containing map of system prompts", + json_schema_extra={"readOnly": True}, + ) + + validation: Optional[str] = Field( + default=None, + init=False, + title="Question validity classifier prompt", + description=( + "Read-only. Loaded from the custom profile Python module " + "(PROFILE_CONFIG['system_prompts']['validation']), not from " + "lightspeed-stack.yaml. Used when a question_validity shield omits " + "model_prompt." + ), + json_schema_extra={"readOnly": True}, + ) + + invalid_resp: Optional[str] = Field( + default=None, + init=False, + title="Invalid question response", + description=( + "Read-only. Loaded from the custom profile Python module " + "(PROFILE_CONFIG['query_responses']['invalid_resp']), not from " + "lightspeed-stack.yaml. Used when a question_validity shield omits " + "invalid_question_response." + ), + json_schema_extra={"readOnly": True}, ) def __post_init__(self) -> None: @@ -1647,7 +1674,16 @@ def _validate_and_process(self) -> None: checks.file_check(Path(self.path), "custom profile") profile_module = checks.import_python_module("profile", self.path) if profile_module is not None and checks.is_valid_profile(profile_module): - self.prompts = profile_module.PROFILE_CONFIG.get("system_prompts", {}) + profile_config = profile_module.PROFILE_CONFIG + self.prompts = profile_config.get("system_prompts", {}) + validation = self.prompts.get("validation") + if isinstance(validation, str): + self.validation = validation + query_responses = profile_config.get("query_responses") + if isinstance(query_responses, dict): + invalid_resp = query_responses.get("invalid_resp") + if isinstance(invalid_resp, str): + self.invalid_resp = invalid_resp def get_prompts(self) -> dict[str, str]: """ @@ -1658,6 +1694,24 @@ def get_prompts(self) -> dict[str, str]: """ return self.prompts + def get_validation(self) -> Optional[str]: + """Return the profile question-validity classifier prompt, if loaded. + + Returns: + The ``system_prompts.validation`` string from the profile module, + or None when that key is absent or not a string. + """ + return self.validation + + def get_invalid_resp(self) -> Optional[str]: + """Return the profile invalid-question refusal text, if loaded. + + Returns: + The ``query_responses.invalid_resp`` string from the profile module, + or None when that key is absent or not a string. + """ + return self.invalid_resp + class Customization(ConfigurationBase): """Service customization.""" @@ -2667,15 +2721,26 @@ class QuestionValidityConfig(ConfigurationBase): model_id: str = Field( ..., title="Model id", description="The model_id to use for the guard" ) - model_prompt: str = Field( - default=constants.DEFAULT_MODEL_PROMPT, + model_prompt: Optional[str] = Field( + default=None, title="Model prompt", - description="The default prompt sent to the LLM used to validate the Users' question.", + description=( + "Classifier prompt for the agent / wrap_run path. Null/omitted at " + "load is filled from the profile module's system_prompts.validation, " + "then the LCORE default; an explicit empty string is kept. Include " + "$message or ${message} so the user question is substituted. Not " + "used by responses-path run(), which sends raw user input." + ), ) - invalid_question_response: str = Field( - default=constants.DEFAULT_INVALID_QUESTION_RESPONSE, + invalid_question_response: Optional[str] = Field( + default=None, title="Invalid question response", - description="The default response when the Users' question is determined to be invalid.", + description=( + "Refusal text used on both agent and responses / run() paths. " + "Null/omitted at load is filled from the profile module's " + "query_responses.invalid_resp, then the LCORE default; an explicit " + "empty string is kept." + ), ) @@ -3189,6 +3254,44 @@ def validate_shield_names_unique(self) -> Self: ) return self + @model_validator(mode="after") + def resolve_question_validity_shield_prompts(self) -> Self: + """Fill omitted QV prompt/refusal fields from profile or LCORE defaults. + + Runs at configuration load so ``GET /v1/shields`` returns the effective + ``model_prompt`` and ``invalid_question_response``. Explicit YAML values + are left unchanged. + + Returns: + Self: The model instance after resolving shield text fields. + """ + profile = None + if self.customization is not None: + # pylint: disable=no-member # Pydantic nested model field + profile = self.customization.custom_profile + profile_validation = None + profile_invalid_resp = None + if profile is not None: + profile_validation = profile.get_validation() + profile_invalid_resp = profile.get_invalid_resp() + + for shield in self.shields: + match shield.config: + case QuestionValidityConfig() as qv_config: + if qv_config.model_prompt is None: + qv_config.model_prompt = ( + profile_validation + if profile_validation is not None + else constants.DEFAULT_MODEL_PROMPT + ) + if qv_config.invalid_question_response is None: + qv_config.invalid_question_response = ( + profile_invalid_resp + if profile_invalid_resp is not None + else constants.DEFAULT_INVALID_QUESTION_RESPONSE + ) + return self + @model_validator(mode="after") def validate_mcp_auth_headers(self) -> Self: """ diff --git a/src/pydantic_ai_lightspeed/capabilities/question_validity/_capability.py b/src/pydantic_ai_lightspeed/capabilities/question_validity/_capability.py index c8097d1ad..a79336bec 100644 --- a/src/pydantic_ai_lightspeed/capabilities/question_validity/_capability.py +++ b/src/pydantic_ai_lightspeed/capabilities/question_validity/_capability.py @@ -116,21 +116,43 @@ class QuestionValidity(AbstractSafetyCapability): The guard function receives the user prompt and returns True if safe. + ``config.model_prompt`` and ``config.invalid_question_response`` must be + set before construction. In service startup, + ``Configuration.resolve_question_validity_shield_prompts`` fills omitted + fields from the customization profile or LCORE defaults. + Example: ```python from pydantic_ai import Agent - from pydantic_ai.models.openai import OpenAIResponsesModel + from models.config import QuestionValidityConfig - model = OpenAIResponsesModel("gpt-4o-mini") - agent = Agent("openai:gpt-4.1", capabilities=[QuestionValidity(model)]) + config = QuestionValidityConfig( + model_id="gpt-4o-mini", + model_prompt="Is this on-topic? ${message}", + invalid_question_response="Off-topic.", + ) + agent = Agent("openai:gpt-4.1", capabilities=[QuestionValidity(config=config)]) ``` """ config: QuestionValidityConfig _model: Model = field(init=False) + _model_prompt: str = field(init=False) + _invalid_question_response: str = field(init=False) def __post_init__(self) -> None: - """Initialize the model instance from the configured model ID.""" + """Validate required prompt fields and initialize the model.""" + model_prompt = self.config.model_prompt + invalid_question_response = self.config.invalid_question_response + if model_prompt is None or invalid_question_response is None: + raise ValueError( + "question_validity model_prompt and invalid_question_response " + "must be set; Configuration resolves omitted fields at load time" + ) + + self._model_prompt = model_prompt + self._invalid_question_response = invalid_question_response + ogx_client = AsyncOgxClientHolder().get_client() self._model = OgxResponsesModel.from_ogx_client( @@ -148,7 +170,7 @@ def _build_prompt(self, message: Optional[str | Sequence[UserContent]]) -> str: Returns: The rendered prompt string ready to send to the validity model. """ - return Template(self.config.model_prompt).substitute( + return Template(self._model_prompt).substitute( message=_message_to_str(message), allowed=SUBJECT_ALLOWED, rejected=SUBJECT_REJECTED, @@ -191,7 +213,7 @@ async def wrap_run( message_history=[ ModelRequest.user_text_prompt(user_message), ModelResponse( - [TextPart(self.config.invalid_question_response)], + [TextPart(self._invalid_question_response)], finish_reason="stop", ), ], @@ -203,7 +225,7 @@ async def wrap_run( AsyncOgxClientHolder().get_client(), conversation_id, user_message, - self.config.invalid_question_response, + self._invalid_question_response, ) else: logger.warning( @@ -211,12 +233,16 @@ async def wrap_run( "skipping v1/conversation persistence for rejected question." ) - return AgentRunResult( - output=self.config.invalid_question_response, _state=state - ) + return AgentRunResult(output=self._invalid_question_response, _state=state) async def run(self, input_text: str) -> ShieldModerationResult: - """Run question-validity check and return a moderation result.""" + """Run question-validity check and return a moderation result. + + Sends ``input_text`` to the validity model as-is (no ``_build_prompt`` / + ``model_prompt`` template rendering). Used by responses-path shield + moderation. The agent path uses ``wrap_run``, which renders + ``model_prompt``. + """ result = await model_request( model=self._model, messages=[ModelRequest.user_text_prompt(input_text)] ) @@ -225,6 +251,6 @@ async def run(self, input_text: str) -> ShieldModerationResult: return ShieldModerationPassed() return ShieldModerationBlocked( - message=self.config.invalid_question_response, + message=self._invalid_question_response, moderation_id=f"modr-{uuid4()}", ) diff --git a/tests/profiles/empty_qv_strings/profile.py b/tests/profiles/empty_qv_strings/profile.py new file mode 100644 index 000000000..94925bd79 --- /dev/null +++ b/tests/profiles/empty_qv_strings/profile.py @@ -0,0 +1,9 @@ +"""Profile that sets QV validation / invalid_resp to empty strings.""" + +PROFILE_CONFIG = { + "system_prompts": { + "default": "Default system prompt only.", + "validation": "", + }, + "query_responses": {"invalid_resp": ""}, +} diff --git a/tests/profiles/no_qv_keys/profile.py b/tests/profiles/no_qv_keys/profile.py new file mode 100644 index 000000000..106f2e024 --- /dev/null +++ b/tests/profiles/no_qv_keys/profile.py @@ -0,0 +1,7 @@ +"""Profile with system prompts but no QV validation / invalid_resp keys.""" + +PROFILE_CONFIG = { + "system_prompts": { + "default": "Default system prompt only.", + }, +} diff --git a/tests/profiles/test/profile.py b/tests/profiles/test/profile.py index 76698f39b..fe1b52ac8 100644 --- a/tests/profiles/test/profile.py +++ b/tests/profiles/test/profile.py @@ -32,6 +32,10 @@ How can I integrate GitOps into my pipeline? Example Response: {SUBJECT_ALLOWED} + +Question: +${{message}} +Response: """ TOPIC_SUMMARY_PROMPT_TEMPLATE = """ diff --git a/tests/unit/app/endpoints/test_shields.py b/tests/unit/app/endpoints/test_shields.py index 704478f51..e305a5210 100644 --- a/tests/unit/app/endpoints/test_shields.py +++ b/tests/unit/app/endpoints/test_shields.py @@ -103,7 +103,7 @@ async def test_shields_endpoint_handler_configured_shields( "provider_id": "question_validity", "config": { "model_id": "openai/gpt-4o-mini", - "model_prompt": "Is this question valid?", + "model_prompt": "Is this question valid? ${message}", "invalid_question_response": "I can only answer product questions.", }, }, diff --git a/tests/unit/models/responses/test_successful_responses.py b/tests/unit/models/responses/test_successful_responses.py index fe27f789e..9ee22ee0d 100644 --- a/tests/unit/models/responses/test_successful_responses.py +++ b/tests/unit/models/responses/test_successful_responses.py @@ -193,7 +193,7 @@ def test_constructor(self) -> None: "type": "shield", "config": { "model_id": "openai/gpt-4o-mini", - "model_prompt": "Is this question valid?", + "model_prompt": "Is this question valid? ${message}", "invalid_question_response": ( "I can only answer questions about the product." ), diff --git a/tests/unit/pydantic_ai_lightspeed/capabilities/question_validity/test_capability.py b/tests/unit/pydantic_ai_lightspeed/capabilities/question_validity/test_capability.py index 9658e917a..da2680adf 100644 --- a/tests/unit/pydantic_ai_lightspeed/capabilities/question_validity/test_capability.py +++ b/tests/unit/pydantic_ai_lightspeed/capabilities/question_validity/test_capability.py @@ -10,10 +10,7 @@ from pydantic_ai.usage import RequestUsage, RunUsage from pytest_mock import MockerFixture, MockType -from constants import ( - DEFAULT_INVALID_QUESTION_RESPONSE, - DEFAULT_MODEL_PROMPT, -) +from constants import DEFAULT_INVALID_QUESTION_RESPONSE from models.common.moderation import ShieldModerationBlocked, ShieldModerationPassed from models.config import ( QuestionValidityConfig, @@ -25,6 +22,7 @@ _extract_conversation_id, _extract_message_str_from_user_content, ) +from tests.unit.qv_config import make_qv_config _MODULE = "pydantic_ai_lightspeed.capabilities.question_validity._capability" @@ -110,17 +108,17 @@ def test_returns_none_when_conversation_not_string( class TestQuestionValidityConfigInit: """Tests for QuestionValidityConfig initialization.""" - def test_default_model_prompt(self) -> None: - """Test that default model_prompt is used.""" + def test_omitted_model_prompt_is_none(self) -> None: + """Omitted model_prompt defaults to None before Configuration load fills it.""" qv_config = QuestionValidityConfig(model_id="test") - assert qv_config.model_prompt == DEFAULT_MODEL_PROMPT + assert qv_config.model_prompt is None - def test_default_invalid_question_response(self) -> None: - """Test that default invalid_question_response is used.""" + def test_omitted_invalid_question_response_is_none(self) -> None: + """Omitted invalid_question_response defaults to None before Configuration load fills it.""" qv_config = QuestionValidityConfig(model_id="test") - assert qv_config.invalid_question_response == DEFAULT_INVALID_QUESTION_RESPONSE + assert qv_config.invalid_question_response is None def test_custom_model_prompt(self) -> None: """Test that custom model_prompt can be provided.""" @@ -162,8 +160,7 @@ def test_post_init_wires_client_and_model(self, mocker: MockerFixture) -> None: f"{_MODULE}.OgxResponsesModel.from_ogx_client", ) - config = QuestionValidityConfig(model_id="test-model") - QuestionValidity(config=config) + QuestionValidity(config=make_qv_config(model_id="test-model")) mock_holder.return_value.get_client.assert_called_once() mock_from_client.assert_called_once_with( @@ -180,12 +177,19 @@ def test_model_is_assigned_from_factory(self, mocker: MockerFixture) -> None: f"{_MODULE}.OgxResponsesModel.from_ogx_client", return_value=mock_model, ) - config = QuestionValidityConfig(model_id="test") - qv = QuestionValidity(config=config) + qv = QuestionValidity(config=make_qv_config()) assert qv._model is mock_model + def test_unset_prompt_fields_raise(self, mocker: MockerFixture) -> None: + """Capability requires prompt fields already set by Configuration load.""" + mocker.patch(f"{_MODULE}.AsyncOgxClientHolder") + mocker.patch(f"{_MODULE}.OgxResponsesModel.from_ogx_client") + + with pytest.raises(ValueError, match="must be set"): + QuestionValidity(config=QuestionValidityConfig(model_id="test")) + class TestBuildPrompt: """Tests for QuestionValidity._build_prompt method.""" @@ -199,7 +203,7 @@ def _mock_create_model(self, mocker: MockerFixture) -> None: @pytest.fixture(name="question_validity") def question_validity_fixture(self) -> QuestionValidity: """Create a QuestionValidity instance with a mock model.""" - config = QuestionValidityConfig(model_id="test") + config = make_qv_config() return QuestionValidity(config=config) def test_string_input(self, question_validity: QuestionValidity) -> None: @@ -238,8 +242,7 @@ def test_substitutes_allowed_and_rejected( def test_custom_prompt_template(self) -> None: """Test with a custom prompt template.""" - config = QuestionValidityConfig( - model_id="test", + config = make_qv_config( model_prompt="Is '${message}' valid? ${allowed}/${rejected}", ) qv = QuestionValidity(config=config) @@ -299,7 +302,7 @@ async def test_allowed_question_calls_handler( return_value=mock_response, ) - config = QuestionValidityConfig(model_id="test") + config = make_qv_config() qv = QuestionValidity(config=config) result = await qv.wrap_run(mock_ctx, handler=mock_handler) @@ -323,7 +326,7 @@ async def test_rejected_question_returns_rejection( return_value=mock_response, ) - config = QuestionValidityConfig(model_id="test") + config = make_qv_config() qv = QuestionValidity(config=config) result = await qv.wrap_run(mock_ctx, handler=mock_handler) @@ -353,7 +356,7 @@ async def test_rejected_question_persists_turn_to_conversation( return_value=mock_response, ) - config = QuestionValidityConfig(model_id="test") + config = make_qv_config() qv = QuestionValidity(config=config) await qv.wrap_run(mock_ctx, handler=mock_handler) @@ -383,7 +386,7 @@ async def test_rejection_skips_persistence_when_conversation_id_missing( return_value=mock_response, ) - config = QuestionValidityConfig(model_id="test") + config = make_qv_config() qv = QuestionValidity(config=config) result = await qv.wrap_run(mock_ctx, handler=mock_handler) @@ -408,7 +411,7 @@ async def test_allowed_question_does_not_persist_turn( return_value=mock_response, ) - config = QuestionValidityConfig(model_id="test") + config = make_qv_config() qv = QuestionValidity(config=config) await qv.wrap_run(mock_ctx, handler=mock_handler) @@ -431,7 +434,7 @@ async def test_unexpected_response_treated_as_rejected( return_value=mock_response, ) - config = QuestionValidityConfig(model_id="test") + config = make_qv_config() qv = QuestionValidity(config=config) result = await qv.wrap_run(mock_ctx, handler=mock_handler) @@ -461,7 +464,7 @@ async def test_allowed_with_whitespace_still_accepted( return_value=mock_response, ) - config = QuestionValidityConfig(model_id="test") + config = make_qv_config() qv = QuestionValidity(config=config) result = await qv.wrap_run(mock_ctx, handler=mock_handler) @@ -486,7 +489,7 @@ async def test_usage_is_incremented( return_value=mock_response, ) - config = QuestionValidityConfig(model_id="test") + config = make_qv_config() qv = QuestionValidity(config=config) await qv.wrap_run(mock_ctx, handler=mock_handler) @@ -511,7 +514,7 @@ async def test_usage_is_incremented_on_rejection( return_value=mock_response, ) - config = QuestionValidityConfig(model_id="test") + config = make_qv_config() qv = QuestionValidity(config=config) await qv.wrap_run(mock_ctx, handler=mock_handler) @@ -536,7 +539,7 @@ async def test_rejection_result_contains_usage_in_state( return_value=mock_response, ) - config = QuestionValidityConfig(model_id="test") + config = make_qv_config() qv = QuestionValidity(config=config) result = await qv.wrap_run(mock_ctx, handler=mock_handler) @@ -559,9 +562,7 @@ async def test_custom_invalid_response( return_value=mock_response, ) - config = QuestionValidityConfig( - model_id="test", invalid_question_response="Custom rejection." - ) + config = make_qv_config(invalid_question_response="Custom rejection.") qv = QuestionValidity(config=config) result = await qv.wrap_run(mock_ctx, handler=mock_handler) @@ -583,7 +584,7 @@ async def test_model_request_receives_correct_prompt( ), ) - config = QuestionValidityConfig(model_id="test") + config = make_qv_config() qv = QuestionValidity(config=config) await qv.wrap_run(mock_ctx, handler=mock_handler) @@ -615,7 +616,7 @@ async def test_wrap_run_with_none_prompt( return_value=mock_response, ) - config = QuestionValidityConfig(model_id="test") + config = make_qv_config() qv = QuestionValidity(config=config) result = await qv.wrap_run(ctx, handler=mock_handler) @@ -634,7 +635,7 @@ async def test_wrap_run_propagates_model_request_error( side_effect=RuntimeError("connection failed"), ) - config = QuestionValidityConfig(model_id="test") + config = make_qv_config() qv = QuestionValidity(config=config) with pytest.raises(RuntimeError, match="connection failed"): @@ -661,7 +662,7 @@ async def test_wrap_run_with_sequence_prompt( ), ) - config = QuestionValidityConfig(model_id="test") + config = make_qv_config() qv = QuestionValidity(config=config) await qv.wrap_run(ctx, handler=mock_handler) @@ -689,7 +690,7 @@ async def test_allowed_returns_passed(self, mocker: MockerFixture) -> None: ) mocker.patch(f"{_MODULE}.model_request", return_value=mock_response) - config = QuestionValidityConfig(model_id="test") + config = make_qv_config() qv = QuestionValidity(config=config) result = await qv.run("How do I create a pod?") @@ -705,7 +706,7 @@ async def test_rejected_returns_blocked(self, mocker: MockerFixture) -> None: ) mocker.patch(f"{_MODULE}.model_request", return_value=mock_response) - config = QuestionValidityConfig(model_id="test") + config = make_qv_config() qv = QuestionValidity(config=config) result = await qv.run("What is the meaning of life?") @@ -726,7 +727,7 @@ async def test_unexpected_response_returns_blocked( ) mocker.patch(f"{_MODULE}.model_request", return_value=mock_response) - config = QuestionValidityConfig(model_id="test") + config = make_qv_config() qv = QuestionValidity(config=config) result = await qv.run("some input") @@ -749,7 +750,7 @@ async def test_allowed_with_whitespace_returns_passed( ) mocker.patch(f"{_MODULE}.model_request", return_value=mock_response) - config = QuestionValidityConfig(model_id="test") + config = make_qv_config() qv = QuestionValidity(config=config) result = await qv.run("How do I scale pods?") @@ -764,9 +765,7 @@ async def test_custom_invalid_response_message(self, mocker: MockerFixture) -> N ) mocker.patch(f"{_MODULE}.model_request", return_value=mock_response) - config = QuestionValidityConfig( - model_id="test", invalid_question_response="Custom rejection." - ) + config = make_qv_config(invalid_question_response="Custom rejection.") qv = QuestionValidity(config=config) result = await qv.run("off-topic question") diff --git a/tests/unit/qv_config.py b/tests/unit/qv_config.py new file mode 100644 index 000000000..c4a23439c --- /dev/null +++ b/tests/unit/qv_config.py @@ -0,0 +1,23 @@ +"""Shared helpers for constructing resolved QuestionValidityConfig in tests.""" + +from constants import ( + DEFAULT_INVALID_QUESTION_RESPONSE, + DEFAULT_MODEL_PROMPT, +) +from models.config import QuestionValidityConfig + + +def make_qv_config(**kwargs: object) -> QuestionValidityConfig: + """Build a QuestionValidityConfig with startup-resolved text fields filled. + + Production code fills omitted prompt/refusal fields in Configuration load. + Unit tests that construct configs directly should use this helper so + ``QuestionValidity`` construction succeeds. + """ + defaults: dict[str, object] = { + "model_id": "test", + "model_prompt": DEFAULT_MODEL_PROMPT, + "invalid_question_response": DEFAULT_INVALID_QUESTION_RESPONSE, + } + defaults.update(kwargs) + return QuestionValidityConfig(**defaults) # type: ignore[arg-type] diff --git a/tests/unit/test_configuration.py b/tests/unit/test_configuration.py index 23f77a4ca..fc9c65d2f 100644 --- a/tests/unit/test_configuration.py +++ b/tests/unit/test_configuration.py @@ -250,9 +250,13 @@ def test_init_from_dict() -> None: assert cfg.shields == [] -def test_init_from_dict_with_shields() -> None: - """Test initialization with guardrail shields configuration.""" - config_dict: dict[str, Any] = { +def _base_config_dict( + *, + shields: list[dict[str, Any]], + customization: dict[str, Any] | None = None, +) -> dict[str, Any]: + """Build a minimal AppConfig dict for shield / QV resolution tests.""" + return { "name": "foo", "service": { "host": "localhost", @@ -271,39 +275,62 @@ def test_init_from_dict_with_shields() -> None: "feedback_enabled": False, }, "mcp_servers": [], - "customization": None, + "customization": customization, "authentication": { "module": "noop", }, - "shields": [ - { - "name": "topic-guard-a", - "provider_id": "question_validity", - "config": {"model_id": "test-model"}, - }, - { - "name": "topic-guard-b", - "provider_id": "question_validity", - "config": {"model_id": "test-model-2"}, - }, - { - "name": "pii-guard", - "provider_id": "redaction", - "config": { - "rules": [ - {"pattern": r"\d+", "replacement": "[NUM]"}, - ], - }, - }, - ], + "shields": shields, } + + +def _init_config_from_dict(config_dict: dict[str, Any]) -> AppConfig: + """Initialize AppConfig from a dict and return it.""" cfg = AppConfig() cfg.init_from_dict(config_dict) + return cfg + + +def test_init_from_dict_with_shields() -> None: + """Test initialization with guardrail shields configuration.""" + cfg = _init_config_from_dict( + _base_config_dict( + shields=[ + { + "name": "topic-guard-a", + "provider_id": "question_validity", + "config": {"model_id": "test-model"}, + }, + { + "name": "topic-guard-b", + "provider_id": "question_validity", + "config": {"model_id": "test-model-2"}, + }, + { + "name": "pii-guard", + "provider_id": "redaction", + "config": { + "rules": [ + {"pattern": r"\d+", "replacement": "[NUM]"}, + ], + }, + }, + ], + ) + ) assert len(cfg.shields) == 3 assert cfg.shields[0].name == "topic-guard-a" assert cfg.shields[0].provider_id == "question_validity" assert cfg.shields[0].config.model_id == "test-model" # type: ignore[union-attr] + # Omitted QV text fields are filled at load from LCORE defaults. + assert ( + cfg.shields[0].config.model_prompt # type: ignore[union-attr] + == constants.DEFAULT_MODEL_PROMPT + ) + assert ( + cfg.shields[0].config.invalid_question_response # type: ignore[union-attr] + == constants.DEFAULT_INVALID_QUESTION_RESPONSE + ) assert cfg.shields[1].name == "topic-guard-b" assert cfg.shields[1].config.model_id == "test-model-2" # type: ignore[union-attr] assert cfg.shields[2].name == "pii-guard" @@ -311,6 +338,246 @@ def test_init_from_dict_with_shields() -> None: assert len(cfg.shields[2].config.compiled_patterns) == 1 # type: ignore[union-attr] +def test_qv_shield_prompts_resolve_from_profile_at_startup() -> None: + """Omitted QV fields are filled from the customization profile at load.""" + cfg = _init_config_from_dict( + _base_config_dict( + customization={ + "profile_path": "tests/profiles/test/profile.py", + }, + shields=[ + { + "name": "topic-guard", + "provider_id": "question_validity", + "config": {"model_id": "test-model"}, + }, + ], + ) + ) + + profile = CustomProfile(path="tests/profiles/test/profile.py") + assert ( + cfg.shields[0].config.model_prompt # type: ignore[union-attr] + == profile.get_validation() + ) + assert ( + cfg.shields[0].config.invalid_question_response # type: ignore[union-attr] + == profile.get_invalid_resp() + ) + + +def test_qv_shield_yaml_prompt_wins_over_profile_at_startup() -> None: + """Explicit YAML model_prompt is kept when a profile is also configured.""" + yaml_prompt = "YAML wins ${message}" + yaml_refusal = "YAML refusal" + cfg = _init_config_from_dict( + _base_config_dict( + customization={ + "profile_path": "tests/profiles/test/profile.py", + }, + shields=[ + { + "name": "topic-guard", + "provider_id": "question_validity", + "config": { + "model_id": "test-model", + "model_prompt": yaml_prompt, + "invalid_question_response": yaml_refusal, + }, + }, + ], + ) + ) + + assert cfg.shields[0].config.model_prompt == yaml_prompt # type: ignore[union-attr] + assert ( + cfg.shields[0].config.invalid_question_response # type: ignore[union-attr] + == yaml_refusal + ) + + +def test_qv_shield_explicit_prompt_without_message_placeholder_still_loads() -> None: + """Explicit YAML prompts without $message remain valid (additive; no hard-fail).""" + explicit_prompt = "Classify whether the question is about OpenShift." + cfg = _init_config_from_dict( + _base_config_dict( + shields=[ + { + "name": "topic-guard", + "provider_id": "question_validity", + "config": { + "model_id": "test-model", + "model_prompt": explicit_prompt, + }, + }, + ], + ) + ) + + assert cfg.shields[0].config.model_prompt == explicit_prompt # type: ignore[union-attr] + + +def test_qv_shield_empty_string_is_not_replaced_by_defaults() -> None: + """Explicit empty-string YAML values are kept; only None is filled.""" + cfg = _init_config_from_dict( + _base_config_dict( + customization={ + "profile_path": "tests/profiles/test/profile.py", + }, + shields=[ + { + "name": "topic-guard", + "provider_id": "question_validity", + "config": { + "model_id": "test-model", + "model_prompt": "", + "invalid_question_response": "", + }, + }, + ], + ) + ) + + assert cfg.shields[0].config.model_prompt == "" # type: ignore[union-attr] + assert cfg.shields[0].config.invalid_question_response == "" # type: ignore[union-attr] + + +def test_qv_shield_partial_yaml_override_fills_only_omitted_fields() -> None: + """Each omitted field is filled independently when the other is set in YAML.""" + yaml_prompt = "YAML prompt only ${message}" + yaml_refusal = "YAML refusal only" + profile = CustomProfile(path="tests/profiles/test/profile.py") + cfg = _init_config_from_dict( + _base_config_dict( + customization={ + "profile_path": "tests/profiles/test/profile.py", + }, + shields=[ + { + "name": "prompt-only", + "provider_id": "question_validity", + "config": { + "model_id": "test-model", + "model_prompt": yaml_prompt, + }, + }, + { + "name": "refusal-only", + "provider_id": "question_validity", + "config": { + "model_id": "test-model-2", + "invalid_question_response": yaml_refusal, + }, + }, + ], + ) + ) + + assert cfg.shields[0].config.model_prompt == yaml_prompt # type: ignore[union-attr] + assert ( + cfg.shields[0].config.invalid_question_response # type: ignore[union-attr] + == profile.get_invalid_resp() + ) + assert ( + cfg.shields[1].config.model_prompt # type: ignore[union-attr] + == profile.get_validation() + ) + assert ( + cfg.shields[1].config.invalid_question_response # type: ignore[union-attr] + == yaml_refusal + ) + + +def test_qv_shield_profile_empty_strings_are_kept() -> None: + """Profile empty-string validation / invalid_resp are kept (not treated as missing).""" + cfg = _init_config_from_dict( + _base_config_dict( + customization={ + "profile_path": "tests/profiles/empty_qv_strings/profile.py", + }, + shields=[ + { + "name": "topic-guard", + "provider_id": "question_validity", + "config": {"model_id": "test-model"}, + }, + ], + ) + ) + + assert cfg.shields[0].config.model_prompt == "" # type: ignore[union-attr] + assert cfg.shields[0].config.invalid_question_response == "" # type: ignore[union-attr] + + +def test_qv_shield_profile_without_qv_keys_falls_back_to_defaults() -> None: + """Missing profile validation / invalid_resp keys fall through to LCORE defaults.""" + cfg = _init_config_from_dict( + _base_config_dict( + customization={ + "profile_path": "tests/profiles/no_qv_keys/profile.py", + }, + shields=[ + { + "name": "topic-guard", + "provider_id": "question_validity", + "config": {"model_id": "test-model"}, + }, + ], + ) + ) + + assert ( + cfg.shields[0].config.model_prompt # type: ignore[union-attr] + == constants.DEFAULT_MODEL_PROMPT + ) + assert ( + cfg.shields[0].config.invalid_question_response # type: ignore[union-attr] + == constants.DEFAULT_INVALID_QUESTION_RESPONSE + ) + + +def test_qv_shield_multiple_shields_resolve_independently() -> None: + """Each question_validity shield resolves its own omitted fields.""" + yaml_prompt = "Shield-A YAML ${message}" + profile = CustomProfile(path="tests/profiles/test/profile.py") + cfg = _init_config_from_dict( + _base_config_dict( + customization={ + "profile_path": "tests/profiles/test/profile.py", + }, + shields=[ + { + "name": "shield-a", + "provider_id": "question_validity", + "config": { + "model_id": "model-a", + "model_prompt": yaml_prompt, + }, + }, + { + "name": "shield-b", + "provider_id": "question_validity", + "config": {"model_id": "model-b"}, + }, + ], + ) + ) + + assert cfg.shields[0].config.model_prompt == yaml_prompt # type: ignore[union-attr] + assert ( + cfg.shields[0].config.invalid_question_response # type: ignore[union-attr] + == profile.get_invalid_resp() + ) + assert ( + cfg.shields[1].config.model_prompt # type: ignore[union-attr] + == profile.get_validation() + ) + assert ( + cfg.shields[1].config.invalid_question_response # type: ignore[union-attr] + == profile.get_invalid_resp() + ) + + def test_init_from_dict_with_mcp_servers() -> None: """Test initialization with MCP servers configuration.""" config_dict = { diff --git a/tests/unit/utils/test_prompts.py b/tests/unit/utils/test_prompts.py index 6b25f7055..d3ea6fb64 100644 --- a/tests/unit/utils/test_prompts.py +++ b/tests/unit/utils/test_prompts.py @@ -333,3 +333,22 @@ def test_get_topic_summary_system_prompt_no_customization( topic_summary_prompt = prompts.get_topic_summary_system_prompt() assert topic_summary_prompt == constants.DEFAULT_TOPIC_SUMMARY_SYSTEM_PROMPT + + +def test_custom_profile_loads_invalid_resp() -> None: + """CustomProfile loads query_responses.invalid_resp from the profile module.""" + custom_profile = CustomProfile(path="tests/profiles/test/profile.py") + invalid_resp = custom_profile.get_invalid_resp() + + assert invalid_resp is not None + assert "Red Hat Developer Hub" in invalid_resp + + +def test_custom_profile_loads_validation() -> None: + """CustomProfile loads system_prompts.validation from the profile module.""" + custom_profile = CustomProfile(path="tests/profiles/test/profile.py") + validation = custom_profile.get_validation() + + assert validation is not None + assert "${message}" in validation + assert validation == custom_profile.get_prompts().get("validation") diff --git a/tests/unit/utils/test_pydantic_ai.py b/tests/unit/utils/test_pydantic_ai.py index bc477ce8e..7b1a9cbd8 100644 --- a/tests/unit/utils/test_pydantic_ai.py +++ b/tests/unit/utils/test_pydantic_ai.py @@ -15,7 +15,6 @@ from configuration import AppConfig from models.common.responses.responses_api_params import ResponsesApiParams from models.config import ( - QuestionValidityConfig, QuestionValidityShieldConfiguration, RedactionConfig, RedactionShieldConfiguration, @@ -23,6 +22,7 @@ ) from pydantic_ai_lightspeed.capabilities import QuestionValidity from pydantic_ai_lightspeed.capabilities.redaction import PiiRedactionCapability +from tests.unit.qv_config import make_qv_config from utils.pydantic_ai_helpers import ( _agent_capabilities, _shield_capability, @@ -74,7 +74,7 @@ def test_question_validity_shield_builds_question_validity_capability( shield = QuestionValidityShieldConfiguration( name="topic-guard", provider_id="question_validity", - config=QuestionValidityConfig(model_id="test-model"), + config=make_qv_config(model_id="test-model"), ) capability = _shield_capability(shield) @@ -131,7 +131,7 @@ def test_returns_shield_capabilities_when_configured(self) -> None: QuestionValidityShieldConfiguration( name="topic-guard", provider_id="question_validity", - config=QuestionValidityConfig(model_id="test-model"), + config=make_qv_config(model_id="test-model"), ), RedactionShieldConfiguration( name="pii-guard", @@ -297,7 +297,7 @@ def test_agent_includes_shield_capabilities_when_configured( QuestionValidityShieldConfiguration( name="topic-guard", provider_id="question_validity", - config=QuestionValidityConfig(model_id="test-model"), + config=make_qv_config(model_id="test-model"), ), RedactionShieldConfiguration( name="pii-guard", @@ -366,7 +366,7 @@ def test_agent_filters_shields_by_name( QuestionValidityShieldConfiguration( name="topic-guard", provider_id="question_validity", - config=QuestionValidityConfig(model_id="test-model"), + config=make_qv_config(model_id="test-model"), ), RedactionShieldConfiguration( name="pii-guard", From c48a77652242730e1a875be524d3ca1046d9cf98 Mon Sep 17 00:00:00 2001 From: Jordan Dubrick Date: Thu, 6 Aug 2026 16:27:31 -0400 Subject: [PATCH 2/2] address coderabbit Signed-off-by: Jordan Dubrick --- docs/models/successful_responses.md | 4 ++-- tests/unit/app/endpoints/test_shields.py | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/models/successful_responses.md b/docs/models/successful_responses.md index 42e6b426a..77cd5cefe 100644 --- a/docs/models/successful_responses.md +++ b/docs/models/successful_responses.md @@ -573,8 +573,8 @@ Custom profile customization for prompts and validation. |-------|------|-------------| | path | string | Path to Python modules containing custom profile. | | prompts | object | Dictionary containing map of system prompts | -| validation | string | Read-only. Loaded from the custom profile Python module (PROFILE_CONFIG['system_prompts']['validation']), not from lightspeed-stack.yaml. Used when a question_validity shield omits model_prompt. | -| invalid_resp | string | Read-only. Loaded from the custom profile Python module (PROFILE_CONFIG['query_responses']['invalid_resp']), not from lightspeed-stack.yaml. Used when a question_validity shield omits invalid_question_response. | +| validation | string | Read-only. Loaded from the custom profile Python module (PROFILE_CONFIG["system_prompts"]["validation"]), not from lightspeed-stack.yaml. Used when a question_validity shield omits model_prompt. | +| invalid_resp | string | Read-only. Loaded from the custom profile Python module (PROFILE_CONFIG["query_responses"]["invalid_resp"]), not from lightspeed-stack.yaml. Used when a question_validity shield omits invalid_question_response. | ## Customization diff --git a/tests/unit/app/endpoints/test_shields.py b/tests/unit/app/endpoints/test_shields.py index e305a5210..648e0eae7 100644 --- a/tests/unit/app/endpoints/test_shields.py +++ b/tests/unit/app/endpoints/test_shields.py @@ -135,6 +135,9 @@ async def test_shields_endpoint_handler_configured_shields( assert response.shields[0].provider_id == "question_validity" assert response.shields[0].type == "shield" assert response.shields[0].config["model_id"] == "openai/gpt-4o-mini" + assert response.shields[0].config["model_prompt"] == ( + "Is this question valid? ${message}" + ) assert response.shields[1].name == "pii-redaction" assert response.shields[1].provider_id == "redaction" assert response.shields[1].type == "shield"