From bf1d9c405587c1b1ddddce45b6a36c6f6624b646 Mon Sep 17 00:00:00 2001 From: Victor Valbuena Date: Thu, 23 Jul 2026 15:06:52 -0700 Subject: [PATCH 1/2] FEAT: Chain-of-Thought Rendering for attack results and scenario results. --- pyrit/output/attack_result/base.py | 27 ++- pyrit/output/attack_result/markdown.py | 82 +++++++- pyrit/output/attack_result/pretty.py | 68 ++++++- pyrit/output/conversation/base.py | 122 ++++++++++++ pyrit/output/conversation/markdown.py | 81 ++++++-- pyrit/output/conversation/pretty.py | 55 +++--- pyrit/output/helpers.py | 10 +- pyrit/output/scenario_result/base.py | 8 +- pyrit/output/scenario_result/pretty.py | 92 ++++++++- .../output/attack_result/test_markdown.py | 86 ++++++++ .../unit/output/attack_result/test_pretty.py | 61 ++++-- tests/unit/output/conftest.py | 55 ++++++ .../output/conversation/test_reasoning.py | 187 ++++++++++++++++++ .../output/scenario_result/test_pretty.py | 36 ++++ tests/unit/output/test_helpers.py | 28 ++- 15 files changed, 911 insertions(+), 87 deletions(-) create mode 100644 tests/unit/output/conftest.py create mode 100644 tests/unit/output/conversation/test_reasoning.py diff --git a/pyrit/output/attack_result/base.py b/pyrit/output/attack_result/base.py index 4bbe463f7b..194fac676a 100644 --- a/pyrit/output/attack_result/base.py +++ b/pyrit/output/attack_result/base.py @@ -3,7 +3,7 @@ from abc import abstractmethod -from pyrit.models import AttackOutcome, Message, Score +from pyrit.models import AttackOutcome, AttackResult, Message, Score from pyrit.output.base import PrinterBase @@ -18,6 +18,31 @@ class AttackResultPrinterBase(PrinterBase): Thin-client implementations can fetch data via REST endpoints. """ + @abstractmethod + async def render_async( + self, + result: AttackResult, + *, + include_auxiliary_scores: bool = False, + include_pruned_conversations: bool = False, + include_adversarial_conversation: bool = False, + include_reasoning_trace: bool = False, + ) -> str: + """ + Render an attack result. + + Args: + result (AttackResult): The attack result to render. + include_auxiliary_scores (bool): Whether to include auxiliary scores. Defaults to False. + include_pruned_conversations (bool): Whether to include pruned conversations. Defaults to False. + include_adversarial_conversation (bool): Whether to include the adversarial conversation. + Defaults to False. + include_reasoning_trace (bool): Whether to include reasoning traces. Defaults to False. + + Returns: + str: The rendered attack result. + """ + @abstractmethod async def _get_conversation_async(self, conversation_id: str) -> list[Message]: """ diff --git a/pyrit/output/attack_result/markdown.py b/pyrit/output/attack_result/markdown.py index ccb7d93d04..4aded2d668 100644 --- a/pyrit/output/attack_result/markdown.py +++ b/pyrit/output/attack_result/markdown.py @@ -67,6 +67,7 @@ async def render_async( include_auxiliary_scores: bool = False, include_pruned_conversations: bool = False, include_adversarial_conversation: bool = False, + include_reasoning_trace: bool = False, ) -> str: """ Render the complete attack result as markdown and return it as a string. @@ -77,6 +78,7 @@ async def render_async( include_pruned_conversations (bool): Whether to include pruned conversations. Defaults to False. include_adversarial_conversation (bool): Whether to include the adversarial conversation. Defaults to False. + include_reasoning_trace (bool): Whether to include the reasoning trace. Defaults to False. Returns: str: The rendered markdown text. @@ -93,17 +95,25 @@ async def render_async( markdown_lines.append("\n## Conversation History\n") conversation_lines = await self._get_conversation_markdown_async( - result=result, include_scores=include_auxiliary_scores + result=result, + include_scores=include_auxiliary_scores, + include_reasoning_trace=include_reasoning_trace, ) markdown_lines.extend(conversation_lines) if include_pruned_conversations: - pruned_lines = await self._get_pruned_conversations_markdown_async(result) + pruned_lines = await self._get_pruned_conversations_markdown_async( + result, + include_reasoning_trace=include_reasoning_trace, + ) if pruned_lines: markdown_lines.extend(pruned_lines) if include_adversarial_conversation: - adversarial_lines = await self._get_adversarial_conversation_markdown_async(result) + adversarial_lines = await self._get_adversarial_conversation_markdown_async( + result, + include_reasoning_trace=include_reasoning_trace, + ) if adversarial_lines: markdown_lines.extend(adversarial_lines) @@ -123,7 +133,11 @@ async def render_async( return "\n".join(markdown_lines) async def _get_conversation_markdown_async( - self, *, result: AttackResult, include_scores: bool = False + self, + *, + result: AttackResult, + include_scores: bool = False, + include_reasoning_trace: bool = False, ) -> list[str]: """ Generate markdown lines for the conversation history. @@ -131,6 +145,7 @@ async def _get_conversation_markdown_async( Args: result (AttackResult): The attack result containing the conversation ID. include_scores (bool): Whether to include scores. Defaults to False. + include_reasoning_trace (bool): Whether to include reasoning traces. Defaults to False. Returns: list[str]: Markdown strings for the conversation. @@ -143,7 +158,11 @@ async def _get_conversation_markdown_async( if not messages: return [f"*No conversation found for ID: {result.conversation_id}*\n"] - rendered = await self._conversation_printer.render_async(messages, include_scores=include_scores) + rendered = await self._conversation_printer.render_async( + messages, + include_scores=include_scores, + include_reasoning_trace=include_reasoning_trace, + ) return [rendered] async def _get_summary_markdown_async(self, result: AttackResult) -> list[str]: @@ -189,12 +208,18 @@ async def _get_summary_markdown_async(self, result: AttackResult) -> list[str]: return markdown_lines - async def _get_pruned_conversations_markdown_async(self, result: AttackResult) -> list[str]: + async def _get_pruned_conversations_markdown_async( + self, + result: AttackResult, + *, + include_reasoning_trace: bool = False, + ) -> list[str]: """ Generate markdown lines for pruned conversations. Args: result (AttackResult): The attack result containing related conversations. + include_reasoning_trace (bool): Whether to include reasoning traces. Defaults to False. Returns: list[str]: Markdown strings for pruned conversations. @@ -221,11 +246,26 @@ async def _get_pruned_conversations_markdown_async(self, result: AttackResult) - continue last_message = messages[-1] + pieces = self._conversation_printer._get_renderable_pieces( + message=last_message, + include_reasoning_trace=include_reasoning_trace, + ) + if not pieces: + continue + role_label = last_message.api_role.upper() markdown_lines.append(f"**Last Message ({role_label}):**\n") - for piece in last_message.message_pieces: + for piece in pieces: + if self._conversation_printer._is_reasoning_piece(piece=piece): + markdown_lines.extend( + self._conversation_printer._format_reasoning_summary( + self._conversation_printer._get_reasoning_value(piece=piece) + ) + ) + continue + content = piece.converted_value or "" if "\n" in content: markdown_lines.append("```") @@ -241,12 +281,18 @@ async def _get_pruned_conversations_markdown_async(self, result: AttackResult) - return markdown_lines - async def _get_adversarial_conversation_markdown_async(self, result: AttackResult) -> list[str]: + async def _get_adversarial_conversation_markdown_async( + self, + result: AttackResult, + *, + include_reasoning_trace: bool = False, + ) -> list[str]: """ Generate markdown lines for the adversarial conversation. Args: result (AttackResult): The attack result containing related conversations. + include_reasoning_trace (bool): Whether to include reasoning traces. Defaults to False. Returns: list[str]: Markdown strings for the adversarial conversation. @@ -278,6 +324,13 @@ async def _get_adversarial_conversation_markdown_async(self, result: AttackResul turn_number = 0 for message in messages: + pieces = self._conversation_printer._get_renderable_pieces( + message=message, + include_reasoning_trace=include_reasoning_trace, + ) + if not pieces: + continue + if message.api_role == "user": turn_number += 1 markdown_lines.append(f"\n#### Turn {turn_number} - USER\n") @@ -286,7 +339,15 @@ async def _get_adversarial_conversation_markdown_async(self, result: AttackResul else: markdown_lines.append(f"\n#### {message.api_role.upper()}\n") - for piece in message.message_pieces: + for piece in pieces: + if self._conversation_printer._is_reasoning_piece(piece=piece): + markdown_lines.extend( + self._conversation_printer._format_reasoning_summary( + self._conversation_printer._get_reasoning_value(piece=piece) + ) + ) + continue + content = piece.converted_value or "" if len(content) > 200 or "\n" in content: markdown_lines.append("```") @@ -355,6 +416,7 @@ async def render_async( include_auxiliary_scores: bool = False, include_pruned_conversations: bool = False, include_adversarial_conversation: bool = False, + include_reasoning_trace: bool = False, ) -> str: """ Render the complete attack result as markdown and return it as a string. @@ -365,6 +427,7 @@ async def render_async( include_pruned_conversations (bool): Whether to include pruned conversations. Defaults to False. include_adversarial_conversation (bool): Whether to include the adversarial conversation. Defaults to False. + include_reasoning_trace (bool): Whether to include the reasoning trace. Defaults to False. Returns: str: The rendered markdown text. @@ -374,6 +437,7 @@ async def render_async( include_auxiliary_scores=include_auxiliary_scores, include_pruned_conversations=include_pruned_conversations, include_adversarial_conversation=include_adversarial_conversation, + include_reasoning_trace=include_reasoning_trace, ) async def _get_conversation_async(self, conversation_id: str) -> list[Message]: diff --git a/pyrit/output/attack_result/pretty.py b/pyrit/output/attack_result/pretty.py index 304d60becf..e28cf3a47e 100644 --- a/pyrit/output/attack_result/pretty.py +++ b/pyrit/output/attack_result/pretty.py @@ -91,6 +91,7 @@ async def render_async( include_auxiliary_scores: bool = False, include_pruned_conversations: bool = False, include_adversarial_conversation: bool = False, + include_reasoning_trace: bool = False, ) -> str: """ Render the complete attack result and return it as a string. @@ -101,6 +102,7 @@ async def render_async( include_pruned_conversations (bool): Whether to include pruned conversations. Defaults to False. include_adversarial_conversation (bool): Whether to include the adversarial conversation. Defaults to False. + include_reasoning_trace (bool): Whether to include the reasoning trace. Defaults to False. Returns: str: The rendered attack result text. @@ -109,11 +111,27 @@ async def render_async( lines.append(self._render_header(result)) lines.append(await self._render_summary_async(result)) lines.append(self._render_section_header("Conversation History with Objective Target")) - lines.append(await self._render_conversation_async(result, include_scores=include_auxiliary_scores)) + lines.append( + await self._render_conversation_async( + result, + include_scores=include_auxiliary_scores, + include_reasoning_trace=include_reasoning_trace, + ) + ) if include_pruned_conversations: - lines.append(await self._render_pruned_conversations_async(result)) + lines.append( + await self._render_pruned_conversations_async( + result, + include_reasoning_trace=include_reasoning_trace, + ) + ) if include_adversarial_conversation: - lines.append(await self._render_adversarial_conversation_async(result)) + lines.append( + await self._render_adversarial_conversation_async( + result, + include_reasoning_trace=include_reasoning_trace, + ) + ) if result.metadata: lines.append(self._render_metadata(result.metadata)) lines.append(self._render_footer()) @@ -270,12 +288,18 @@ def _render_metadata(self, metadata: dict[str, Any]) -> str: lines.append(self._format_colored(f"{self._indent}• {key}: {value}", Fore.CYAN)) return "".join(lines) - async def _render_pruned_conversations_async(self, result: AttackResult) -> str: + async def _render_pruned_conversations_async( + self, + result: AttackResult, + *, + include_reasoning_trace: bool = False, + ) -> str: """ Render pruned conversations showing only the last message and score for each. Args: result (AttackResult): The attack result containing related conversations. + include_reasoning_trace (bool): Whether to include reasoning traces. Defaults to False. Returns: str: The rendered pruned conversations text. @@ -308,10 +332,25 @@ async def _render_pruned_conversations_async(self, result: AttackResult) -> str: continue last_message = messages[-1] + pieces = self._conversation_printer._get_renderable_pieces( + message=last_message, + include_reasoning_trace=include_reasoning_trace, + ) + if not pieces: + continue + role_label = last_message.api_role.upper() lines.append(self._format_colored(f"{self._indent}Last Message ({role_label}):", Style.BRIGHT, Fore.WHITE)) - for piece in last_message.message_pieces: + for piece in pieces: + if self._conversation_printer._is_reasoning_piece(piece=piece): + rendered = self._conversation_printer._render_reasoning_summary( + self._conversation_printer._get_reasoning_value(piece=piece) + ) + if rendered: + lines.append(rendered) + continue + lines.append(self._conversation_printer._render_wrapped_text(piece.converted_value, Fore.WHITE)) scores = await self._get_scores_async(prompt_ids=[str(piece.id)]) @@ -324,12 +363,18 @@ async def _render_pruned_conversations_async(self, result: AttackResult) -> str: lines.append(self._format_colored("─" * self._width, Fore.RED)) return "".join(lines) - async def _render_adversarial_conversation_async(self, result: AttackResult) -> str: + async def _render_adversarial_conversation_async( + self, + result: AttackResult, + *, + include_reasoning_trace: bool = False, + ) -> str: """ Render the adversarial conversation for the best-scoring attack branch. Args: result (AttackResult): The attack result containing related conversations. + include_reasoning_trace (bool): Whether to include reasoning traces. Defaults to False. Returns: str: The rendered adversarial conversation text. @@ -368,7 +413,13 @@ async def _render_adversarial_conversation_async(self, result: AttackResult) -> ) continue - lines.append(await self._conversation_printer.render_async(messages, include_scores=False)) + lines.append( + await self._conversation_printer.render_async( + messages, + include_scores=False, + include_reasoning_trace=include_reasoning_trace, + ) + ) return "".join(lines) @@ -452,6 +503,7 @@ async def render_async( include_auxiliary_scores: bool = False, include_pruned_conversations: bool = False, include_adversarial_conversation: bool = False, + include_reasoning_trace: bool = False, ) -> str: """ Render the complete attack result and return it as a string. @@ -462,6 +514,7 @@ async def render_async( include_pruned_conversations (bool): Whether to include pruned conversations. Defaults to False. include_adversarial_conversation (bool): Whether to include the adversarial conversation. Defaults to False. + include_reasoning_trace (bool): Whether to include the reasoning traces. Defaults to False. Returns: str: The rendered attack result text. @@ -471,6 +524,7 @@ async def render_async( include_auxiliary_scores=include_auxiliary_scores, include_pruned_conversations=include_pruned_conversations, include_adversarial_conversation=include_adversarial_conversation, + include_reasoning_trace=include_reasoning_trace, ) async def _get_conversation_async(self, conversation_id: str) -> list[Message]: diff --git a/pyrit/output/conversation/base.py b/pyrit/output/conversation/base.py index 1c9b40b392..3fe8e7d5e6 100644 --- a/pyrit/output/conversation/base.py +++ b/pyrit/output/conversation/base.py @@ -1,7 +1,9 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. +import json from abc import abstractmethod +from typing import ClassVar from pyrit.models import Message, MessagePiece, Score from pyrit.output.base import PrinterBase @@ -15,6 +17,126 @@ class ConversationPrinterBase(PrinterBase): ``_display_image_async``) and rendering via ``render_async``. """ + _REASONING_STATUSES: ClassVar[frozenset[str]] = frozenset({"in_progress", "completed", "incomplete"}) + + @staticmethod + def _is_reasoning_piece(*, piece: MessagePiece) -> bool: + """ + Check whether either stored representation marks the piece as reasoning. + + Returns: + bool: True when the original or converted data type is reasoning. + """ + return piece.original_value_data_type == "reasoning" or piece.converted_value_data_type == "reasoning" + + @classmethod + def _get_reasoning_value(cls, *, piece: MessagePiece) -> str: + """ + Return the value associated with the reasoning-typed representation. + + Args: + piece (MessagePiece): The reasoning piece whose serialized value should be returned. + + Returns: + str: The converted value when it remains reasoning, otherwise the original value. + + Raises: + ValueError: If neither representation is reasoning. + """ + if piece.converted_value_data_type == "reasoning": + return piece.converted_value + if piece.original_value_data_type == "reasoning": + return piece.original_value + raise ValueError("Message piece is not a reasoning piece.") + + @staticmethod + def _get_renderable_pieces( + *, + message: Message, + include_reasoning_trace: bool, + ) -> list[MessagePiece]: + """ + Return message pieces visible under the selected reasoning policy. + + Args: + message (Message): The message whose pieces should be filtered. + include_reasoning_trace (bool): Whether reasoning pieces should remain visible. + + Returns: + list[MessagePiece]: The pieces that should be rendered. + """ + return [ + piece + for piece in message.message_pieces + if include_reasoning_trace or not ConversationPrinterBase._is_reasoning_piece(piece=piece) + ] + + @classmethod + def _extract_reasoning_summary(cls, reasoning_value: str) -> str: + """ + Extract a provider-visible reasoning summary from an OpenAI Responses item. + + The expected value is the JSON serialization of an OpenAI Responses + ``reasoning`` output item. The returned text is a provider-generated + summary, not raw model chain-of-thought. + + Args: + reasoning_value (str): Serialized OpenAI Responses reasoning item. + + Returns: + str: The concatenated summary text. An empty summary list produces an empty string. + + Raises: + ValueError: If the value is not valid JSON or does not match the expected + OpenAI Responses reasoning-summary shape. + """ + try: + data = json.loads(reasoning_value) + except (json.JSONDecodeError, TypeError) as exc: + raise ValueError("Reasoning pieces must contain a valid JSON object.") from exc + + if not isinstance(data, dict) or data.get("type") != "reasoning": + raise ValueError("Reasoning pieces must contain an OpenAI Responses item with type='reasoning'.") + + if not isinstance(data.get("id"), str): + raise ValueError("Reasoning pieces must contain a string 'id'.") + + summary = data.get("summary") + if not isinstance(summary, list): + raise ValueError("Reasoning pieces must contain a 'summary' list.") + + parts: list[str] = [] + for item in summary: + if ( + not isinstance(item, dict) + or item.get("type") != "summary_text" + or not isinstance(item.get("text"), str) + ): + raise ValueError("Each reasoning summary item must have type='summary_text' and a string 'text'.") + parts.append(item["text"]) + + status = data.get("status") + if status is not None and status not in cls._REASONING_STATUSES: + raise ValueError("Reasoning piece 'status' must be in_progress, completed, incomplete, or null.") + + encrypted_content = data.get("encrypted_content") + if encrypted_content is not None and not isinstance(encrypted_content, str): + raise ValueError("Reasoning piece 'encrypted_content' must be a string or null.") + + content = data.get("content") + if content is not None: + if not isinstance(content, list): + raise ValueError("Reasoning piece 'content' must be a list or null.") + for item in content: + if ( + not isinstance(item, dict) + or item.get("type") != "reasoning_text" + or not isinstance(item.get("text"), str) + ): + raise ValueError("Each reasoning content item must have type='reasoning_text' and a string 'text'.") + + return "\n".join(parts) + @abstractmethod async def _get_scores_async(self, *, prompt_ids: list[str]) -> list[Score]: """ diff --git a/pyrit/output/conversation/markdown.py b/pyrit/output/conversation/markdown.py index 7147c23981..c89cdee9d1 100644 --- a/pyrit/output/conversation/markdown.py +++ b/pyrit/output/conversation/markdown.py @@ -70,7 +70,7 @@ async def render_async( Args: messages (list[Message]): The messages to render. include_scores (bool): Whether to include scores. Defaults to False. - include_reasoning_trace (bool): Accepted for interface compatibility. Unused. + include_reasoning_trace (bool): Whether to include reasoning traces. Defaults to False. Returns: str: The rendered conversation markdown text. @@ -82,44 +82,59 @@ async def render_async( turn_number = 0 for message in messages: - if not message.message_pieces: + pieces = self._get_renderable_pieces( + message=message, + include_reasoning_trace=include_reasoning_trace, + ) + if not pieces: continue message_role = message.get_piece().api_role if message_role == "system": - markdown_lines.extend(self._format_system_message(message)) + markdown_lines.extend(await self._format_system_message_async(pieces=pieces)) elif message_role == "user": turn_number += 1 - markdown_lines.extend(await self._format_user_message_async(message=message, turn_number=turn_number)) + markdown_lines.extend( + await self._format_user_message_async( + pieces=pieces, + turn_number=turn_number, + ) + ) else: - markdown_lines.extend(await self._format_assistant_message_async(message=message)) + markdown_lines.extend(await self._format_assistant_message_async(pieces=pieces)) if include_scores: - markdown_lines.extend(await self._format_message_scores_async(message)) + markdown_lines.extend(await self._format_message_scores_async(pieces=pieces)) return "\n".join(markdown_lines) - def _format_system_message(self, message: Message) -> list[str]: + async def _format_system_message_async(self, *, pieces: list[MessagePiece]) -> list[str]: """ Format a system message as markdown. Args: - message (Message): The system message to format. + pieces (list[MessagePiece]): The filtered system-message pieces to format. Returns: list[str]: Markdown strings for the system message. """ lines = ["\n### System Message\n"] - lines.extend(f"{piece.converted_value}\n" for piece in message.message_pieces) + for piece in pieces: + lines.extend(await self._format_piece_content_async(piece=piece, show_original=False)) return lines - async def _format_user_message_async(self, *, message: Message, turn_number: int) -> list[str]: + async def _format_user_message_async( + self, + *, + pieces: list[MessagePiece], + turn_number: int, + ) -> list[str]: """ Format a user message as markdown with turn numbering. Args: - message (Message): The user message to format. + pieces (list[MessagePiece]): The filtered user-message pieces to format. turn_number (int): The conversation turn number. Returns: @@ -127,28 +142,28 @@ async def _format_user_message_async(self, *, message: Message, turn_number: int """ lines = [f"\n### Turn {turn_number}\n", "#### User\n"] - for piece in message.message_pieces: + for piece in pieces: lines.extend(await self._format_piece_content_async(piece=piece, show_original=True)) return lines - async def _format_assistant_message_async(self, *, message: Message) -> list[str]: + async def _format_assistant_message_async(self, *, pieces: list[MessagePiece]) -> list[str]: """ Format an assistant response message as markdown. Args: - message (Message): The response message to format. + pieces (list[MessagePiece]): The filtered assistant-message pieces to format. Returns: list[str]: Markdown strings for the response message. """ lines: list[str] = [] - piece = message.message_pieces[0] + piece = pieces[0] role_name = "Assistant (Simulated)" if piece.is_simulated else piece.api_role.capitalize() lines.append(f"\n#### {role_name}\n") - for piece in message.message_pieces: + for piece in pieces: lines.extend(await self._format_piece_content_async(piece=piece, show_original=False)) return lines @@ -164,6 +179,8 @@ async def _format_piece_content_async(self, *, piece: MessagePiece, show_origina Returns: list[str]: Markdown lines for this piece. """ + if self._is_reasoning_piece(piece=piece): + return self._format_reasoning_summary(self._get_reasoning_value(piece=piece)) if piece.converted_value_data_type == "image_path": return self._format_image_content(image_path=piece.converted_value) if piece.converted_value_data_type == "audio_path": @@ -172,6 +189,30 @@ async def _format_piece_content_async(self, *, piece: MessagePiece, show_origina return self._format_error_content(piece=piece) return self._format_text_content(piece=piece, show_original=show_original) + def _format_reasoning_summary(self, reasoning_value: str) -> list[str]: + """ + Format a provider-generated reasoning summary as Markdown. + + Args: + reasoning_value (str): Serialized OpenAI Responses reasoning item. + + Returns: + list[str]: A single tagged Markdown block, or an empty list for an empty summary. + """ + summary = self._extract_reasoning_summary(reasoning_value) + if not summary: + return [] + + # PyRIT adds these visible tags. They label a provider-generated summary + # and do not represent raw model chain-of-thought. + block_lines = [ + r"\", + "> **Provider-generated reasoning summary (not raw chain-of-thought)**", + *(f"> {line}" if line else ">" for line in summary.splitlines()), + r"\", + ] + return ["\n".join(block_lines) + "\n"] + def _format_text_content(self, *, piece: MessagePiece, show_original: bool) -> list[str]: """ Format regular text content. @@ -358,18 +399,18 @@ def _format_error_content(self, *, piece: MessagePiece) -> list[str]: lines.append("```\n") return lines - async def _format_message_scores_async(self, message: Message) -> list[str]: + async def _format_message_scores_async(self, *, pieces: list[MessagePiece]) -> list[str]: """ Format scores for all pieces in a message as markdown. Args: - message (Message): The message containing pieces to format scores for. + pieces (list[MessagePiece]): The filtered pieces whose scores should be formatted. Returns: list[str]: Markdown strings for the scores. """ lines: list[str] = [] - for piece in message.message_pieces: + for piece in pieces: scores = await self._get_scores_async(prompt_ids=[str(piece.id)]) if scores: lines.append("\n##### Scores\n") @@ -432,7 +473,7 @@ async def render_async( Args: messages (list[Message]): The messages to render. include_scores (bool): Whether to include scores. Defaults to False. - include_reasoning_trace (bool): Accepted for interface compatibility. Unused. + include_reasoning_trace (bool): Whether to include reasoning traces. Defaults to False. Returns: str: The rendered conversation markdown text. diff --git a/pyrit/output/conversation/pretty.py b/pyrit/output/conversation/pretty.py index 10aff03afb..08855ae2ef 100644 --- a/pyrit/output/conversation/pretty.py +++ b/pyrit/output/conversation/pretty.py @@ -1,7 +1,6 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. -import json import logging import textwrap @@ -85,6 +84,13 @@ async def render_async( image_pieces: list[MessagePiece] = [] turn_number = 0 for message in messages: + pieces = self._get_renderable_pieces( + message=message, + include_reasoning_trace=include_reasoning_trace, + ) + if not pieces: + continue + if message.api_role == "user": turn_number += 1 lines.append("\n") @@ -103,16 +109,11 @@ async def render_async( lines.append(self._format_colored(f"🔸 {role_label}", Style.BRIGHT, Fore.YELLOW)) lines.append(self._format_colored("─" * self._width, Fore.YELLOW)) - for piece in message.message_pieces: - if piece.original_value_data_type == "reasoning": - if include_reasoning_trace: - summary_text = self._extract_reasoning_summary(piece.original_value) - if summary_text: - lines.append( - self._format_colored(f"{self._indent}💭 Reasoning Summary:", Style.DIM, Fore.CYAN) - ) - lines.append(self._render_wrapped_text(summary_text, Fore.CYAN)) - lines.append("\n") + for piece in pieces: + if self._is_reasoning_piece(piece=piece): + rendered = self._render_reasoning_summary(self._get_reasoning_value(piece=piece)) + if rendered: + lines.append(rendered) continue if piece.is_blocked(): @@ -218,28 +219,32 @@ def _render_wrapped_text(self, text: str, color: str) -> str: return "".join(lines) - @staticmethod - def _extract_reasoning_summary(reasoning_value: str) -> str: + def _render_reasoning_summary(self, reasoning_value: str) -> str: """ - Extract human-readable summary text from a reasoning piece's JSON value. + Render a provider-generated reasoning summary in subdued gray. Args: - reasoning_value (str): The JSON string stored in the reasoning piece. + reasoning_value (str): Serialized OpenAI Responses reasoning item. Returns: - str: The concatenated summary text, or empty string if no summary is present. + str: The tagged reasoning block, or an empty string when the summary is empty. """ - try: - data = json.loads(reasoning_value) - except (json.JSONDecodeError, TypeError): + summary = self._extract_reasoning_summary(reasoning_value) + if not summary: return "" - summary = data.get("summary") if isinstance(data, dict) else None - if not summary or not isinstance(summary, list): - return "" - - parts = [item.get("text", "") for item in summary if isinstance(item, dict) and item.get("text")] - return "\n".join(parts) + # PyRIT adds these presentation tags. They label a provider-generated + # summary and do not represent raw model chain-of-thought. + label = "Provider-generated reasoning summary (not raw chain-of-thought)" + return "".join( + [ + self._format_colored(f"{self._indent}", Fore.LIGHTBLACK_EX), + self._format_colored(f"{self._indent}{label}", Style.DIM, Fore.LIGHTBLACK_EX), + self._render_wrapped_text(summary, Fore.LIGHTBLACK_EX), + self._format_colored(f"{self._indent}", Fore.LIGHTBLACK_EX), + self._format_colored("", Fore.LIGHTBLACK_EX), + ] + ) class PrettyConversationMemoryPrinter(PrettyConversationPrinter): diff --git a/pyrit/output/helpers.py b/pyrit/output/helpers.py index c1a4b21b15..13f1ba234c 100644 --- a/pyrit/output/helpers.py +++ b/pyrit/output/helpers.py @@ -29,6 +29,7 @@ async def output_attack_async( include_auxiliary_scores: bool = False, include_pruned_conversations: bool = False, include_adversarial_conversation: bool = False, + include_reasoning_trace: bool = False, blur_images: bool = False, blur_radius: int = 20, blurred_dir: str | os.PathLike[str] | None = None, @@ -45,6 +46,7 @@ async def output_attack_async( include_pruned_conversations (bool): Whether to include pruned conversations. Defaults to False. include_adversarial_conversation (bool): Whether to include the adversarial conversation. Defaults to False. + include_reasoning_trace (bool): Whether to include the reasoning traces. Defaults to False. blur_images (bool): If True, apply a Gaussian blur to image outputs before rendering them. For "pretty" output, image bytes are blurred in-memory before display. For "markdown" output, a blurred file is written to disk and the @@ -80,6 +82,7 @@ async def output_attack_async( include_auxiliary_scores=include_auxiliary_scores, include_pruned_conversations=include_pruned_conversations, include_adversarial_conversation=include_adversarial_conversation, + include_reasoning_trace=include_reasoning_trace, ) @@ -89,6 +92,7 @@ async def output_scenario_async( format: OutputFormat = "pretty", # noqa: A002 sink: Sink | None = None, sort_groups_by_success_rate: bool = False, + include_reasoning_trace: bool = False, ) -> None: """ Print a scenario result in the specified format to the specified destination. @@ -100,6 +104,7 @@ async def output_scenario_async( sort_groups_by_success_rate (bool): When True, the Per-Group Breakdown is sorted so that the group with the highest success rate appears first. Defaults to False, which preserves the original insertion order. + include_reasoning_trace (bool): Whether to include reasoning traces. Defaults to False. Raises: ValueError: If ``format`` is not a supported value. @@ -111,7 +116,10 @@ async def output_scenario_async( sink=sink or get_default_sink(StdoutSink), sort_groups_by_success_rate=sort_groups_by_success_rate, ) - await printer.write_async(result) + await printer.write_async( + result, + include_reasoning_trace=include_reasoning_trace, + ) async def output_scorer_async( diff --git a/pyrit/output/scenario_result/base.py b/pyrit/output/scenario_result/base.py index 1ad2422ef7..a170f1b657 100644 --- a/pyrit/output/scenario_result/base.py +++ b/pyrit/output/scenario_result/base.py @@ -16,12 +16,18 @@ class ScenarioResultPrinterBase(PrinterBase): """ @abstractmethod - async def render_async(self, result: ScenarioResult) -> str: + async def render_async( + self, + result: ScenarioResult, + *, + include_reasoning_trace: bool = False, + ) -> str: """ Render a scenario result summary and return it as a string. Args: result (ScenarioResult): The scenario result to summarize. + include_reasoning_trace (bool): Whether to include reasoning traces. Defaults to False. Returns: str: The rendered scenario result text. diff --git a/pyrit/output/scenario_result/pretty.py b/pyrit/output/scenario_result/pretty.py index 0584643ea5..d86a11ae6c 100644 --- a/pyrit/output/scenario_result/pretty.py +++ b/pyrit/output/scenario_result/pretty.py @@ -1,14 +1,20 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. +from __future__ import annotations + import textwrap +from typing import TYPE_CHECKING from colorama import Fore, Style from pyrit.models import AttackOutcome, ScenarioResult from pyrit.output.scenario_result.base import ScenarioResultPrinterBase -from pyrit.output.scorer.base import ScorerPrinterBase -from pyrit.output.sink import Sink + +if TYPE_CHECKING: + from pyrit.output.attack_result.pretty import PrettyAttackResultPrinter + from pyrit.output.scorer.base import ScorerPrinterBase + from pyrit.output.sink import Sink class PrettyScenarioResultPrinter(ScenarioResultPrinterBase): @@ -27,6 +33,7 @@ def __init__( indent_size: int = 2, enable_colors: bool = True, scorer_printer: ScorerPrinterBase | None = None, + attack_result_printer: PrettyAttackResultPrinter | None = None, sort_groups_by_success_rate: bool = False, ) -> None: """ @@ -39,6 +46,8 @@ def __init__( enable_colors (bool): Whether to enable ANSI color output. Defaults to True. scorer_printer (ScorerPrinterBase | None): Scorer printer for rendering scorer information. Defaults to None; leaf classes should provide a default. + attack_result_printer (PrettyAttackResultPrinter | None): Attack-result printer used + to render reasoning traces from contained attacks. Defaults to None. sort_groups_by_success_rate (bool): When True, the Per-Group Breakdown is sorted so that the group with the highest success rate appears first. Groups that tie on success rate retain their original relative order. Defaults to False, which @@ -49,6 +58,7 @@ def __init__( self._indent = " " * indent_size self._enable_colors = enable_colors self._scorer_printer = scorer_printer + self._attack_result_printer = attack_result_printer self._sort_groups_by_success_rate = sort_groups_by_success_rate def _format_colored(self, text: str, *colors: str) -> str: @@ -132,12 +142,18 @@ def _get_rate_color(self, rate: int) -> str: return str(Fore.CYAN) return str(Fore.GREEN) - async def render_async(self, result: ScenarioResult) -> str: + async def render_async( + self, + result: ScenarioResult, + *, + include_reasoning_trace: bool = False, + ) -> str: """ Render the scenario result summary and return it as a string. Args: result (ScenarioResult): The scenario result to summarize. + include_reasoning_trace (bool): Whether to include reasoning traces. Defaults to False. Returns: str: The rendered scenario result text. @@ -235,9 +251,57 @@ async def render_async(self, result: ScenarioResult) -> str: lines.append(self._render_footer()) parts.append("".join(lines)) + parts.append( + await self._render_attack_results_async( + result=result, + include_reasoning_trace=include_reasoning_trace, + ) + ) return "".join(parts) + async def _render_attack_results_async( + self, + *, + result: ScenarioResult, + include_reasoning_trace: bool, + ) -> str: + """ + Render attack results when scenario reasoning output is requested. + + Args: + result (ScenarioResult): The scenario result containing attacks. + include_reasoning_trace (bool): Whether attack reasoning traces should be rendered. + + Returns: + str: Rendered attack-result details, or an empty string when disabled. + + Raises: + ValueError: If reasoning is requested without an attack-result printer. + """ + if not include_reasoning_trace: + return "" + if self._attack_result_printer is None: + raise ValueError("attack_result_printer is required when reasoning traces are requested") + + lines = [self._render_section_header("Attack Reasoning Summaries")] + for group_name, attack_results in result.get_display_groups().items(): + for index, attack_result in enumerate(attack_results, 1): + lines.append( + self._format_colored( + f"{self._indent}{group_name} #{index}", + Style.BRIGHT, + Fore.CYAN, + ) + ) + lines.append( + await self._attack_result_printer.render_async( + attack_result, + include_reasoning_trace=True, + ) + ) + return "".join(lines) + class PrettyScenarioResultMemoryPrinter(PrettyScenarioResultPrinter): """ @@ -267,11 +331,20 @@ def __init__( sort_groups_by_success_rate (bool): When True, the Per-Group Breakdown is sorted so that the group with the highest success rate appears first. Defaults to False. """ + from pyrit.output.attack_result.pretty import PrettyAttackResultMemoryPrinter + + attack_result_printer = PrettyAttackResultMemoryPrinter( + sink=sink, + width=width, + indent_size=indent_size, + enable_colors=enable_colors, + ) super().__init__( sink=sink, width=width, indent_size=indent_size, enable_colors=enable_colors, + attack_result_printer=attack_result_printer, sort_groups_by_success_rate=sort_groups_by_success_rate, ) from pyrit.output.scorer.pretty import PrettyScorerMemoryPrinter @@ -281,14 +354,23 @@ def __init__( ) self._scorer_printer = scorer_printer - async def render_async(self, result: ScenarioResult) -> str: + async def render_async( + self, + result: ScenarioResult, + *, + include_reasoning_trace: bool = False, + ) -> str: """ Render the scenario result summary and return it as a string. Args: result (ScenarioResult): The scenario result to summarize. + include_reasoning_trace (bool): Whether to include reasoning traces. Defaults to False. Returns: str: The rendered scenario result text. """ - return await super().render_async(result) + return await super().render_async( + result, + include_reasoning_trace=include_reasoning_trace, + ) diff --git a/tests/unit/output/attack_result/test_markdown.py b/tests/unit/output/attack_result/test_markdown.py index 99b6daea7b..3f88d7f5f4 100644 --- a/tests/unit/output/attack_result/test_markdown.py +++ b/tests/unit/output/attack_result/test_markdown.py @@ -374,3 +374,89 @@ async def test_write_async_adversarial_with_no_messages(printer, attack_result, async def test_write_async_include_adversarial_with_no_refs(printer, attack_result, capsys): await printer.write_async(attack_result, include_adversarial_conversation=True) assert "## Adversarial Conversation" not in capsys.readouterr().out + + +async def test_write_async_main_reasoning_uses_escaped_tags( + printer, + attack_result, + sqlite_instance, + reasoning_value, +): + piece = MessagePiece( + role="assistant", + original_value=reasoning_value, + converted_value=reasoning_value, + original_value_data_type="reasoning", + converted_value_data_type="reasoning", + ) + _seed_messages(sqlite_instance, "conv-main", [piece]) + + rendered = await printer.render_async(attack_result, include_reasoning_trace=True) + + assert r"\" in rendered + assert r"\" in rendered + assert "" not in rendered + + +async def test_write_async_pruned_reasoning_uses_escaped_tags( + printer, + attack_result, + sqlite_instance, + reasoning_value, +): + piece = MessagePiece( + role="assistant", + original_value=reasoning_value, + converted_value=reasoning_value, + original_value_data_type="reasoning", + converted_value_data_type="reasoning", + conversation_id="pruned-reasoning", + ) + sqlite_instance.add_message_to_memory(request=Message(message_pieces=[piece])) + attack_result.related_conversations.add( + ConversationReference( + conversation_id="pruned-reasoning", + conversation_type=ConversationType.PRUNED, + ) + ) + + rendered = await printer.render_async( + attack_result, + include_pruned_conversations=True, + include_reasoning_trace=True, + ) + + assert r"\" in rendered + assert "step one" in rendered + + +async def test_write_async_adversarial_reasoning_uses_escaped_tags( + printer, + attack_result, + sqlite_instance, + reasoning_value, +): + piece = MessagePiece( + role="assistant", + original_value=reasoning_value, + converted_value=reasoning_value, + original_value_data_type="reasoning", + converted_value_data_type="reasoning", + conversation_id="adversarial-reasoning", + ) + sqlite_instance.add_message_to_memory(request=Message(message_pieces=[piece])) + attack_result.related_conversations.add( + ConversationReference( + conversation_id="adversarial-reasoning", + conversation_type=ConversationType.ADVERSARIAL, + ) + ) + + rendered = await printer.render_async( + attack_result, + include_adversarial_conversation=True, + include_reasoning_trace=True, + ) + + assert r"\" in rendered + assert "step one" in rendered diff --git a/tests/unit/output/attack_result/test_pretty.py b/tests/unit/output/attack_result/test_pretty.py index 436c875122..4e5235d266 100644 --- a/tests/unit/output/attack_result/test_pretty.py +++ b/tests/unit/output/attack_result/test_pretty.py @@ -320,8 +320,9 @@ async def test_write_async_adversarial_with_no_messages(printer, attack_result, # --- reasoning trace path --- -async def test_write_async_renders_reasoning_summary_when_requested(printer, attack_result, sqlite_instance, capsys): - reasoning_value = '{"summary": [{"text": "step one"}, {"text": "step two"}]}' +async def test_write_async_renders_reasoning_summary_when_requested( + printer, attack_result, sqlite_instance, reasoning_value +): piece = MessagePiece( role="assistant", original_value=reasoning_value, @@ -332,7 +333,8 @@ async def test_write_async_renders_reasoning_summary_when_requested(printer, att _seed_messages(sqlite_instance, "conv-main", [piece]) content = await printer._render_conversation_async(attack_result, include_reasoning_trace=True) - assert "Reasoning Summary" in content + assert "" in content + assert "Provider-generated reasoning summary (not raw chain-of-thought)" in content assert "step one" in content assert "step two" in content @@ -364,10 +366,7 @@ async def test_write_async_with_colors_enabled_emits_ansi_codes( assert "\x1b[" in capsys.readouterr().out -async def test_write_async_invalid_reasoning_summary_is_silently_skipped( - printer, attack_result, sqlite_instance, capsys -): - # Reasoning piece with non-JSON value should not blow up, just produce no summary section. +async def test_write_async_invalid_reasoning_summary_is_rejected(printer, attack_result, sqlite_instance): piece = MessagePiece( role="assistant", original_value="not-json", @@ -376,20 +375,50 @@ async def test_write_async_invalid_reasoning_summary_is_silently_skipped( converted_value_data_type="reasoning", ) _seed_messages(sqlite_instance, "conv-main", [piece]) - content = await printer._render_conversation_async(attack_result, include_reasoning_trace=True) - assert "Reasoning Summary" not in content + with pytest.raises(ValueError, match="valid JSON object"): + await printer._render_conversation_async(attack_result, include_reasoning_trace=True) -async def test_write_async_reasoning_summary_without_summary_key_is_silently_skipped( - printer, attack_result, sqlite_instance -): +async def test_write_async_reasoning_summary_without_summary_key_is_rejected(printer, attack_result, sqlite_instance): piece = MessagePiece( role="assistant", - original_value='{"other": "data"}', - converted_value='{"other": "data"}', + original_value='{"id": "r1", "type": "reasoning"}', + converted_value='{"id": "r1", "type": "reasoning"}', original_value_data_type="reasoning", converted_value_data_type="reasoning", ) _seed_messages(sqlite_instance, "conv-main", [piece]) - content = await printer._render_conversation_async(attack_result, include_reasoning_trace=True) - assert "Reasoning Summary" not in content + with pytest.raises(ValueError, match="'summary' list"): + await printer._render_conversation_async(attack_result, include_reasoning_trace=True) + + +async def test_write_async_pruned_reasoning_uses_pretty_tags( + printer, + attack_result, + sqlite_instance, + reasoning_value, +): + piece = MessagePiece( + role="assistant", + original_value=reasoning_value, + converted_value=reasoning_value, + original_value_data_type="reasoning", + converted_value_data_type="reasoning", + conversation_id="pruned-reasoning", + ) + sqlite_instance.add_message_to_memory(request=Message(message_pieces=[piece])) + attack_result.related_conversations.add( + ConversationReference( + conversation_id="pruned-reasoning", + conversation_type=ConversationType.PRUNED, + ) + ) + + rendered = await printer.render_async( + attack_result, + include_pruned_conversations=True, + include_reasoning_trace=True, + ) + + assert "" in rendered + assert "step one" in rendered diff --git a/tests/unit/output/conftest.py b/tests/unit/output/conftest.py new file mode 100644 index 0000000000..e080ca43ab --- /dev/null +++ b/tests/unit/output/conftest.py @@ -0,0 +1,55 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import json + +import pytest + +from pyrit.models import Message, MessagePiece + + +@pytest.fixture +def reasoning_value() -> str: + return json.dumps( + { + "id": "reasoning_123", + "type": "reasoning", + "summary": [ + {"type": "summary_text", "text": "step one"}, + {"type": "summary_text", "text": "step two"}, + ], + "status": "completed", + } + ) + + +@pytest.fixture +def empty_reasoning_value() -> str: + return json.dumps( + { + "id": "reasoning_empty", + "type": "reasoning", + "summary": [], + "status": "completed", + } + ) + + +@pytest.fixture +def reasoning_message(reasoning_value: str) -> Message: + return Message( + message_pieces=[ + MessagePiece( + role="assistant", + original_value=reasoning_value, + converted_value=reasoning_value, + original_value_data_type="reasoning", + converted_value_data_type="reasoning", + ), + MessagePiece( + role="assistant", + original_value="Final answer.", + converted_value="Final answer.", + ), + ] + ) diff --git a/tests/unit/output/conversation/test_reasoning.py b/tests/unit/output/conversation/test_reasoning.py new file mode 100644 index 0000000000..fd239dc008 --- /dev/null +++ b/tests/unit/output/conversation/test_reasoning.py @@ -0,0 +1,187 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import json + +import pytest +from colorama import Fore +from pydantic import ValidationError + +from pyrit.models import Message, MessagePiece +from pyrit.output.conversation.markdown import MarkdownConversationMemoryPrinter +from pyrit.output.conversation.pretty import PrettyConversationMemoryPrinter + + +@pytest.fixture +def pretty_printer(patch_central_database) -> PrettyConversationMemoryPrinter: + return PrettyConversationMemoryPrinter(enable_colors=False) + + +@pytest.fixture +def markdown_printer(patch_central_database) -> MarkdownConversationMemoryPrinter: + return MarkdownConversationMemoryPrinter() + + +async def test_pretty_reasoning_uses_literal_tags_and_spacing(pretty_printer, reasoning_message): + rendered = await pretty_printer.render_async([reasoning_message], include_reasoning_trace=True) + + assert "" in rendered + assert "\n\n Final answer." in rendered + assert "Provider-generated reasoning summary (not raw chain-of-thought)" in rendered + assert "step one" in rendered + assert "step two" in rendered + + +async def test_markdown_reasoning_uses_escaped_literal_tags_and_spacing(markdown_printer, reasoning_message): + rendered = await markdown_printer.render_async([reasoning_message], include_reasoning_trace=True) + + expected = ( + r"\" + "\n" + "> **Provider-generated reasoning summary (not raw chain-of-thought)**\n" + "> step one\n" + "> step two\n" + r"\" + "\n\n" + "Final answer." + ) + assert expected in rendered + assert "" not in rendered + + +@pytest.mark.parametrize("format_name", ["pretty", "markdown"]) +async def test_reasoning_is_hidden_by_default( + format_name, + pretty_printer, + markdown_printer, + reasoning_message, +): + printer = pretty_printer if format_name == "pretty" else markdown_printer + rendered = await printer.render_async([reasoning_message]) + + assert "reasoning-summary" not in rendered + assert "step one" not in rendered + assert "Final answer." in rendered + + +@pytest.mark.parametrize("format_name", ["pretty", "markdown"]) +async def test_empty_reasoning_summary_omits_tags( + format_name, + pretty_printer, + markdown_printer, + empty_reasoning_value, +): + message = Message( + message_pieces=[ + MessagePiece( + role="assistant", + original_value=empty_reasoning_value, + converted_value=empty_reasoning_value, + original_value_data_type="reasoning", + converted_value_data_type="reasoning", + ), + MessagePiece(role="assistant", original_value="Final answer."), + ] + ) + printer = pretty_printer if format_name == "pretty" else markdown_printer + + rendered = await printer.render_async([message], include_reasoning_trace=True) + + assert "reasoning-summary" not in rendered + assert "Final answer." in rendered + + +@pytest.mark.parametrize( + "reasoning_value", + [ + "not-json", + json.dumps({"type": "message", "summary": []}), + json.dumps({"type": "reasoning", "summary": []}), + json.dumps({"id": "r1", "type": "reasoning"}), + json.dumps({"id": "r1", "type": "reasoning", "summary": [{"type": "text", "text": "step"}]}), + json.dumps({"id": "r1", "type": "reasoning", "summary": [{"type": "summary_text", "text": 1}]}), + json.dumps({"id": "r1", "type": "reasoning", "summary": [], "status": "invalid"}), + json.dumps({"id": "r1", "type": "reasoning", "summary": [], "content": [{"type": "text"}]}), + ], +) +async def test_reasoning_payload_must_match_openai_responses_shape(markdown_printer, reasoning_value): + message = Message( + message_pieces=[ + MessagePiece( + role="assistant", + original_value=reasoning_value, + converted_value=reasoning_value, + original_value_data_type="reasoning", + converted_value_data_type="reasoning", + ) + ] + ) + + with pytest.raises(ValueError, match="Reasoning piece|reasoning summary item|reasoning content item"): + await markdown_printer.render_async([message], include_reasoning_trace=True) + + +def test_reasoning_prompt_data_type_requires_exact_literal(): + with pytest.raises(ValidationError): + MessagePiece( + role="assistant", + original_value="{}", + converted_value_data_type="thinking", # type: ignore[arg-type] + ) + + +async def test_original_reasoning_converted_to_text_remains_hidden_by_default( + markdown_printer, + reasoning_value, +): + message = Message( + message_pieces=[ + MessagePiece( + role="assistant", + original_value=reasoning_value, + converted_value="converted reasoning leak", + original_value_data_type="reasoning", + converted_value_data_type="text", + ), + MessagePiece(role="assistant", original_value="Final answer."), + ] + ) + + rendered = await markdown_printer.render_async([message]) + + assert "converted reasoning leak" not in rendered + assert "step one" not in rendered + assert "Final answer." in rendered + + +async def test_original_reasoning_converted_to_text_uses_original_reasoning_payload( + markdown_printer, + reasoning_value, +): + message = Message( + message_pieces=[ + MessagePiece( + role="assistant", + original_value=reasoning_value, + converted_value="converted reasoning leak", + original_value_data_type="reasoning", + converted_value_data_type="text", + ), + ] + ) + + rendered = await markdown_printer.render_async([message], include_reasoning_trace=True) + + assert "converted reasoning leak" not in rendered + assert "step one" in rendered + + +async def test_pretty_reasoning_is_gray_and_answer_keeps_assistant_color( + patch_central_database, + reasoning_message, +): + printer = PrettyConversationMemoryPrinter(enable_colors=True) + + rendered = await printer.render_async([reasoning_message], include_reasoning_trace=True) + + assert f"{Fore.LIGHTBLACK_EX} " in rendered + assert f"{Fore.LIGHTBLACK_EX} step one" in rendered + assert f"{Fore.YELLOW} Final answer." in rendered diff --git a/tests/unit/output/scenario_result/test_pretty.py b/tests/unit/output/scenario_result/test_pretty.py index ff870322de..7644ee9365 100644 --- a/tests/unit/output/scenario_result/test_pretty.py +++ b/tests/unit/output/scenario_result/test_pretty.py @@ -10,6 +10,8 @@ AttackOutcome, AttackResult, ComponentIdentifier, + Message, + MessagePiece, ScenarioResult, ) from pyrit.output.scenario_result.pretty import PrettyScenarioResultMemoryPrinter @@ -239,3 +241,37 @@ async def test_write_async_sort_is_stable_for_ties(patch_central_database, capsy await sorting_printer.write_async(result) # Tied 100% groups retain their original relative order; 0% group goes last. assert _group_order(capsys.readouterr().out) == ["first_success", "second_success", "fail"] + + +async def test_write_async_renders_attack_reasoning_when_requested( + printer, + sqlite_instance, + reasoning_value, +): + conversation_id = "scenario-reasoning" + reasoning_piece = MessagePiece( + role="assistant", + original_value=reasoning_value, + converted_value=reasoning_value, + original_value_data_type="reasoning", + converted_value_data_type="reasoning", + conversation_id=conversation_id, + ) + answer_piece = MessagePiece( + role="assistant", + original_value="Final answer.", + conversation_id=conversation_id, + ) + sqlite_instance.add_message_to_memory(request=Message(message_pieces=[reasoning_piece, answer_piece])) + attack_result = AttackResult( + conversation_id=conversation_id, + objective="scenario objective", + outcome=AttackOutcome.SUCCESS, + ) + result = _scenario_result(attack_results={"technique_a": [attack_result]}) + + rendered = await printer.render_async(result, include_reasoning_trace=True) + + assert "Attack Reasoning Summaries" in rendered + assert "" in rendered + assert "step one" in rendered diff --git a/tests/unit/output/test_helpers.py b/tests/unit/output/test_helpers.py index fe8d9fbde4..fb06c38851 100644 --- a/tests/unit/output/test_helpers.py +++ b/tests/unit/output/test_helpers.py @@ -99,7 +99,7 @@ async def test_output_scenario_async_pretty(mock_cls): await output_scenario_async(result) mock_cls.assert_called_once() - mock_printer.write_async.assert_called_once_with(result) + mock_printer.write_async.assert_called_once_with(result, include_reasoning_trace=False) @patch("pyrit.output.helpers.PrettyScenarioResultMemoryPrinter") @@ -112,7 +112,31 @@ async def test_output_scenario_async_forwards_sort_groups_by_success_rate(mock_c await output_scenario_async(result, sort_groups_by_success_rate=True) assert mock_cls.call_args.kwargs["sort_groups_by_success_rate"] is True - mock_printer.write_async.assert_called_once_with(result) + mock_printer.write_async.assert_called_once_with(result, include_reasoning_trace=False) + + +@patch("pyrit.output.helpers.PrettyAttackResultMemoryPrinter") +async def test_output_attack_async_forwards_reasoning_trace(mock_cls): + mock_printer = MagicMock() + mock_printer.write_async = AsyncMock() + mock_cls.return_value = mock_printer + result = MagicMock() + + await output_attack_async(result, include_reasoning_trace=True) + + assert mock_printer.write_async.call_args.kwargs["include_reasoning_trace"] is True + + +@patch("pyrit.output.helpers.PrettyScenarioResultMemoryPrinter") +async def test_output_scenario_async_forwards_reasoning_trace(mock_cls): + mock_printer = MagicMock() + mock_printer.write_async = AsyncMock() + mock_cls.return_value = mock_printer + result = MagicMock() + + await output_scenario_async(result, include_reasoning_trace=True) + + mock_printer.write_async.assert_called_once_with(result, include_reasoning_trace=True) async def test_output_scenario_async_unsupported_format(): From e31c427f9a114e18072362dd193270b6e3c91c02 Mon Sep 17 00:00:00 2001 From: Victor Valbuena Date: Fri, 24 Jul 2026 16:56:22 -0700 Subject: [PATCH 2/2] FEAT: Generated documentation and renamed _render_attack_results_async to _render_attack_reasoning_summaries_async for clarity. --- doc/code/output/0_output.ipynb | 55 +++++-- doc/code/output/0_output.py | 28 ++++ pyrit/output/scenario_result/pretty.py | 4 +- .../output/attack_result/test_markdown.py | 144 ++++++++++++++++++ 4 files changed, 218 insertions(+), 13 deletions(-) diff --git a/doc/code/output/0_output.ipynb b/doc/code/output/0_output.ipynb index 3ec202574c..2c47773eab 100644 --- a/doc/code/output/0_output.ipynb +++ b/doc/code/output/0_output.ipynb @@ -515,6 +515,39 @@ "cell_type": "markdown", "id": "13", "metadata": {}, + "source": [ + "## Including Reasoning Summaries\n", + "\n", + "Reasoning-summary output is opt in: the conversation, attack-result, and scenario-result\n", + "helpers hide reasoning by default. For OpenAI Responses targets, PyRIT renders\n", + "provider-generated reasoning summaries exposed by OpenAI, not raw hidden chain-of-thought.\n", + "\n", + "- Pretty output shows the `` and `` tags in subdued gray.\n", + "- Attack Markdown output shows visible escaped `\\` and\n", + " `\\` tags.\n", + "\n", + "```python\n", + "from pyrit.output import output_attack_async, output_conversation_async, output_scenario_async\n", + "\n", + "# Direct conversation\n", + "await output_conversation_async(messages=conversation, include_reasoning_trace=True)\n", + "\n", + "# Attack result (Pretty or Markdown)\n", + "await output_attack_async(attack_result, include_reasoning_trace=True)\n", + "await output_attack_async(attack_result, format=\"markdown\", include_reasoning_trace=True)\n", + "\n", + "# Existing scenario result (Pretty only)\n", + "await output_scenario_async(scenario_result, include_reasoning_trace=True)\n", + "```\n", + "\n", + "For a scenario, enabling reasoning expands every contained attack result so its summaries\n", + "can be rendered. This can produce verbose output for large scenario runs." + ] + }, + { + "cell_type": "markdown", + "id": "14", + "metadata": {}, "source": [ "## Printing Scores\n", "\n", @@ -524,7 +557,7 @@ { "cell_type": "code", "execution_count": null, - "id": "14", + "id": "15", "metadata": {}, "outputs": [ { @@ -551,7 +584,7 @@ }, { "cell_type": "markdown", - "id": "15", + "id": "16", "metadata": {}, "source": [ "## Sinks — Redirecting Output\n", @@ -565,7 +598,7 @@ { "cell_type": "code", "execution_count": null, - "id": "16", + "id": "17", "metadata": {}, "outputs": [ { @@ -604,7 +637,7 @@ }, { "cell_type": "markdown", - "id": "17", + "id": "18", "metadata": {}, "source": [ "### Available Sinks\n", @@ -621,7 +654,7 @@ }, { "cell_type": "markdown", - "id": "18", + "id": "19", "metadata": {}, "source": [ "## Using Printers Directly\n", @@ -634,7 +667,7 @@ { "cell_type": "code", "execution_count": null, - "id": "19", + "id": "20", "metadata": {}, "outputs": [ { @@ -675,7 +708,7 @@ }, { "cell_type": "markdown", - "id": "20", + "id": "21", "metadata": {}, "source": [ "### `render_async` vs `write_async`\n", @@ -689,7 +722,7 @@ { "cell_type": "code", "execution_count": null, - "id": "21", + "id": "22", "metadata": {}, "outputs": [ { @@ -728,7 +761,7 @@ }, { "cell_type": "markdown", - "id": "22", + "id": "23", "metadata": {}, "source": [ "## Architecture Overview\n", @@ -785,7 +818,7 @@ }, { "cell_type": "markdown", - "id": "23", + "id": "24", "metadata": {}, "source": [ "## Convenience Functions Reference\n", @@ -805,7 +838,7 @@ }, { "cell_type": "markdown", - "id": "24", + "id": "25", "metadata": {}, "source": [ "## Extending the Printer Module\n", diff --git a/doc/code/output/0_output.py b/doc/code/output/0_output.py index 10a03840b3..83616f5bf9 100644 --- a/doc/code/output/0_output.py +++ b/doc/code/output/0_output.py @@ -143,6 +143,34 @@ # print the conversation using the print conversation helper await output_conversation_async(messages=conversation) # type: ignore +# %% [markdown] +# ## Including Reasoning Summaries +# +# Reasoning-summary output is opt in: the conversation, attack-result, and scenario-result +# helpers hide reasoning by default. For OpenAI Responses targets, PyRIT renders +# provider-generated reasoning summaries exposed by OpenAI, not raw hidden chain-of-thought. +# +# - Pretty output shows the `` and `` tags in subdued gray. +# - Attack Markdown output shows visible escaped `\` and +# `\` tags. +# +# ```python +# from pyrit.output import output_attack_async, output_conversation_async, output_scenario_async +# +# # Direct conversation +# await output_conversation_async(messages=conversation, include_reasoning_trace=True) +# +# # Attack result (Pretty or Markdown) +# await output_attack_async(attack_result, include_reasoning_trace=True) +# await output_attack_async(attack_result, format="markdown", include_reasoning_trace=True) +# +# # Existing scenario result (Pretty only) +# await output_scenario_async(scenario_result, include_reasoning_trace=True) +# ``` +# +# For a scenario, enabling reasoning expands every contained attack result so its summaries +# can be rendered. This can produce verbose output for large scenario runs. + # %% [markdown] # ## Printing Scores # diff --git a/pyrit/output/scenario_result/pretty.py b/pyrit/output/scenario_result/pretty.py index d86a11ae6c..9c36a0343d 100644 --- a/pyrit/output/scenario_result/pretty.py +++ b/pyrit/output/scenario_result/pretty.py @@ -252,7 +252,7 @@ async def render_async( lines.append(self._render_footer()) parts.append("".join(lines)) parts.append( - await self._render_attack_results_async( + await self._render_attack_reasoning_summaries_async( result=result, include_reasoning_trace=include_reasoning_trace, ) @@ -260,7 +260,7 @@ async def render_async( return "".join(parts) - async def _render_attack_results_async( + async def _render_attack_reasoning_summaries_async( self, *, result: ScenarioResult, diff --git a/tests/unit/output/attack_result/test_markdown.py b/tests/unit/output/attack_result/test_markdown.py index 3f88d7f5f4..5fa7e1cee2 100644 --- a/tests/unit/output/attack_result/test_markdown.py +++ b/tests/unit/output/attack_result/test_markdown.py @@ -460,3 +460,147 @@ async def test_write_async_adversarial_reasoning_uses_escaped_tags( assert r"\" in rendered assert "step one" in rendered + + +# --- pruned conversations: internal rendering branches --- + + +async def test_write_async_pruned_renders_only_last_message_with_role_label( + printer, attack_result, sqlite_instance, capsys +): + earlier = MessagePiece(role="user", original_value="older pruned msg", converted_value="older pruned msg") + latest = MessagePiece(role="assistant", original_value="newest pruned msg", converted_value="newest pruned msg") + _seed_messages(sqlite_instance, "pruned-multi", [earlier, latest]) + attack_result.related_conversations.add( + ConversationReference(conversation_id="pruned-multi", conversation_type=ConversationType.PRUNED) + ) + + await printer.write_async(attack_result, include_pruned_conversations=True) + out = capsys.readouterr().out + assert "**Last Message (ASSISTANT):**" in out + assert "newest pruned msg" in out + assert "older pruned msg" not in out + + +async def test_write_async_pruned_single_line_uses_blockquote_without_score( + printer, attack_result, sqlite_instance, capsys +): + piece = MessagePiece(role="assistant", original_value="solo pruned line", converted_value="solo pruned line") + _seed_messages(sqlite_instance, "pruned-solo", [piece]) + attack_result.related_conversations.add( + ConversationReference(conversation_id="pruned-solo", conversation_type=ConversationType.PRUNED) + ) + + await printer.write_async(attack_result, include_pruned_conversations=True) + out = capsys.readouterr().out + assert "> solo pruned line" in out + assert "**Score:**" not in out + + +async def test_write_async_pruned_skips_last_message_with_no_renderable_pieces( + printer, attack_result, sqlite_instance, reasoning_value +): + piece = MessagePiece( + role="assistant", + original_value=reasoning_value, + converted_value=reasoning_value, + original_value_data_type="reasoning", + converted_value_data_type="reasoning", + conversation_id="pruned-reasoning-only", + ) + sqlite_instance.add_message_to_memory(request=Message(message_pieces=[piece])) + attack_result.related_conversations.add( + ConversationReference(conversation_id="pruned-reasoning-only", conversation_type=ConversationType.PRUNED) + ) + + rendered = await printer.render_async( + attack_result, + include_pruned_conversations=True, + include_reasoning_trace=False, + ) + + assert "## Pruned Conversations" in rendered + assert "**Last Message" not in rendered + assert "step one" not in rendered + + +# --- adversarial conversation: internal rendering branches --- + + +async def test_write_async_adversarial_renders_system_header(printer, attack_result, sqlite_instance, capsys): + piece = MessagePiece(role="system", original_value="adv system directive", converted_value="adv system directive") + _seed_messages(sqlite_instance, "adv-system", [piece]) + attack_result.related_conversations.add( + ConversationReference(conversation_id="adv-system", conversation_type=ConversationType.ADVERSARIAL) + ) + + await printer.write_async(attack_result, include_adversarial_conversation=True) + out = capsys.readouterr().out + assert "#### SYSTEM" in out + assert "adv system directive" in out + + +async def test_write_async_adversarial_turn_numbering_and_role_headers(printer, attack_result, sqlite_instance, capsys): + first_user = MessagePiece(role="user", original_value="first adv prompt", converted_value="first adv prompt") + assistant = MessagePiece(role="assistant", original_value="adv answer", converted_value="adv answer") + second_user = MessagePiece(role="user", original_value="second adv prompt", converted_value="second adv prompt") + _seed_messages(sqlite_instance, "adv-turns", [first_user, assistant, second_user]) + attack_result.related_conversations.add( + ConversationReference(conversation_id="adv-turns", conversation_type=ConversationType.ADVERSARIAL) + ) + + await printer.write_async(attack_result, include_adversarial_conversation=True) + out = capsys.readouterr().out + assert "#### Turn 1 - USER" in out + assert "#### ASSISTANT" in out + assert "#### Turn 2 - USER" in out + + +async def test_write_async_adversarial_skips_message_with_no_renderable_pieces( + printer, attack_result, sqlite_instance, reasoning_value +): + piece = MessagePiece( + role="assistant", + original_value=reasoning_value, + converted_value=reasoning_value, + original_value_data_type="reasoning", + converted_value_data_type="reasoning", + conversation_id="adv-reasoning-only", + ) + sqlite_instance.add_message_to_memory(request=Message(message_pieces=[piece])) + attack_result.related_conversations.add( + ConversationReference(conversation_id="adv-reasoning-only", conversation_type=ConversationType.ADVERSARIAL) + ) + + rendered = await printer.render_async( + attack_result, + include_adversarial_conversation=True, + include_reasoning_trace=False, + ) + + assert "## Adversarial Conversation" in rendered + assert "#### " not in rendered + assert "step one" not in rendered + + +async def test_write_async_adversarial_best_id_with_no_matching_ref(patch_central_database, sqlite_instance, capsys): + piece = MessagePiece(role="user", original_value="unmatched adv content", converted_value="unmatched adv content") + _seed_messages(sqlite_instance, "adv-present", [piece]) + + result = AttackResult( + objective="o", + conversation_id="conv-main", + outcome=AttackOutcome.SUCCESS, + metadata={"best_adversarial_conversation_id": "adv-missing"}, + related_conversations={ + ConversationReference(conversation_id="adv-present", conversation_type=ConversationType.ADVERSARIAL), + }, + ) + + await MarkdownAttackResultMemoryPrinter(display_inline=False).write_async( + result, include_adversarial_conversation=True + ) + out = capsys.readouterr().out + assert "## Adversarial Conversation" in out + assert "best-scoring branch" not in out + assert "unmatched adv content" not in out