From 7a5fcbc0d8e35f7ac86cc20852d25de0ec7a74aa Mon Sep 17 00:00:00 2001 From: Erica Pisani Date: Tue, 11 Aug 2026 14:54:28 -0400 Subject: [PATCH] fix(ai): Gate response tool calls on the gen_ai inputs setting `gen_ai.response.tool_calls` was gated on `data_collection.gen_ai.outputs` in the OpenAI, Anthropic, OpenAI Agents, and Google GenAI integrations. Tool calls belong to the inputs category, so they are now gated on `data_collection.gen_ai.inputs` while response text stays on `outputs`. Where both attributes previously shared a single gate, the block is split so each is evaluated independently. The legacy `send_default_pii`/`include_prompts` path is unchanged. Refs PY-2588 Refs #6748 --- sentry_sdk/integrations/anthropic.py | 12 ++-- .../integrations/google_genai/streaming.py | 2 +- sentry_sdk/integrations/google_genai/utils.py | 2 +- sentry_sdk/integrations/openai.py | 9 ++- .../integrations/openai_agents/utils.py | 16 +++-- .../integrations/anthropic/test_anthropic.py | 66 ++++++++++++++++--- .../google_genai/test_google_genai.py | 20 +++++- tests/integrations/openai/test_openai.py | 27 ++++++-- .../openai_agents/test_openai_agents.py | 26 ++++++-- 9 files changed, 143 insertions(+), 37 deletions(-) diff --git a/sentry_sdk/integrations/anthropic.py b/sentry_sdk/integrations/anthropic.py index cc1d468be9..5aca42e4b7 100644 --- a/sentry_sdk/integrations/anthropic.py +++ b/sentry_sdk/integrations/anthropic.py @@ -608,14 +608,16 @@ def _set_output_data( set_on_span(SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS, [finish_reason]) client = sentry_sdk.get_client() + record_inputs = False record_outputs = False if has_data_collection_enabled(client.options): - if client.options["data_collection"]["gen_ai"]["outputs"]: - record_outputs = True + record_inputs = client.options["data_collection"]["gen_ai"]["inputs"] + record_outputs = client.options["data_collection"]["gen_ai"]["outputs"] elif should_send_default_pii() and integration.include_prompts: + record_inputs = True record_outputs = True - if record_outputs: + if record_inputs or record_outputs: output_messages: "dict[str, list[Any]]" = { "response": [], "tool": [], @@ -627,7 +629,7 @@ def _set_output_data( elif output["type"] == "tool_use": output_messages["tool"].append(output) - if len(output_messages["tool"]) > 0: + if record_inputs and len(output_messages["tool"]) > 0: set_data_normalized( span, SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS, @@ -635,7 +637,7 @@ def _set_output_data( unpack=False, ) - if len(output_messages["response"]) > 0: + if record_outputs and len(output_messages["response"]) > 0: set_data_normalized( span, SPANDATA.GEN_AI_RESPONSE_TEXT, output_messages["response"] ) diff --git a/sentry_sdk/integrations/google_genai/streaming.py b/sentry_sdk/integrations/google_genai/streaming.py index 86cdcf29ba..99e0a87b6e 100644 --- a/sentry_sdk/integrations/google_genai/streaming.py +++ b/sentry_sdk/integrations/google_genai/streaming.py @@ -160,7 +160,7 @@ def set_span_data_for_streaming_response( if accumulated_response.get("tool_calls"): if has_data_collection_enabled(client.options): - if client.options["data_collection"]["gen_ai"]["outputs"]: + if client.options["data_collection"]["gen_ai"]["inputs"]: set_on_span( SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS, safe_serialize(accumulated_response["tool_calls"]), diff --git a/sentry_sdk/integrations/google_genai/utils.py b/sentry_sdk/integrations/google_genai/utils.py index 75ba30c199..cd2b2bc9a8 100644 --- a/sentry_sdk/integrations/google_genai/utils.py +++ b/sentry_sdk/integrations/google_genai/utils.py @@ -1034,7 +1034,7 @@ def set_span_data_for_response( tool_calls = extract_tool_calls(response) if tool_calls: if has_data_collection_enabled(client.options): - if client.options["data_collection"]["gen_ai"]["outputs"]: + if client.options["data_collection"]["gen_ai"]["inputs"]: set_on_span( SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS, safe_serialize(tool_calls) ) diff --git a/sentry_sdk/integrations/openai.py b/sentry_sdk/integrations/openai.py index ae6dafe174..63653bdee0 100644 --- a/sentry_sdk/integrations/openai.py +++ b/sentry_sdk/integrations/openai.py @@ -712,7 +712,10 @@ def _set_common_output_data( } if has_data_collection_enabled(client.options): - if client.options["data_collection"]["gen_ai"]["outputs"]: + record_inputs = client.options["data_collection"]["gen_ai"]["inputs"] + record_outputs = client.options["data_collection"]["gen_ai"]["outputs"] + + if record_inputs or record_outputs: for output in response.output: if output.type == "function_call": output_messages["tool"].append(output.dict()) @@ -726,7 +729,7 @@ def _set_common_output_data( output_message.dict() ) - if len(output_messages["tool"]) > 0: + if record_inputs and len(output_messages["tool"]) > 0: set_data_normalized( span, SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS, @@ -734,7 +737,7 @@ def _set_common_output_data( unpack=False, ) - if len(output_messages["response"]) > 0: + if record_outputs and len(output_messages["response"]) > 0: set_data_normalized( span, SPANDATA.GEN_AI_RESPONSE_TEXT, output_messages["response"] ) diff --git a/sentry_sdk/integrations/openai_agents/utils.py b/sentry_sdk/integrations/openai_agents/utils.py index f8f605fe0a..f1d81fd413 100644 --- a/sentry_sdk/integrations/openai_agents/utils.py +++ b/sentry_sdk/integrations/openai_agents/utils.py @@ -209,10 +209,16 @@ def _set_output_data( span: "Union[sentry_sdk.tracing.Span, StreamedSpan]", result: "Any" ) -> None: client = sentry_sdk.get_client() + record_inputs = False + record_outputs = False if has_data_collection_enabled(client.options): - if not client.options["data_collection"]["gen_ai"]["outputs"]: - return - elif not should_send_default_pii(): + record_inputs = client.options["data_collection"]["gen_ai"]["inputs"] + record_outputs = client.options["data_collection"]["gen_ai"]["outputs"] + elif should_send_default_pii(): + record_inputs = True + record_outputs = True + + if not record_inputs and not record_outputs: return output_messages: "dict[str, list[Any]]" = { @@ -231,7 +237,7 @@ def _set_output_data( # Unknown output message type, just return the json output_messages["response"].append(output_message.dict()) - if len(output_messages["tool"]) > 0: + if record_inputs and len(output_messages["tool"]) > 0: if isinstance(span, StreamedSpan): span.set_attribute( SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS, @@ -243,7 +249,7 @@ def _set_output_data( safe_serialize(output_messages["tool"]), ) - if len(output_messages["response"]) > 0: + if record_outputs and len(output_messages["response"]) > 0: set_data_normalized( span, SPANDATA.GEN_AI_RESPONSE_TEXT, output_messages["response"] ) diff --git a/tests/integrations/anthropic/test_anthropic.py b/tests/integrations/anthropic/test_anthropic.py index e57afd67e6..7a54dfa1ae 100644 --- a/tests/integrations/anthropic/test_anthropic.py +++ b/tests/integrations/anthropic/test_anthropic.py @@ -639,13 +639,14 @@ async def test_nonstreaming_create_message_data_collection_async( @pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize("stream_gen_ai_spans", [True, False]) @pytest.mark.parametrize( - "data_collection,send_default_pii,include_prompts,outputs_collected", + "data_collection,send_default_pii,include_prompts,outputs_collected,tool_calls_collected", [ pytest.param( {"gen_ai": {"outputs": True}}, False, False, True, + True, id="gen-ai-outputs-enabled-overrides-pii-and-include-prompts", ), pytest.param( @@ -653,20 +654,39 @@ async def test_nonstreaming_create_message_data_collection_async( True, True, False, - id="gen-ai-outputs-disabled-overrides-pii-and-include-prompts", + True, + id="gen-ai-outputs-disabled-still-collects-tool-calls-gated-on-inputs", + ), + pytest.param( + {"gen_ai": {"inputs": False}}, + True, + True, + True, + False, + id="gen-ai-inputs-disabled-drops-tool-calls-only", + ), + pytest.param( + {"gen_ai": {"inputs": False, "outputs": False}}, + True, + True, + False, + False, + id="gen-ai-inputs-and-outputs-disabled-overrides-pii-and-include-prompts", ), pytest.param( {"gen_ai": {}}, False, False, True, - id="gen-ai-outputs-omitted-defaults-to-enabled", + True, + id="gen-ai-inputs-and-outputs-omitted-defaults-to-enabled", ), pytest.param( None, True, True, True, + True, id="legacy-pii-and-include-prompts-enabled", ), pytest.param( @@ -674,6 +694,7 @@ async def test_nonstreaming_create_message_data_collection_async( False, True, False, + False, id="legacy-pii-disabled", ), ], @@ -686,6 +707,7 @@ def test_nonstreaming_create_message_data_collection_outputs( send_default_pii, include_prompts, outputs_collected, + tool_calls_collected, stream_gen_ai_spans, span_streaming, ): @@ -743,12 +765,15 @@ def test_nonstreaming_create_message_data_collection_outputs( span_data[SPANDATA.GEN_AI_RESPONSE_TEXT] == DATA_COLLECTION_EXPECTED_RESPONSE_TEXT ) + else: + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span_data + + if tool_calls_collected: assert ( json.loads(span_data[SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS]) == DATA_COLLECTION_EXPECTED_TOOL_CALLS ) else: - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span_data assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS not in span_data @@ -760,13 +785,14 @@ def test_nonstreaming_create_message_data_collection_outputs( @pytest.mark.parametrize("stream_gen_ai_spans", [True, False]) @pytest.mark.asyncio @pytest.mark.parametrize( - "data_collection,send_default_pii,include_prompts,outputs_collected", + "data_collection,send_default_pii,include_prompts,outputs_collected,tool_calls_collected", [ pytest.param( {"gen_ai": {"outputs": True}}, False, False, True, + True, id="gen-ai-outputs-enabled-overrides-pii-and-include-prompts", ), pytest.param( @@ -774,20 +800,39 @@ def test_nonstreaming_create_message_data_collection_outputs( True, True, False, - id="gen-ai-outputs-disabled-overrides-pii-and-include-prompts", + True, + id="gen-ai-outputs-disabled-still-collects-tool-calls-gated-on-inputs", + ), + pytest.param( + {"gen_ai": {"inputs": False}}, + True, + True, + True, + False, + id="gen-ai-inputs-disabled-drops-tool-calls-only", + ), + pytest.param( + {"gen_ai": {"inputs": False, "outputs": False}}, + True, + True, + False, + False, + id="gen-ai-inputs-and-outputs-disabled-overrides-pii-and-include-prompts", ), pytest.param( {"gen_ai": {}}, False, False, True, - id="gen-ai-outputs-omitted-defaults-to-enabled", + True, + id="gen-ai-inputs-and-outputs-omitted-defaults-to-enabled", ), pytest.param( None, True, True, True, + True, id="legacy-pii-and-include-prompts-enabled", ), pytest.param( @@ -795,6 +840,7 @@ def test_nonstreaming_create_message_data_collection_outputs( False, True, False, + False, id="legacy-pii-disabled", ), ], @@ -807,6 +853,7 @@ async def test_nonstreaming_create_message_data_collection_outputs_async( send_default_pii, include_prompts, outputs_collected, + tool_calls_collected, stream_gen_ai_spans, span_streaming, ): @@ -864,12 +911,15 @@ async def test_nonstreaming_create_message_data_collection_outputs_async( span_data[SPANDATA.GEN_AI_RESPONSE_TEXT] == DATA_COLLECTION_EXPECTED_RESPONSE_TEXT ) + else: + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span_data + + if tool_calls_collected: assert ( json.loads(span_data[SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS]) == DATA_COLLECTION_EXPECTED_TOOL_CALLS ) else: - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span_data assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS not in span_data diff --git a/tests/integrations/google_genai/test_google_genai.py b/tests/integrations/google_genai/test_google_genai.py index 756d83572a..a31e983da0 100644 --- a/tests/integrations/google_genai/test_google_genai.py +++ b/tests/integrations/google_genai/test_google_genai.py @@ -3843,11 +3843,19 @@ def test_generate_content_data_collection( {"gen_ai": {"inputs": True, "outputs": False}}, [ SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS, + SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS, ], + [], + id="gen-ai-inputs-enabled-outputs-disabled-tools-collected", + ), + pytest.param( + {"gen_ai": {"inputs": False, "outputs": True}}, + [], [ + SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS, SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS, ], - id="gen-ai-inputs-enabled-outputs-disabled-available-tools-only", + id="gen-ai-inputs-disabled-outputs-enabled-tools-not-collected", ), pytest.param( {"gen_ai": {}}, @@ -4195,11 +4203,19 @@ def test_streaming_generate_content_data_collection( {"gen_ai": {"inputs": True, "outputs": False}}, [ SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS, + SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS, ], + [], + id="gen-ai-inputs-enabled-outputs-disabled-tools-collected", + ), + pytest.param( + {"gen_ai": {"inputs": False, "outputs": True}}, + [], [ + SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS, SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS, ], - id="gen-ai-inputs-enabled-outputs-disabled-available-tools-only", + id="gen-ai-inputs-disabled-outputs-enabled-tools-not-collected", ), pytest.param( {"gen_ai": {}}, diff --git a/tests/integrations/openai/test_openai.py b/tests/integrations/openai/test_openai.py index b253c6e6b5..a86843a3ca 100644 --- a/tests/integrations/openai/test_openai.py +++ b/tests/integrations/openai/test_openai.py @@ -5980,42 +5980,55 @@ def _collect_responses_span_data( @pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize("stream_gen_ai_spans", [True, False]) @pytest.mark.parametrize( - "data_collection,send_default_pii,expect_output", + "data_collection,send_default_pii,expect_output,expect_tool_calls", [ pytest.param( {"gen_ai": {"outputs": True}}, False, True, + True, id="gen-ai-outputs-enabled-overrides-pii-disabled", ), pytest.param( {"gen_ai": {"outputs": False}}, True, False, - id="gen-ai-outputs-disabled-overrides-pii-enabled", + True, + id="gen-ai-outputs-disabled-still-collects-tool-calls-gated-on-inputs", + ), + pytest.param( + {"gen_ai": {"inputs": False}}, + True, + True, + False, + id="gen-ai-inputs-disabled-drops-tool-calls-only", ), pytest.param( {}, False, True, + True, id="gen-ai-omitted-defaults-to-enabled", ), pytest.param( - {"gen_ai": {"outputs": False}}, + {"gen_ai": {"inputs": False, "outputs": False}}, False, False, - id="gen-ai-outputs-disabled-and-pii-disabled", + False, + id="gen-ai-inputs-and-outputs-disabled-and-pii-disabled", ), pytest.param( None, False, False, + False, id="no-gen-ai-data-collection-falls-back-to-send-default-pii", ), pytest.param( None, True, True, + True, id="no-gen-ai-data-collection-pii-enabled-collects", ), ], @@ -6028,6 +6041,7 @@ def test_responses_api_data_collection_outputs( data_collection, send_default_pii, expect_output, + expect_tool_calls, stream_gen_ai_spans, span_streaming, ): @@ -6074,9 +6088,12 @@ def test_responses_api_data_collection_outputs( if expect_output: assert "the model response" in span_data[SPANDATA.GEN_AI_RESPONSE_TEXT] - assert "get_current_weather" in span_data[SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS] else: assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span_data + + if expect_tool_calls: + assert "get_current_weather" in span_data[SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS] + else: assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS not in span_data diff --git a/tests/integrations/openai_agents/test_openai_agents.py b/tests/integrations/openai_agents/test_openai_agents.py index cf055b762a..0bd9fb9de5 100644 --- a/tests/integrations/openai_agents/test_openai_agents.py +++ b/tests/integrations/openai_agents/test_openai_agents.py @@ -1180,48 +1180,55 @@ async def test_data_collection_inputs( @pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize("stream_gen_ai_spans", [True, False]) @pytest.mark.parametrize( - "data_collection,send_default_pii,expect_output", + "data_collection,send_default_pii,expect_output,expect_tool_calls", [ pytest.param( {"gen_ai": {"outputs": True}}, False, True, + True, id="gen-ai-outputs-enabled-overrides-pii-disabled", ), pytest.param( {"gen_ai": {"outputs": False}}, True, False, - id="gen-ai-outputs-disabled-overrides-pii-enabled", + True, + id="gen-ai-outputs-disabled-still-collects-tool-calls-gated-on-inputs", ), pytest.param( {}, False, True, + True, id="gen-ai-omitted-defaults-to-enabled", ), pytest.param( - {"gen_ai": {"outputs": False}}, + {"gen_ai": {"inputs": False, "outputs": False}}, False, False, - id="gen-ai-outputs-disabled-and-pii-disabled", + False, + id="gen-ai-inputs-and-outputs-disabled-and-pii-disabled", ), pytest.param( {"gen_ai": {"inputs": False}}, False, True, - id="gen-ai-inputs-disabled-does-not-affect-outputs", + False, + id="gen-ai-inputs-disabled-drops-tool-calls-only", ), pytest.param( None, False, False, + False, id="no-data-collection-falls-back-to-send-default-pii", ), pytest.param( None, True, True, + True, id="no-data-collection-pii-enabled-collects", ), ], @@ -1238,6 +1245,7 @@ async def test_data_collection_outputs( data_collection, send_default_pii, expect_output, + expect_tool_calls, stream_gen_ai_spans, span_streaming, ): @@ -1318,11 +1326,16 @@ async def test_data_collection_outputs( assert len(chat_span_data) == 2 - if expect_output: + if expect_tool_calls: assert any( "simple_test_tool" in data.get(SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS, "") for data in chat_span_data ) + else: + for data in chat_span_data: + assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS not in data + + if expect_output: assert any( "Task completed using the tool" in str(data.get(SPANDATA.GEN_AI_RESPONSE_TEXT, "")) @@ -1330,7 +1343,6 @@ async def test_data_collection_outputs( ) else: for data in chat_span_data: - assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS not in data assert SPANDATA.GEN_AI_RESPONSE_TEXT not in data # Non-PII data is unaffected by the gate