Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,10 @@
from opentelemetry import trace
from opentelemetry.instrumentation.anthropic.version import __version__
from opentelemetry.instrumentation.fortifyroot import (
FR_HAS_ATTEMPT_CHILD_KEY,
is_framework_owned,
llm_attempt_attributes,
next_llm_attempt,
)
from opentelemetry.instrumentation.utils import unwrap
from opentelemetry.semconv_ai import SUPPRESS_LANGUAGE_MODEL_INSTRUMENTATION_KEY
Expand All @@ -52,10 +55,10 @@


# ST-10 §4.4 / §4.5 constants.
_FR_RETRY_ATTEMPT_SPAN_NAME = "fortifyroot.anthropic.retry_attempt"
_FR_SPAN_ROLE_KEY = "fortifyroot.span.role"
_FR_SPAN_ROLE_RETRY_ATTEMPT = "retry_attempt"
_FR_HAS_RETRY_ATTEMPT_CHILD_KEY = "fortifyroot.span.has_retry_attempt_child"
_FR_LLM_ATTEMPT_SPAN_NAME_PREFIX = "fortifyroot.anthropic"
_FR_SPAN_ROLE_LLM_ATTEMPT = "llm_attempt"
_FR_HAS_ATTEMPT_CHILD_KEY = FR_HAS_ATTEMPT_CHILD_KEY

_ANTHROPIC_BASE_CLIENT_MODULE = "anthropic._base_client"
_SYNC_WRAPPER_CLASS = "SyncHttpxClientWrapper"
Expand Down Expand Up @@ -179,15 +182,18 @@ def _resolve_parent_span() -> Optional["trace.Span"]:

def _set_parent_marker(parent_span: "trace.Span") -> None:
try:
parent_span.set_attribute(_FR_HAS_RETRY_ATTEMPT_CHILD_KEY, True)
parent_span.set_attribute(_FR_HAS_ATTEMPT_CHILD_KEY, True)
except Exception:
logger.debug("failed to set has_retry_attempt_child on parent", exc_info=True)
logger.debug("failed to set has_attempt_child on parent", exc_info=True)


def _start_attempt_span(request: Any, parent_span: "trace.Span") -> "trace.Span":
path = _request_path(request)
span_name, attempt_number, is_retry = next_llm_attempt(
parent_span,
_FR_LLM_ATTEMPT_SPAN_NAME_PREFIX,
)
attrs: dict[str, Any] = {
_FR_SPAN_ROLE_KEY: _FR_SPAN_ROLE_RETRY_ATTEMPT,
# Match the existing Anthropic instrumentor's gen_ai.system
# value ("Anthropic" title-case — see
# ``anthropic/__init__.py`` ``_wrap``). The earlier draft of
Expand All @@ -201,6 +207,7 @@ def _start_attempt_span(request: Any, parent_span: "trace.Span") -> "trace.Span"
"gen_ai.system": "Anthropic",
"gen_ai.operation.name": _operation_for_path(path),
}
attrs.update(llm_attempt_attributes(attempt_number, is_retry))
model = _resolve_model_from_request(request)
if model:
attrs["gen_ai.request.model"] = model
Expand All @@ -209,7 +216,7 @@ def _start_attempt_span(request: Any, parent_span: "trace.Span") -> "trace.Span"
tracer = trace.get_tracer(__name__, __version__, _tracer_provider)
parent_ctx = trace.set_span_in_context(parent_span)
span = tracer.start_span(
_FR_RETRY_ATTEMPT_SPAN_NAME,
span_name,
kind=SpanKind.CLIENT,
attributes=attrs,
context=parent_ctx,
Expand Down Expand Up @@ -272,12 +279,12 @@ def _finalize_success(span: "trace.Span", response: Any, *, is_streaming: bool =
span.set_attribute("error.type", err_type)
span.set_status(Status(StatusCode.ERROR, f"http {status_code}"))
except Exception:
logger.debug("failed to set response attrs on anthropic retry_attempt", exc_info=True)
logger.debug("failed to set response attrs on anthropic llm_attempt", exc_info=True)


def _extract_usage_from_body(span: "trace.Span", response: Any) -> None:
"""Parse a non-streaming Anthropic response body and copy usage /
response id / response model attrs to the retry_attempt span.
response id / response model attrs to the llm_attempt span.

Anthropic Messages response: ``{"id": "...", "model": "...",
"usage": {"input_tokens": N, "output_tokens": M, ...}}``. Legacy
Expand Down Expand Up @@ -326,7 +333,7 @@ def _finalize_error(span: "trace.Span", error: BaseException) -> None:
pass
span.set_status(Status(StatusCode.ERROR, str(error)))
except Exception:
logger.debug("failed to set error attrs on anthropic retry_attempt", exc_info=True)
logger.debug("failed to set error attrs on anthropic llm_attempt", exc_info=True)


def _should_emit_for(request: Any) -> bool:
Expand Down Expand Up @@ -552,7 +559,7 @@ def _is_installed_for_test() -> bool:
__all__ = [
"instrument_retry_emitter",
"uninstrument_retry_emitter",
"_FR_RETRY_ATTEMPT_SPAN_NAME",
"_FR_HAS_RETRY_ATTEMPT_CHILD_KEY",
"_FR_LLM_ATTEMPT_SPAN_NAME_PREFIX",
"_FR_HAS_ATTEMPT_CHILD_KEY",
"_LLM_PATH_SUFFIXES",
]
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ class _NoFortifyRootSpanExporter(InMemorySpanExporter):
"""ST-10.4 (2026-05-17): filter out any span whose name starts
with ``fortifyroot.`` from the upstream-test span exporter.

ST-10.4 added per-attempt ``fortifyroot.anthropic.retry_attempt``
ST-10.4 added per-attempt ``fortifyroot.anthropic.attempt_1``
sibling spans under every anthropic logical call (once the
Anthropic ``_wrap`` started using ``trace.use_span`` so the retry
handler can find the parent span). Upstream / legacy Anthropic
Expand All @@ -54,7 +54,7 @@ def get_finished_spans(self): # type: ignore[override]
# visible to tests that inspect them.
return tuple(
s for s in super().get_finished_spans()
if (s.attributes or {}).get("fortifyroot.span.role") != "retry_attempt"
if (s.attributes or {}).get("fortifyroot.span.role") != "llm_attempt"
)


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@
Covers (per RETRY_LOOP.md §4.4 Anthropic row + §4.7 suppression):
- Instrumentor symmetry: install/uninstall flips state cleanly and is
idempotent.
- Single-attempt happy path: ONE retry_attempt span under the active
parent; parent gets has_retry_attempt_child=true; span carries
- Single-attempt happy path: ONE llm_attempt span under the active
parent; parent gets has_attempt_child=true; span carries
role / gen_ai.system=anthropic / gen_ai.request.model / http.status_code.
- Multi-attempt retry path: N siblings under one parent.
- Error-attempt: status=ERROR, http.status_code, error.type.
Expand All @@ -28,8 +28,8 @@
from opentelemetry import context as context_api
from opentelemetry import trace
from opentelemetry.instrumentation.anthropic.retry_handler import (
_FR_HAS_RETRY_ATTEMPT_CHILD_KEY,
_FR_RETRY_ATTEMPT_SPAN_NAME,
_FR_HAS_ATTEMPT_CHILD_KEY,
_FR_LLM_ATTEMPT_SPAN_NAME_PREFIX,
_async_send_wrapper,
_has_wrappable_symbol,
_is_installed_for_test,
Expand All @@ -38,6 +38,7 @@
uninstrument_retry_emitter,
)
from opentelemetry.instrumentation.fortifyroot import (
clear_attempt_counters_for_test,
is_framework_owned,
register_framework_attempt,
retry_registry,
Expand Down Expand Up @@ -72,12 +73,14 @@ def fresh_tracer():
@pytest.fixture(autouse=True)
def reset_state():
retry_registry._reset_for_test()
clear_attempt_counters_for_test()
try:
uninstrument_retry_emitter()
except Exception:
pass
yield
retry_registry._reset_for_test()
clear_attempt_counters_for_test()
try:
uninstrument_retry_emitter()
except Exception:
Expand Down Expand Up @@ -116,7 +119,18 @@ def _make_response(status_code: int = 200, request_id: str = "req-abc",

def _retry_spans(exporter: InMemorySpanExporter):
return [s for s in exporter.get_finished_spans()
if s.name == _FR_RETRY_ATTEMPT_SPAN_NAME]
if s.name.startswith(f"{_FR_LLM_ATTEMPT_SPAN_NAME_PREFIX}.attempt_")]


def _assert_attempt_sequence(spans):
by_number = sorted(
spans,
key=lambda s: int(s.attributes.get("fortifyroot.attempt.number") or -1),
)
for expected, span in enumerate(by_number, start=1):
assert span.name == f"{_FR_LLM_ATTEMPT_SPAN_NAME_PREFIX}.attempt_{expected}"
assert span.attributes.get("fortifyroot.attempt.number") == expected
assert span.attributes.get("fortifyroot.attempt.is_retry") is (expected > 1)


# ---------------------------------------------------------------------------
Expand Down Expand Up @@ -212,8 +226,9 @@ def test_single_attempt_emits_one_span_with_marker(fresh_tracer):
retry_spans = _retry_spans(exporter)
assert len(retry_spans) == 1
rs = retry_spans[0]
_assert_attempt_sequence(retry_spans)
assert rs.parent.span_id == parent_exported.context.span_id
assert rs.attributes.get("fortifyroot.span.role") == "retry_attempt"
assert rs.attributes.get("fortifyroot.span.role") == "llm_attempt"
assert rs.attributes.get("gen_ai.system") == "Anthropic"
assert rs.attributes.get("gen_ai.request.model") == "claude-haiku-4-5"
assert rs.attributes.get("gen_ai.operation.name") == "chat"
Expand All @@ -225,7 +240,7 @@ def test_single_attempt_emits_one_span_with_marker(fresh_tracer):
assert rs.attributes.get("gen_ai.usage.output_tokens") == 4
assert rs.attributes.get("server.address") == "api.anthropic.com"
assert rs.attributes.get("server.port") == 443
assert parent_exported.attributes.get(_FR_HAS_RETRY_ATTEMPT_CHILD_KEY) is True
assert parent_exported.attributes.get(_FR_HAS_ATTEMPT_CHILD_KEY) is True


# ---------------------------------------------------------------------------
Expand All @@ -249,6 +264,7 @@ def test_three_attempts_share_parent(fresh_tracer):
parent_exported = next(s for s in spans if s.name == "anthropic.chat")
retry_spans = _retry_spans(exporter)
assert len(retry_spans) == 3
_assert_attempt_sequence(retry_spans)

parent_ids = {s.parent.span_id for s in retry_spans}
assert parent_ids == {parent_exported.context.span_id}
Expand Down Expand Up @@ -299,10 +315,10 @@ def test_marker_set_AFTER_first_attempt_not_at_parent_creation(fresh_tracer):
tracer, exporter, _ = fresh_tracer
request = _make_request()
parent = tracer.start_span("anthropic.chat")
assert _FR_HAS_RETRY_ATTEMPT_CHILD_KEY not in dict(parent.attributes or {})
assert _FR_HAS_ATTEMPT_CHILD_KEY not in dict(parent.attributes or {})
with trace.use_span(parent, end_on_exit=False):
_sync_send_wrapper(lambda *a, **kw: _make_response(200), None, (request,), {})
assert dict(parent.attributes or {}).get(_FR_HAS_RETRY_ATTEMPT_CHILD_KEY) is True
assert dict(parent.attributes or {}).get(_FR_HAS_ATTEMPT_CHILD_KEY) is True
parent.end()


Expand All @@ -313,7 +329,7 @@ def test_marker_NOT_set_when_no_attempts_fire(fresh_tracer):
parent_exported = next(
s for s in exporter.get_finished_spans() if s.name == "anthropic.chat"
)
assert _FR_HAS_RETRY_ATTEMPT_CHILD_KEY not in (parent_exported.attributes or {})
assert _FR_HAS_ATTEMPT_CHILD_KEY not in (parent_exported.attributes or {})


# ---------------------------------------------------------------------------
Expand Down Expand Up @@ -531,7 +547,7 @@ def explode():
parent_exported = next(
s for s in exporter.get_finished_spans() if s.name == "anthropic.chat"
)
assert _FR_HAS_RETRY_ATTEMPT_CHILD_KEY not in (parent_exported.attributes or {})
assert _FR_HAS_ATTEMPT_CHILD_KEY not in (parent_exported.attributes or {})


def test_non_2xx_response_with_usage_in_body_populates_usage_tokens(fresh_tracer):
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,10 @@
from opentelemetry import trace
from opentelemetry.instrumentation.bedrock.version import __version__
from opentelemetry.instrumentation.fortifyroot import (
FR_HAS_ATTEMPT_CHILD_KEY,
is_framework_owned,
llm_attempt_attributes,
next_llm_attempt,
)
from opentelemetry.semconv_ai import SUPPRESS_LANGUAGE_MODEL_INSTRUMENTATION_KEY
from opentelemetry.trace import SpanKind, Status, StatusCode
Expand All @@ -67,10 +70,10 @@


# ST-10 §4.4 / §4.5 constants.
_FR_RETRY_ATTEMPT_SPAN_NAME = "fortifyroot.bedrock.retry_attempt"
_FR_SPAN_ROLE_KEY = "fortifyroot.span.role"
_FR_SPAN_ROLE_RETRY_ATTEMPT = "retry_attempt"
_FR_HAS_RETRY_ATTEMPT_CHILD_KEY = "fortifyroot.span.has_retry_attempt_child"
_FR_LLM_ATTEMPT_SPAN_NAME_PREFIX = "fortifyroot.bedrock"
_FR_SPAN_ROLE_LLM_ATTEMPT = "llm_attempt"
_FR_HAS_ATTEMPT_CHILD_KEY = FR_HAS_ATTEMPT_CHILD_KEY

# Event-name patterns. ``*`` matches any operation under bedrock-runtime.
_BEFORE_SEND_PATTERN = "before-send.bedrock-runtime.*"
Expand Down Expand Up @@ -120,9 +123,9 @@ def _resolve_parent_span() -> Optional["trace.Span"]:

def _set_parent_marker(parent_span: "trace.Span") -> None:
try:
parent_span.set_attribute(_FR_HAS_RETRY_ATTEMPT_CHILD_KEY, True)
parent_span.set_attribute(_FR_HAS_ATTEMPT_CHILD_KEY, True)
except Exception:
logger.debug("failed to set has_retry_attempt_child on parent", exc_info=True)
logger.debug("failed to set has_attempt_child on parent", exc_info=True)


def _operation_from_event_name(event_name: str) -> str:
Expand Down Expand Up @@ -199,11 +202,15 @@ def _start_attempt_span(
event_name: str,
parent_span: "trace.Span",
) -> "trace.Span":
span_name, attempt_number, is_retry = next_llm_attempt(
parent_span,
_FR_LLM_ATTEMPT_SPAN_NAME_PREFIX,
)
attrs: dict[str, Any] = {
_FR_SPAN_ROLE_KEY: _FR_SPAN_ROLE_RETRY_ATTEMPT,
"gen_ai.system": "AWS", # matches existing Bedrock instrumentor
"gen_ai.operation.name": _operation_from_event_name(event_name),
}
attrs.update(llm_attempt_attributes(attempt_number, is_retry))
model = _model_from_request(request)
if model:
attrs["gen_ai.request.model"] = model
Expand All @@ -212,7 +219,7 @@ def _start_attempt_span(
tracer = trace.get_tracer(__name__, __version__, _tracer_provider)
parent_ctx = trace.set_span_in_context(parent_span)
span = tracer.start_span(
_FR_RETRY_ATTEMPT_SPAN_NAME,
span_name,
kind=SpanKind.CLIENT,
attributes=attrs,
context=parent_ctx,
Expand Down Expand Up @@ -271,7 +278,7 @@ def _finalize_success(span: "trace.Span", http_response: Any, parsed: Any) -> No
span.set_attribute("error.type", err_type)
span.set_status(Status(StatusCode.ERROR, f"http {status_code}"))
except Exception:
logger.debug("failed to set success attrs on bedrock retry_attempt", exc_info=True)
logger.debug("failed to set success attrs on bedrock llm_attempt", exc_info=True)


def _finalize_error(span: "trace.Span", exception: BaseException) -> None:
Expand All @@ -292,7 +299,7 @@ def _finalize_error(span: "trace.Span", exception: BaseException) -> None:
pass
span.set_status(Status(StatusCode.ERROR, str(exception)))
except Exception:
logger.debug("failed to set error attrs on bedrock retry_attempt", exc_info=True)
logger.debug("failed to set error attrs on bedrock llm_attempt", exc_info=True)


def _before_send_hook(event_name: Optional[str] = None, request: Any = None, **_kwargs) -> None:
Expand Down Expand Up @@ -476,12 +483,12 @@ def install_event_hooks_on_client(client: Any, tracer_provider=None) -> None:
events.register(
_BEFORE_SEND_PATTERN,
_before_send_hook,
unique_id="fortifyroot.bedrock.retry_attempt.before-send",
unique_id="fortifyroot.bedrock.llm_attempt.before-send",
)
events.register(
_RESPONSE_RECEIVED_PATTERN,
_response_received_hook,
unique_id="fortifyroot.bedrock.retry_attempt.response-received",
unique_id="fortifyroot.bedrock.llm_attempt.response-received",
)
except Exception:
logger.warning(
Expand Down Expand Up @@ -510,14 +517,14 @@ def uninstall_event_hooks_on_client(client: Any) -> None:
try:
events.unregister(
_BEFORE_SEND_PATTERN,
unique_id="fortifyroot.bedrock.retry_attempt.before-send",
unique_id="fortifyroot.bedrock.llm_attempt.before-send",
)
except Exception:
pass
try:
events.unregister(
_RESPONSE_RECEIVED_PATTERN,
unique_id="fortifyroot.bedrock.retry_attempt.response-received",
unique_id="fortifyroot.bedrock.llm_attempt.response-received",
)
except Exception:
pass
Expand All @@ -535,8 +542,8 @@ def _reset_tracer_provider_for_test() -> None:
__all__ = [
"install_event_hooks_on_client",
"uninstall_event_hooks_on_client",
"_FR_RETRY_ATTEMPT_SPAN_NAME",
"_FR_HAS_RETRY_ATTEMPT_CHILD_KEY",
"_FR_LLM_ATTEMPT_SPAN_NAME_PREFIX",
"_FR_HAS_ATTEMPT_CHILD_KEY",
"_before_send_hook",
"_response_received_hook",
]
Loading
Loading