ref(pyreqwest): Move crumbs to integration

sentry_sdk/integrations/pyreqwest.py

@@ -9,6 +9,7 @@
 from sentry_sdk.traces import StreamedSpan
 from sentry_sdk.tracing import BAGGAGE_HEADER_NAME
 from sentry_sdk.tracing_utils import (
+    add_http_breadcrumb,
     add_http_request_source,
     add_sentry_baggage_to_headers,
     has_span_streaming_enabled,
@@ -67,15 +68,19 @@
     original_method = getattr(cls, method_name)
 
     def sentry_patched_method(self: "Any", *args: "Any", **kwargs: "Any") -> "Any":
-        if not getattr(self, "_sentry_instrumented", False):
-            integration = sentry_sdk.get_client().get_integration(PyreqwestIntegration)
-            if integration is not None:
-                self.with_middleware(middleware)
-                try:
-                    self._sentry_instrumented = True
-                except (TypeError, AttributeError):
-                    # In case the instance itself is immutable or doesn't allow extra attributes
-                    pass
+        integration = sentry_sdk.get_client().get_integration(PyreqwestIntegration)
+
+        if getattr(self, "_sentry_instrumented", False) or integration is None:
+            return original_method(self, *args, **kwargs)
+
+        self.with_middleware(middleware)
+
+        try:
+            self._sentry_instrumented = True
+        except (TypeError, AttributeError):
+            # In case the instance itself is immutable or doesn't allow extra attributes
+            pass
+
         return original_method(self, *args, **kwargs)
 
     setattr(cls, method_name, sentry_patched_method)
@@ -151,11 +156,20 @@
 
 
 async def sentry_async_middleware(
-    request: "Request", next_handler: "Next"
+    request: "Request",
+    next_handler: "Next",
 ) -> "Response":
     if sentry_sdk.get_client().get_integration(PyreqwestIntegration) is None:
         return await next_handler.run(request)
 
+    method = request.method
+    parsed_url = None
+    with capture_internal_exceptions():
+        # This needs to be done early because the URL is no longer accessible
+        # after the request has been sent
+        parsed_url = parse_url(str(request.url), sanitize=False)
+
+    response = None
     with _sentry_pyreqwest_span(request) as span:
         response = await next_handler.run(request)
         if isinstance(span, StreamedSpan):
@@ -167,6 +181,23 @@
         elif span is not None:
             span.set_http_status(response.status)
 
+    if response is not None:
+        breadcrumb_data = {
+            SPANDATA.HTTP_METHOD: method,
+            SPANDATA.HTTP_STATUS_CODE: response.status,
+        }
+
+        if parsed_url and should_send_default_pii():
+            breadcrumb_data.update(
+                {
+                    "url": parsed_url.url,
+                    SPANDATA.HTTP_QUERY: parsed_url.query,
+                    SPANDATA.HTTP_FRAGMENT: parsed_url.fragment,
+                }
+            )
+
+        add_http_breadcrumb(response.status, breadcrumb_data)
+
     return response
 
 
@@ -176,6 +207,14 @@
     if sentry_sdk.get_client().get_integration(PyreqwestIntegration) is None:
         return next_handler.run(request)
 
+    method = request.method
+    parsed_url = None
+    with capture_internal_exceptions():
+        # This needs to be done early because the URL is no longer accessible
+        # after the request has been sent
+        parsed_url = parse_url(str(request.url), sanitize=False)
+
+    response = None
     with _sentry_pyreqwest_span(request) as span:
         response = next_handler.run(request)
         if isinstance(span, StreamedSpan):
@@ -187,4 +226,21 @@
         elif span is not None:
             span.set_http_status(response.status)
 
+    if response is not None:
+        breadcrumb_data = {
+            SPANDATA.HTTP_METHOD: method,
+            SPANDATA.HTTP_STATUS_CODE: response.status,
+        }
+
+        if parsed_url and should_send_default_pii():
+            breadcrumb_data.update(
+                {
+                    "url": parsed_url.url,
+                    SPANDATA.HTTP_QUERY: parsed_url.query,
+                    SPANDATA.HTTP_FRAGMENT: parsed_url.fragment,
+                }
+            )
+
+        add_http_breadcrumb(response.status, breadcrumb_data)
+
     return response

sentry_sdk/tracing_utils.py

@@ -228,7 +228,10 @@ def add_http_breadcrumb(status_code: "Optional[int]", data: "dict[str, Any]") ->
 def maybe_create_breadcrumbs_from_span(
     scope: "sentry_sdk.Scope", span: "sentry_sdk.tracing.Span"
 ) -> None:
-    if span.op == OP.HTTP_CLIENT and span.origin not in ("auto.http.aiohttp",):
+    if span.op == OP.HTTP_CLIENT and span.origin not in (
+        "auto.http.aiohttp",
+        "auto.http.pyreqwest",
+    ):
         level = None
         status_code = span._data.get(SPANDATA.HTTP_STATUS_CODE)
         if status_code:

tests/integrations/pyreqwest/test_pyreqwest.py

@@ -11,10 +11,10 @@
 from pyreqwest.simple.sync_request import pyreqwest_get as sync_pyreqwest_get
 
 import sentry_sdk
-from sentry_sdk import start_transaction
+from sentry_sdk import capture_message, start_transaction
 from sentry_sdk.consts import MATCH_ALL, SPANDATA
 from sentry_sdk.integrations.pyreqwest import PyreqwestIntegration
-from tests.conftest import get_free_port
+from tests.conftest import ApproxDict, get_free_port
 
 
 class PyreqwestMockHandler(BaseHTTPRequestHandler):
@@ -956,3 +956,246 @@ def fake_start_span(*args, **kwargs):
         assert SPANDATA.CODE_NAMESPACE in data
         assert SPANDATA.CODE_FILEPATH in data
     assert SPANDATA.CODE_FUNCTION in data
+
+
+@pytest.mark.parametrize("send_default_pii", [True, False])
+@pytest.mark.parametrize("span_streaming", [True, False])
+def test_crumb_capture(
+    sentry_init,
+    capture_events,
+    server_port,
+    send_default_pii,
+    span_streaming,
+):
+    def before_breadcrumb(crumb, hint):
+        crumb["data"]["extra"] = "foo"
+        return crumb
+
+    sentry_init(
+        integrations=[PyreqwestIntegration()],
+        before_breadcrumb=before_breadcrumb,
+        send_default_pii=send_default_pii,
+        trace_lifecycle="stream" if span_streaming else "static",
+    )
+
+    url = f"http://localhost:{server_port}/hello?q=test#frag"
+
+    events = capture_events()
+
+    client = SyncClientBuilder().build()
+    response = client.get(url).build().send()
+    assert response.status == 200
+
+    capture_message("Testing!")
+
+    (event,) = events
+
+    crumb = event["breadcrumbs"]["values"][0]
+    assert crumb["type"] == "http"
+    assert crumb["category"] == "httplib"
+
+    expected = {
+        SPANDATA.HTTP_METHOD: "GET",
+        SPANDATA.HTTP_STATUS_CODE: 200,
+        "extra": "foo",
+    }
+    if send_default_pii:
+        expected["url"] = f"http://localhost:{server_port}/hello"
+        expected[SPANDATA.HTTP_QUERY] = "q=test"
+        expected[SPANDATA.HTTP_FRAGMENT] = "frag"
+
+    assert crumb["data"] == ApproxDict(expected)
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("send_default_pii", [True, False])
+async def test_async_crumb_capture(
+    sentry_init,
+    capture_events,
+    server_port,
+    send_default_pii,
+):
+    sentry_init(
+        integrations=[PyreqwestIntegration()],
+        send_default_pii=send_default_pii,
+    )
+
+    url = f"http://localhost:{server_port}/hello?q=test#frag"
+
+    events = capture_events()
+
+    # Ensure the isolation scope contextvar is set before pyreqwest spawns
+    # its middleware on a separate asyncio Task. Without this, the child task
+    # lazily creates its own isolation scope, and breadcrumbs added there
+    # don't propagate back to this task's context.
+    sentry_sdk.get_isolation_scope()
+
+    with sentry_sdk.start_transaction():
+        async with ClientBuilder().build() as client:
+            response = await client.get(url).build().send()
+            assert response.status == 200
+
+        capture_message("Testing!")
+
+    (event,) = events
+
+    crumb = event["breadcrumbs"]["values"][0]
+    assert crumb["type"] == "http"
+    assert crumb["category"] == "httplib"
+
+    expected = {
+        SPANDATA.HTTP_METHOD: "GET",
+        SPANDATA.HTTP_STATUS_CODE: 200,
+    }
+    if send_default_pii:
+        expected["url"] = f"http://localhost:{server_port}/hello"
+        expected[SPANDATA.HTTP_QUERY] = "q=test"
+        expected[SPANDATA.HTTP_FRAGMENT] = "frag"
+
+    assert crumb["data"] == ApproxDict(expected)
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("send_default_pii", [True, False])
+async def test_async_crumb_capture_span_streaming(
+    sentry_init,
+    capture_events,
+    server_port,
+    send_default_pii,
+):
+    sentry_init(
+        integrations=[PyreqwestIntegration()],
+        send_default_pii=send_default_pii,
+        trace_lifecycle="stream",
+    )
+
+    url = f"http://localhost:{server_port}/hello?q=test#frag"
+
+    events = capture_events()
+
+    with sentry_sdk.traces.start_span(name="segment"):
+        async with ClientBuilder().build() as client:
+            response = await client.get(url).build().send()
+            assert response.status == 200
+
+            capture_message("Testing!")
+
+    (event,) = events
+
+    crumb = event["breadcrumbs"]["values"][0]
+    assert crumb["type"] == "http"
+    assert crumb["category"] == "httplib"
+
+    expected = {
+        SPANDATA.HTTP_METHOD: "GET",
+        SPANDATA.HTTP_STATUS_CODE: 200,
+    }
+    if send_default_pii:
+        expected["url"] = f"http://localhost:{server_port}/hello"
+        expected[SPANDATA.HTTP_QUERY] = "q=test"
+        expected[SPANDATA.HTTP_FRAGMENT] = "frag"
+
+    assert crumb["data"] == ApproxDict(expected)
+
+
+@pytest.mark.parametrize(
+    "status_code,level",
+    [
+        (200, None),
+        (301, None),
+        (403, "warning"),
+        (405, "warning"),
+        (500, "error"),
+    ],
+)
+def test_crumb_capture_client_error(
+    sentry_init,
+    capture_events,
+    server_port,
+    status_code,
+    level,
+):
+    sentry_init(
+        integrations=[PyreqwestIntegration()],
+    )
+
+    url = f"http://localhost:{server_port}/status/{status_code}"
+
+    events = capture_events()
+
+    with sentry_sdk.start_transaction():
+        client = SyncClientBuilder().build()
+        response = client.get(url).build().send()
+        assert response.status == status_code
+
+        capture_message("Testing!")
+
+    (event,) = events
+
+    crumb = event["breadcrumbs"]["values"][0]
+    assert crumb["type"] == "http"
+    assert crumb["category"] == "httplib"
+
+    if level is None:
+        assert "level" not in crumb
+    else:
+        assert crumb["level"] == level
+
+    assert crumb["data"] == ApproxDict(
+        {
+            SPANDATA.HTTP_METHOD: "GET",
+            SPANDATA.HTTP_STATUS_CODE: status_code,
+        }
+    )
+
+
+@pytest.mark.parametrize(
+    "status_code,level",
+    [
+        (200, None),
+        (301, None),
+        (403, "warning"),
+        (405, "warning"),
+        (500, "error"),
+    ],
+)
+def test_crumb_capture_client_error_span_streaming(
+    sentry_init,
+    capture_events,
+    server_port,
+    status_code,
+    level,
+):
+    sentry_init(
+        integrations=[PyreqwestIntegration()],
+        trace_lifecycle="stream",
+    )
+
+    url = f"http://localhost:{server_port}/status/{status_code}"
+
+    events = capture_events()
+
+    with sentry_sdk.traces.start_span(name="segment"):
+        client = SyncClientBuilder().build()
+        response = client.get(url).build().send()
+        assert response.status == status_code
+
+        capture_message("Testing!")
+
+    (event,) = events
+
+    crumb = event["breadcrumbs"]["values"][0]
+    assert crumb["type"] == "http"
+    assert crumb["category"] == "httplib"
+
+    if level is None:
+        assert "level" not in crumb
+    else:
+        assert crumb["level"] == level
+
+    assert crumb["data"] == ApproxDict(
+        {
+            SPANDATA.HTTP_METHOD: "GET",
+            SPANDATA.HTTP_STATUS_CODE: status_code,
+        }
+    )