feat(system-metrics): Change gc.get_count with gc.get_stats with corresponding metrics.

Signed-off-by: Cagri Yonca <cagri@ibm.com>

Author: CagriYonca

src/instana/agent/host.py

@@ -99,13 +99,17 @@ def reset(self) -> None:
 
     def is_timed_out(self) -> bool:
         """
-        If we haven't heard from the Instana host agent in 60 seconds, this
-        method will return True.
+        If we haven't heard from the Instana host agent within the timeout
+        window, this method will return True.  The window is the larger of
+        60 seconds or twice the configured poll_rate so that high poll_rate
+        values (e.g. 120 s) do not cause spurious resets.
         @return: Boolean
         """
-        if self.last_seen and self.can_send:
+        if self.last_seen and self.can_send():
+            poll_rate = getattr(getattr(self, "options", None), "poll_rate", 1)
+            timeout_threshold = max(60, poll_rate * 2)
             diff = datetime.now() - self.last_seen
-            if diff.seconds > 60:
+            if diff.seconds > timeout_threshold:
                 return True
         return False
 
@@ -276,31 +280,40 @@ def report_data_payload(
     ) -> Optional[Response]:
         """
         Used to report collection payload to the host agent.  This can be metrics, spans and snapshot data.
+        When there is nothing to send (no spans, profiles, or metrics), a lightweight HEAD heartbeat
+        is sent instead so that the host-agent timeout detection continues to work correctly even
+        when poll_rate is larger than the 60-second timeout window.
         """
         response = None
+        data_was_sent = False
         try:
             # Report spans (if any)
             response = self.report_spans(payload)
-
             if response is not None and 200 <= response.status_code <= 204:
                 self.last_seen = datetime.now()
+                data_was_sent = True
 
             # Report profiles (if any)
             response = self.report_profiles(payload)
-
             if response is not None and 200 <= response.status_code <= 204:
                 self.last_seen = datetime.now()
+                data_was_sent = True
 
             # Report metrics
             response = self.report_metrics(payload)
-
             if response is not None and 200 <= response.status_code <= 204:
                 self.last_seen = datetime.now()
+                data_was_sent = True
 
                 if response.status_code == 200 and len(response.content) > 2:
-                    # The host agent returned something indicating that is has a request for us that we
-                    # need to process.
+                    # The host agent returned something indicating that it has a request for us
+                    # that we need to process.
                     self.handle_agent_tasks(json.loads(response.content)[0])
+
+            # Nothing was sent this cycle — send a heartbeat HEAD request so that
+            # is_timed_out() keeps working correctly at high poll_rate values.
+            if not data_was_sent:
+                self._send_heartbeat()
         except requests.exceptions.ConnectionError:
             pass
         except urllib3.exceptions.MaxRetryError:
@@ -312,6 +325,37 @@ def report_data_payload(
             )
         return response
 
+    def _send_heartbeat(self) -> None:
+        """
+        Sends a lightweight HEAD request to the host agent data endpoint to confirm
+        connectivity and update last_seen when no metrics, spans or profiles were sent.
+
+        Guards:
+        - Only runs in "good2go" state — during wait4init the FSM polling already
+          performs HEAD checks via is_agent_ready(), so a second HEAD is unnecessary.
+        - announce_data must be set (skipped silently during pre-announce).
+        - A local copy of announce_data is taken before the request to avoid a race
+          condition where reset() sets announce_data=None mid-flight.
+        """
+        try:
+            announce_data = self.announce_data  # local copy — avoids race with reset()
+            if announce_data is None:
+                return
+            if self.machine.fsm.current != "good2go":
+                return
+            response = self.client.head(self.__data_url(), timeout=0.8)
+            if response is not None and 200 <= response.status_code <= 204:
+                self.last_seen = datetime.now()
+        except requests.exceptions.ConnectionError:
+            pass
+        except urllib3.exceptions.MaxRetryError:
+            pass
+        except Exception as exc:
+            logger.debug(
+                f"_send_heartbeat: connection error ({type(exc)})",
+                exc_info=True,
+            )
+
     def report_metrics(self, payload: dict[str, Any]) -> Optional[Response]:
         metrics = payload.get("metrics", [])
         if len(metrics) > 0 and len(metrics.get("plugins", [])) > 0:

src/instana/collector/helpers/runtime.py

@@ -44,10 +44,11 @@ def __init__(
         self.previous = DictionaryOfStan()
         self.previous_rusage = get_resource_usage()
 
-        if gc.isenabled():
-            self.previous_gc_count = gc.get_count()
-        else:
-            self.previous_gc_count = None
+        # Initialise to None so the first collect_metrics call establishes the
+        # baseline snapshot and reports all-zero deltas.  Any GC activity that
+        # occurs between process start and the first collection is intentionally
+        # excluded — we only report incremental deltas from the first poll onward.
+        self.previous_gc_stats = None
 
     def collect_metrics(self, **kwargs: Dict[str, Any]) -> List[Dict[str, Any]]:
         plugin_data = dict()
@@ -82,6 +83,7 @@ def _collect_runtime_metrics(
             return
 
         """ Collect up and return the runtime metrics """
+        rusage = self.previous_rusage
         try:
             rusage = get_resource_usage()
             if gc.isenabled():
@@ -232,52 +234,27 @@ def _collect_runtime_metrics(
 
     def _collect_gc_metrics(self, plugin_data, with_snapshot):
         try:
-            gc_count = gc.get_count()
-            gc_threshold = gc.get_threshold()
-
-            self.apply_delta(
-                gc_count[0],
-                self.previous["data"]["metrics"]["gc"],
-                plugin_data["data"]["metrics"]["gc"],
-                "collect0",
-                with_snapshot,
-            )
-            self.apply_delta(
-                gc_count[1],
-                self.previous["data"]["metrics"]["gc"],
-                plugin_data["data"]["metrics"]["gc"],
-                "collect1",
-                with_snapshot,
-            )
-            self.apply_delta(
-                gc_count[2],
-                self.previous["data"]["metrics"]["gc"],
-                plugin_data["data"]["metrics"]["gc"],
-                "collect2",
-                with_snapshot,
-            )
-
-            self.apply_delta(
-                gc_threshold[0],
-                self.previous["data"]["metrics"]["gc"],
-                plugin_data["data"]["metrics"]["gc"],
-                "threshold0",
-                with_snapshot,
-            )
-            self.apply_delta(
-                gc_threshold[1],
-                self.previous["data"]["metrics"]["gc"],
-                plugin_data["data"]["metrics"]["gc"],
-                "threshold1",
-                with_snapshot,
-            )
-            self.apply_delta(
-                gc_threshold[2],
-                self.previous["data"]["metrics"]["gc"],
-                plugin_data["data"]["metrics"]["gc"],
-                "threshold2",
-                with_snapshot,
-            )
+            gc_stats = gc.get_stats()
+            if self.previous_gc_stats is None:
+                # First call: establish baseline, report all-zero deltas so the
+                # snapshot payload carries zeros rather than the cumulative counts
+                # that accumulated since process start (which are not meaningful
+                # as deltas).
+                self.previous_gc_stats = gc_stats
+                return
+
+            # Use a plain dict as the staging target so that accessing it never
+            # auto-creates keys in plugin_data (DictionaryOfStan creates keys on
+            # read, which would leave an empty "gc": {} even when nothing changed).
+            staging: dict = {}
+            prev_gc = self.previous["data"]["metrics"]["gc"]
+            for i, (stat, prev_stat) in enumerate(zip(gc_stats, self.previous_gc_stats)):
+                for key in ("collections", "collected", "uncollectable"):
+                    delta = stat[key] - prev_stat.get(key, 0)
+                    self.apply_delta(delta, prev_gc, staging, f"{key}{i}", with_snapshot)
+            if staging:
+                plugin_data["data"]["metrics"]["gc"].update(staging)
+            self.previous_gc_stats = gc_stats
         except Exception:
             logger.debug("_collect_gc_metrics", exc_info=True)
 

tests/agent/test_host.py

@@ -334,8 +334,8 @@ def test_is_timed_out(
         assert not agent.is_timed_out()
 
         agent.last_seen = datetime.datetime.now() - datetime.timedelta(minutes=5)
-        agent.can_send = True
-        assert agent.is_timed_out()
+        with patch.object(agent, "can_send", return_value=True):
+            assert agent.is_timed_out()
 
     def test_can_send_test_env(
         self,

tests/collector/helpers/test_collector_runtime.py

@@ -27,7 +27,7 @@ def test_default_while_gc_disabled(self) -> None:
 
         gc.disable()
         helper = RuntimeHelper(collector=HostCollector(HostAgent()))
-        assert helper.previous_gc_count is None
+        assert helper.previous_gc_stats is None
 
     def test_collect_metrics(self) -> None:
         response = self.helper.collect_metrics()
@@ -65,8 +65,73 @@ def test_collect_runtime_snapshot_webhook(self) -> None:
     def test_collect_gc_metrics(self) -> None:
         plugin_data = self.helper.collect_metrics()
 
+        # First call establishes the baseline (previous_gc_stats was None); no
+        # data is written yet.
         self.helper._collect_gc_metrics(plugin_data[0], True)
-        assert len(self.helper.previous["data"]["metrics"]["gc"]) == 6
+        assert self.helper.previous_gc_stats is not None
+
+        # Second call computes deltas from the baseline and writes them.
+        self.helper._collect_gc_metrics(plugin_data[0], True)
+        gc_metrics = self.helper.previous["data"]["metrics"]["gc"]
+        for i in range(3):
+            for key in ("collections", "collected", "uncollectable"):
+                assert f"{key}{i}" in gc_metrics
+
+    def test_collect_gc_metrics_reports_delta_between_polls(self) -> None:
+        """GC metrics must be deltas between successive polls, not kumulatif values."""
+        # Simulate first poll: previous_gc_stats set to known baseline
+        baseline = [
+            {"collections": 100, "collected": 200, "uncollectable": 0},
+            {"collections": 10, "collected": 50, "uncollectable": 0},
+            {"collections": 1, "collected": 5, "uncollectable": 0},
+        ]
+        self.helper.previous_gc_stats = baseline
+
+        # Simulate gc.get_stats() returning incremented counts
+        after = [
+            {"collections": 103, "collected": 206, "uncollectable": 0},
+            {"collections": 11, "collected": 53, "uncollectable": 0},
+            {"collections": 1, "collected": 5, "uncollectable": 0},
+        ]
+
+        plugin_data = [{"data": {"metrics": {"gc": {}}}}]
+
+        import unittest.mock as mock
+        with mock.patch("gc.get_stats", return_value=after):
+            self.helper._collect_gc_metrics(plugin_data[0], True)
+
+        gc_metrics = plugin_data[0]["data"]["metrics"]["gc"]
+        # Gen 0: collections delta = 3, collected delta = 6
+        assert gc_metrics["collections0"] == 3
+        assert gc_metrics["collected0"] == 6
+        assert gc_metrics["uncollectable0"] == 0
+        # Gen 1: collections delta = 1, collected delta = 3
+        assert gc_metrics["collections1"] == 1
+        assert gc_metrics["collected1"] == 3
+        # Gen 2: no change — delta = 0, still reported because with_snapshot=True
+        assert gc_metrics["collections2"] == 0
+        assert gc_metrics["collected2"] == 0
+
+        # previous_gc_stats must be updated to the latest snapshot
+        assert self.helper.previous_gc_stats == after
+
+    def test_collect_gc_metrics_no_change_not_sent_without_snapshot(self) -> None:
+        """When nothing changed and with_snapshot=False, gc metrics must be empty."""
+        same = [
+            {"collections": 50, "collected": 100, "uncollectable": 0},
+            {"collections": 5, "collected": 20, "uncollectable": 0},
+            {"collections": 0, "collected": 0, "uncollectable": 0},
+        ]
+        self.helper.previous_gc_stats = same
+
+        plugin_data = [{"data": {"metrics": {"gc": {}}}}]
+
+        import unittest.mock as mock
+        with mock.patch("gc.get_stats", return_value=same):
+            self.helper._collect_gc_metrics(plugin_data[0], False)
+
+        # All deltas are 0 and with_snapshot=False → nothing written
+        assert plugin_data[0]["data"]["metrics"]["gc"] == {}
 
     def test_collect_runtime_metrics(self) -> None:
         """Test that _collect_runtime_metrics properly collects metrics"""