Remove telemetry, SmartDashboard, and indirect entrypoint functionality

.readthedocs.yaml

@@ -21,13 +21,11 @@ build:
         fi
     pre_create_environment:
       - git clone --depth 1 https://github.com/CrayLabs/SmartRedis.git smartredis
-      - git clone --depth 1 https://github.com/CrayLabs/SmartDashboard.git smartdashboard
     post_create_environment:
       - python -m pip install .[dev,docs]
       - cd smartredis; python -m pip install .
       - cd smartredis/doc; doxygen Doxyfile_c; doxygen Doxyfile_cpp; doxygen Doxyfile_fortran
       - ln -s smartredis/examples ./examples
-      - cd smartdashboard; python -m pip install .
     pre_build:
       - pip install typing_extensions==4.8.0
       - pip install pydantic==1.10.13

conftest.py

@@ -26,7 +26,6 @@
 
 from __future__ import annotations
 
-import asyncio
 from collections import defaultdict
 from dataclasses import dataclass
 import json
@@ -43,18 +42,15 @@
 import uuid
 import warnings
 from subprocess import run
-import time
 
 import psutil
 import pytest
 
 import smartsim
 from smartsim import Experiment
 from smartsim._core.launcher.dragon.dragonConnector import DragonConnector
-from smartsim._core.launcher.dragon.dragonLauncher import DragonLauncher
 from smartsim._core.config import CONFIG
 from smartsim._core.config.config import Config
-from smartsim._core.utils.telemetry.telemetry import JobEntity
 from smartsim.database import Orchestrator
 from smartsim.entity import Model
 from smartsim.error import SSConfigError, SSInternalError
@@ -706,143 +702,7 @@ def config() -> Config:
     return CONFIG
 
 
-class MockSink:
-    """Telemetry sink that writes console output for testing purposes"""
-
-    def __init__(self, delay_ms: int = 0) -> None:
-        self._delay_ms = delay_ms
-        self.num_saves = 0
-        self.args: t.Any = None
-
-    async def save(self, *args: t.Any) -> None:
-        """Save all arguments as console logged messages"""
-        self.num_saves += 1
-        if self._delay_ms:
-            # mimic slow collection....
-            delay_s = self._delay_ms / 1000
-            await asyncio.sleep(delay_s)
-        self.args = args
-
-
-@pytest.fixture
-def mock_sink() -> t.Type[MockSink]:
-    return MockSink
-
-
-@pytest.fixture
-def mock_con() -> t.Callable[[int, int], t.Iterable[t.Any]]:
-    """Generates mock db connection telemetry"""
-
-    def _mock_con(min: int = 1, max: int = 254) -> t.Iterable[t.Any]:
-        for i in range(min, max):
-            yield [
-                {"addr": f"127.0.0.{i}:1234", "id": f"ABC{i}"},
-                {"addr": f"127.0.0.{i}:2345", "id": f"XYZ{i}"},
-            ]
-
-    return _mock_con
-
-
-@pytest.fixture
-def mock_mem() -> t.Callable[[int, int], t.Iterable[t.Any]]:
-    """Generates mock db memory usage telemetry"""
-
-    def _mock_mem(min: int = 1, max: int = 1000) -> t.Iterable[t.Any]:
-        for i in range(min, max):
-            yield {
-                "total_system_memory": 1000 * i,
-                "used_memory": 1111 * i,
-                "used_memory_peak": 1234 * i,
-            }
-
-    return _mock_mem
-
-
-@pytest.fixture
-def mock_redis() -> t.Callable[..., t.Any]:
-    def _mock_redis(
-        conn_side_effect=None,
-        mem_stats=None,
-        client_stats=None,
-        coll_side_effect=None,
-    ):
-        """Generate a mock object for the redis.Redis contract"""
-
-        class MockConn:
-            def __init__(self, *args: t.Any, **kwargs: t.Any) -> None:
-                if conn_side_effect is not None:
-                    conn_side_effect()
-
-            async def info(self, *args: t.Any, **kwargs: t.Any) -> t.Dict[str, t.Any]:
-                if coll_side_effect:
-                    await coll_side_effect()
-
-                if mem_stats:
-                    return next(mem_stats)
-                return {
-                    "total_system_memory": "111",
-                    "used_memory": "222",
-                    "used_memory_peak": "333",
-                }
-
-            async def client_list(
-                self, *args: t.Any, **kwargs: t.Any
-            ) -> t.Dict[str, t.Any]:
-                if coll_side_effect:
-                    await coll_side_effect()
-
-                if client_stats:
-                    return next(client_stats)
-                return {"addr": "127.0.0.1", "id": "111"}
-
-            async def ping(self):
-                return True
-
-        return MockConn
-
-    return _mock_redis
-
-
-class MockCollectorEntityFunc(t.Protocol):
-    @staticmethod
-    def __call__(
-        host: str = "127.0.0.1",
-        port: int = 6379,
-        name: str = "",
-        type: str = "",
-        telemetry_on: bool = False,
-    ) -> "JobEntity": ...
-
-
-@pytest.fixture
-def mock_entity(test_dir: str) -> MockCollectorEntityFunc:
-    def _mock_entity(
-        host: str = "127.0.0.1",
-        port: int = 6379,
-        name: str = "",
-        type: str = "",
-        telemetry_on: bool = False,
-    ) -> "JobEntity":
-        test_path = pathlib.Path(test_dir)
-
-        entity = JobEntity()
-        entity.name = name if name else str(uuid.uuid4())
-        entity.status_dir = str(test_path / entity.name)
-        entity.type = type
-        entity.telemetry_on = True
-        entity.collectors = {
-            "client": "",
-            "client_count": "",
-            "memory": "",
-        }
-        entity.config = {
-            "host": host,
-            "port": str(port),
-        }
-        entity.telemetry_on = telemetry_on
-        return entity
 
-    return _mock_entity
 
 
 class CountingCallable:

doc/api/smartsim_api.rst

@@ -27,7 +27,6 @@ Experiment
    Experiment.reconnect_orchestrator
    Experiment.preview
    Experiment.summary
-   Experiment.telemetry
 
 .. autoclass:: Experiment
    :show-inheritance:
@@ -368,7 +367,6 @@ Orchestrator
    Orchestrator.set_max_clients
    Orchestrator.set_max_message_size
    Orchestrator.set_db_conf
-   Orchestrator.telemetry
    Orchestrator.checkpoint_file
    Orchestrator.batch
 

doc/changelog.md

@@ -1,20 +1,21 @@
 # Changelog
 
-Listed here are the changes between each release of SmartSim,
-SmartRedis and SmartDashboard.
+Listed here are the changes between each release of SmartSim and SmartRedis.
 
 Jump to:
 - {ref}`SmartRedis changelog<smartredis-changelog>`
-- {ref}`SmartDashboard changelog<smartdashboard-changelog>`
 
 ## SmartSim
 
 To be released at some point in the future
 
 Description
 
+- Removed telemetry functionality, LaunchedManifest tracking
+  classes, and SmartDashboard integration
 - Update copyright headers from 2021-2024 to 2021-2025 across the entire codebase
-- Python 3.12 is now supported; where available, installed TensorFlow version is now 2.16.2, PyTorch is 2.7.1.
+- Python 3.12 is now supported; where available, installed TensorFlow version
+  is now 2.16.2, PyTorch is 2.7.1.
 - Drop Python 3.9 support
 - Terminate LSF and LSB support
 - Implement workaround for Tensorflow that allows RedisAI to build with GCC-14
@@ -23,20 +24,47 @@ Description
 
 Detailed Notes
 
-- Copyright headers have been updated from "2021-2024" to "2021-2025" across 271 files
-  including Python source files, configuration files, documentation, tests, Docker files,
-  shell scripts, and other supporting files to reflect the new year.
+- Removed telemetry functionality, LaunchedManifest tracking
+  system, and SmartDashboard integration.
+  This includes complete removal of the telemetry monitor and collection system,
+  telemetry configuration classes (`TelemetryConfiguration`,
+  `ExperimentTelemetryConfiguration`), all telemetry-related API methods
+  (`Experiment.telemetry`, `Orchestrator.telemetry`), telemetry collectors and
+  sinks, and the `watchdog` dependency. Also removed SmartDashboard integration
+  and CLI plugin, along with the indirect entrypoint launching mechanism.
+  Additionally removed the `LaunchedManifest`, `_LaunchedManifestMetadata`, and
+  `LaunchedManifestBuilder` classes that were used for telemetry data collection
+  during entity launches. Simplified the controller launch workflow by removing
+  telemetry metadata tracking and launch manifest serialization. Cleaned up the
+  `serialize.py` module by removing orphaned telemetry functions (80% code
+  reduction), preserving only essential type definitions. Updated all test files
+  to remove LaunchedManifest dependencies and deleted obsolete telemetry test
+  files. The core `Manifest` class for entity organization remains unchanged,
+  maintaining backward compatibility for entity management while removing the
+  telemetry overhead. Enhanced the metadata directory system to use a centralized
+  `.smartsim/metadata/` structure for job output files with entity-specific
+  subdirectories (`ensemble/{name}`, `model/{name}`, `database/{name}`) and
+  proper symlink management.
+  Added new `CONFIG` path helpers (`smartsim_base_dir`, `metadata_subdir`,
+  `dragon_default_subdir`, `dragon_logs_subdir`) that now return
+  `pathlib.Path` instances to provide a single source of truth for SmartSim's
+  hidden workspace directories and Dragon launcher log locations.
+  ([SmartSim-PR789](https://github.com/CrayLabs/SmartSim/pull/789))
+- Copyright headers have been updated from "2021-2024" to "2021-2025" across
+  271 files including Python source files, configuration files, documentation,
+  tests, Docker files, shell scripts, and other supporting files to reflect the
+  new year.
   ([SmartSim-PR790](https://github.com/CrayLabs/SmartSim/pull/790))
-- Python 3.12 is now supported. TensorFlow 2.16.2 and PyTorch 2.7.1 library files
-  are installed as part of `smart build` process when available. On Mac, ONNX runtime
-  1.22.0 is now installed, together with ONNX 1.16.
+- Python 3.12 is now supported. TensorFlow 2.16.2 and PyTorch 2.7.1 library
+  files are installed as part of `smart build` process when available. On Mac,
+  ONNX runtime 1.22.0 is now installed, together with ONNX 1.16.
   ([SmartSim-PR785](https://github.com/CrayLabs/SmartSim/pull/785))
 - Python 3.9 will not be supported anymore, the last stable version of SmartSim
   with support for Python 3.9 will be 0.8.
   ([SmartSim-PR781](https://github.com/CrayLabs/SmartSim/pull/781))
 - After the supercomputer Summit was decommissioned, a decision was made to
-  terminate SmartSim's support of the LSF launcher and LSB scheduler. If
-  this impacts your work, please contact us.
+  terminate SmartSim's support of the LSF launcher and LSB scheduler. If this
+  impacts your work, please contact us.
   ([SmartSim-PR780](https://github.com/CrayLabs/SmartSim/pull/780))
 - Fix typos in the `train_surrogate` tutorial documentation.
   ([SmartSim-PR758](https://github.com/CrayLabs/SmartSim/pull/758))
@@ -1104,12 +1132,3 @@ Description:
 ```{include} ../smartredis/doc/changelog.md
 :start-line: 2
 ```
-
-------------------------------------------------------------------------
-
-(smartdashboard-changelog)=
-## SmartDashboard
-
-```{include} ../smartdashboard/doc/changelog.md
-:start-line: 2
-```

doc/index.rst

@@ -55,12 +55,6 @@
    sr_advanced_topics
    api/smartredis_api
 
-.. toctree::
-   :maxdepth: 2
-   :caption: SmartDashboard
-
-   smartdashboard
-
 .. toctree::
    :maxdepth: 2
    :caption: Reference

docker/docs/dev/Dockerfile

@@ -48,12 +48,6 @@ RUN git clone https://github.com/CrayLabs/SmartRedis.git --branch develop --dept
     && python -m pip install . \
     && rm -rf ~/.cache/pip
 
-# Install smartdashboard
-RUN git clone https://github.com/CrayLabs/SmartDashboard.git --branch develop --depth=1 smartdashboard \
-    && cd smartdashboard \
-    && python -m pip install . \
-    && rm -rf ~/.cache/pip
-
 # Install docs dependencies and SmartSim
 RUN NO_CHECKS=1 SMARTSIM_SUFFIX=dev python -m pip install .[docs]
 

setup.py

@@ -176,7 +176,6 @@ class BuildError(Exception):
         "GitPython<=3.1.43",
         "protobuf<=3.20.3",
         "jinja2>=3.1.2",
-        "watchdog>4,<5",
         "pydantic>2",
         "pyzmq>=25.1.2",
         "pygithub>=2.3.0",

smartsim/_core/_cli/cli.py

@@ -62,7 +62,9 @@ def __init__(self, menu: t.List[MenuItemConfig]) -> None:
         )
 
         self.register_menu_items(menu)
-        self.register_menu_items([plugin() for plugin in plugins])
+        # Register plugin menu items (currently empty since all plugins were removed)
+        plugin_items = [plugin() for plugin in plugins]
+        self.register_menu_items(plugin_items)
 
     def execute(self, cli_args: t.List[str]) -> int:
         if len(cli_args) < 2:

smartsim/_core/_cli/plugin.py

@@ -38,18 +38,5 @@ def process_execute(
     return process_execute
 
 
-def dashboard() -> MenuItemConfig:
-    return MenuItemConfig(
-        "dashboard",
-        (
-            "Start the SmartSim dashboard to monitor experiment output from a "
-            "graphical user interface. This requires that the SmartSim Dashboard "
-            "Package be installed. For more infromation please visit "
-            "https://github.com/CrayLabs/SmartDashboard"
-        ),
-        dynamic_execute("smartdashboard", "Dashboard"),
-        is_plugin=True,
-    )
-
-
-plugins = (dashboard,)
+# No plugins currently available
+plugins: t.Tuple[t.Callable[[], MenuItemConfig], ...] = ()

smartsim/_core/_cli/validate.py

@@ -150,7 +150,6 @@ def test_install(
     with_onnx: bool,
 ) -> None:
     exp = Experiment("ValidationExperiment", exp_path=location, launcher="local")
-    exp.telemetry.disable()
     port = find_free_port() if port is None else port
 
     with _make_managed_local_orc(exp, port) as client: