ARGO-5808 automate performance data ingestion

automation/README.md

@@ -39,6 +39,7 @@ Submit an ingestion job for a tenant:
 
 **Options:**
 - `-c /path/to/config.yml` - Specify config file (default: `.config.yml`)
+- `--performance` - Enable ingestion of performance data. By default is not enabled
 - `--no-verify` - Skip verification of remote endpoints like AMS
 - `--dry-run` - Preview what would be submitted without executing
 - `--log-level DEBUG` - Adjust logging verbosity

automation/argo_automator

@@ -16,8 +16,8 @@ from argo_config import ArgoConfig
 from check_readiness import check_readiness
 from init_ams import init_ams
 from init_compute_engine import init_compute_engine
-from init_mongo import init_mongo
 from init_desy import init_desy
+from init_mongo import init_mongo
 from init_node_registry import init_node_registry
 
 REQUEST_TIMEOUT = 30
@@ -198,8 +198,9 @@ class ArgoAutomator:
                 self.executor.submit(self.job_init_ams, tenant_id, tenant_name)
                 return
             if job_name == JobName.INIT_COMPUTE_ENGINE.value:
+                performance=props.get("performance")
                 self.executor.submit(
-                    self.job_init_compute_engine, tenant_id, tenant_name
+                    self.job_init_compute_engine, tenant_id, tenant_name, performance
                 )
                 return
 
@@ -359,7 +360,7 @@ class ArgoAutomator:
         except Exception as e:
             logger.exception(f"job failed for tenant {tenant_name}: {e}")
 
-    def job_init_compute_engine(self, tenant_id: str, tenant_name: str):
+    def job_init_compute_engine(self, tenant_id: str, tenant_name: str, performance: bool):
         """Job placeholder to init compute engine"""
         try:
 
@@ -399,7 +400,7 @@ class ArgoAutomator:
                 "Initialising Compute Engine",
             )
 
-            job_done = init_compute_engine(self.config, tenant_id, tenant_name)
+            job_done = init_compute_engine(self.config, tenant_id, tenant_name, performance)
 
             if job_done:
                 self.mon_api.update_status(
@@ -534,3 +535,4 @@ def main():
 
 if __name__ == "__main__":
     main()
+

automation/argo_config.py

@@ -22,6 +22,8 @@ def __init__(self, path: str):
         self.automation = automation
         self.tenants = tenants
         self.run = run
+        self.performance_db_url = automation.get("performance_db_url")
+        self.performance_db_token = automation.get("performance_db_token")
         self.node_registry_url = automation.get("node_registry_url")
         self.node_registry_token = automation.get("node_registry_token")
         self.ams_endpoint = automation.get("ams_endpoint")

automation/argo_web_api.py

@@ -128,7 +128,6 @@ def get_topology_feed(
 
         return response.json().get("data")
 
-
     def get_topology(
         self,
         tenant_id: str,
@@ -152,9 +151,7 @@ def get_topology(
 
         except requests.exceptions.HTTPError as e:
             if e.response.status_code == 404:
-                logger.warning(
-                    f"tenant: {tenant_name} ({tenant_id}) - has no topology"
-                )
+                logger.warning(f"tenant: {tenant_name} ({tenant_id}) - has no topology")
                 return []
             else:
                 raise
@@ -333,43 +330,7 @@ def create_default_report(
             else:
                 raise
 
-
-    def create_topology_groups(
-            self,
-            tenant_id: str,
-            tenant_name: str,
-            tenant_access_token: str,
-            payload: object,
-        ):
-        """Http call to web-api to create topology groups"""
-        logger.debug(
-            f"tenant: {tenant_name} ({tenant_id}) - web-api creating topology groups..."
-        )
-
-        url = f"https://{self.config.web_api_endpoint}/api/v2/topology/groups"
-        headers = {
-            "x-api-key": tenant_access_token,
-            "Accept": "application/json",
-        }
-        try:
-            response = requests.post(
-                url, json=payload, headers=headers, timeout=REQUEST_TIMEOUT
-            )
-            response.raise_for_status()
-            logger.info(
-                f"tenant: {tenant_name} ({tenant_id}) - web-api topology groups created"
-            )
-
-        except requests.exceptions.HTTPError as e:
-            if e.response.status_code == 409:
-                logger.warning(
-                    f"tenant: {tenant_name} ({tenant_id}) - web-api topology groups already exist for specific date"
-                )
-                return
-            else:
-                raise
-
-
+
     def create_topology_service_types(
         self,
         tenant_id: str,
@@ -452,7 +413,9 @@ def create_topology_groups(
             f"tenant: {tenant_name} ({tenant_id}) - web-api creating groups..."
         )
 
-        url = f"https://{self.config.web_api_endpoint}/api/v2/topology/groups?force=true"
+        url = (
+            f"https://{self.config.web_api_endpoint}/api/v2/topology/groups?force=true"
+        )
         headers = {
             "x-api-key": tenant_access_token,
             "Accept": "application/json",
@@ -475,7 +438,6 @@ def create_topology_groups(
             else:
                 raise
 
-
     def update_ready_state(
         self,
         tenant_id: str,

automation/check_readiness.py

@@ -112,5 +112,5 @@ def check_readiness(config: ArgoConfig, tenant_id: str, tenant_name: str) -> boo
     }
 
     # update the payload to web-api
-    result = web_api.update_ready_state(tenant_id, tenant_name, payload)
+    web_api.update_ready_state(tenant_id, tenant_name, payload)
     return True

automation/config.yml.example

@@ -8,6 +8,8 @@ automation:
   ams_event_project: ARGO-MON-AUTOMATION-PROJECT
   ams_event_subscription: events-subscription
   ams_admin_token: admin-s3cr3t
+  performance_db_url:  https://perf.example.foo
+  perfomrance_db_token: perf-s3cret
 
   oidc_token_url: localhost:8080
   oidc_client_id: tenant.status.integration.service

automation/init_ams.py

@@ -1,4 +1,5 @@
 import logging
+
 import requests
 from argo_ams_library import (AmsServiceException, AmsUser, AmsUserProject,
                               ArgoMessagingService)
@@ -9,17 +10,22 @@
 
 REQUEST_TIMEOUT = 30
 
+
 # use http request to create component user because ams library doesn't support it
-def create_ams_component_account(ams_endpoint: str, ams_token: str, username: str, email: str, project: str, role: str,  component: str, component_project: str): 
+def create_ams_component_account(
+    ams_endpoint: str,
+    ams_token: str,
+    username: str,
+    email: str,
+    project: str,
+    role: str,
+    component: str,
+    component_project: str,
+):
 
     payload = {
         "email": email,
-        "projects": [
-            {
-                "project": project,
-                "roles": [role]
-            }
-        ],
+        "projects": [{"project": project, "roles": [role]}],
     }
 
     if component and component_project:
@@ -32,13 +38,11 @@ def create_ams_component_account(ams_endpoint: str, ams_token: str, username: st
         "Accept": "application/json",
     }
     try:
-            response = requests.post(
-                url, json=payload, headers=headers, timeout=REQUEST_TIMEOUT
-            )
-            response.raise_for_status()
-            logger.info(
-                f"ams user: {username} created for project: {project}"
-            )
+        response = requests.post(
+            url, json=payload, headers=headers, timeout=REQUEST_TIMEOUT
+        )
+        response.raise_for_status()
+        logger.info(f"ams user: {username} created for project: {project}")
 
     except requests.exceptions.HTTPError as e:
         if e.response.status_code == 409:
@@ -93,21 +97,30 @@ def init_ams(
 
     for username, role, component, component_admin in user_roles:
         try:
+            if component and component_admin: 
+                user = create_ams_component_account(
+                    config.ams_endpoint,
+                    config.ams_admin_token,
+                    username,
+                    config.argo_ops_email,
+                    tenant_name,
+                    role,
+                    component,
+                    component_admin,
+                )
 
-            user = create_ams_component_account(config.ams_endpoint, config.ams_admin_token, username, config.argo_ops_email, tenant_name, role, component, component_admin)
-
-            user = ams.create_user(
-                AmsUser(
-                    name=username,
-                    projects=[AmsUserProject(project=tenant_name, roles=[role])],
-                    email=config.argo_ops_email,
+                user = ams.create_user(
+                    AmsUser(
+                        name=username,
+                        projects=[AmsUserProject(project=tenant_name, roles=[role])],
+                        email=config.argo_ops_email,
+                    )
                 )
-            )
 
-            if user:
-                logger.info(f"ams project {tenant_name} - user created: {username}")
-                if role == "consumer" and username == consumer_username:
-                    config.set_tenant_ams_access(tenant_id, tenant_name, user.token)
+                if user:
+                    logger.info(f"ams project {tenant_name} - user created: {username}")
+                    if role == "consumer" and username == consumer_username:
+                        config.set_tenant_ams_access(tenant_id, tenant_name, user.token)
 
         except AmsServiceException as e:
             if e.code == 409:

automation/init_compute_engine.py

@@ -1,10 +1,11 @@
 import json
 import logging
 import os
+
 from jinja2 import Environment, FileSystemLoader
+
 from argo_config import ArgoConfig
 from argo_web_api import ArgoWebApi
-
 from init_ingest import run_ingest
 
 logger = logging.getLogger(__name__)
@@ -38,6 +39,7 @@ def init_compute_engine(
     config: ArgoConfig,
     tenant_id: str,
     tenant_name: str,
+    performance: bool,
 ) -> bool:
     """Initialise compute engine users"""
 
@@ -153,7 +155,9 @@ def init_compute_engine(
         config=config,
         tenant_name=tenant_name,
         tenant_ams_token=tenant.get("ams_token"),
+        tenant_mon_api_token=tenant.get("web_api_token"),
         dry_run=False,
+        performance=performance,
         verify=VERIFY,
     )