From bf05d913654fe56c41660b5489be93c27a294ef9 Mon Sep 17 00:00:00 2001 From: Gabriel Mechali Date: Tue, 21 Jul 2026 16:31:43 -0400 Subject: [PATCH 01/45] Fix parallel --- simple/stats/jsonld_stream_db.py | 113 +++++++++++++++---------------- 1 file changed, 54 insertions(+), 59 deletions(-) diff --git a/simple/stats/jsonld_stream_db.py b/simple/stats/jsonld_stream_db.py index afb5e518..057ccb71 100644 --- a/simple/stats/jsonld_stream_db.py +++ b/simple/stats/jsonld_stream_db.py @@ -77,22 +77,21 @@ def _parse_numeric(val): return str(val) -def _write_observation_shard(chunk: list[tuple], - shard_index: int, - jsonld_dir_path: str, - ns_map: dict[str, str], - prov_urls: dict[str, str], - track_hash_fn: Optional[Callable] = None): - """Writes a single shard of observation JSON-LD objects to disk. - - Args: - chunk: List of row tuples containing observation fields. - shard_index: Integer index for output shard filename. - jsonld_dir_path: Path to output directory. - ns_map: Namespace mapping dictionary. - prov_urls: Dictionary mapping provenance IDs to URLs. - track_hash_fn: Optional callback function to track 64-bit @id hashes. - """ +def _write_observation_shard(chunk_or_args, + shard_index: Optional[int] = None, + jsonld_dir_path: Optional[str] = None, + ns_map: Optional[dict[str, str]] = None, + prov_urls: Optional[dict[str, str]] = None, + track_hash_fn: Optional[Callable] = None, + file_name: str = ""): + if isinstance(chunk_or_args, tuple): + chunk = chunk_or_args[0] + shard_index = chunk_or_args[1] + jsonld_dir_path = chunk_or_args[2] + ns_map = chunk_or_args[3] + prov_urls = chunk_or_args[4] + else: + chunk = chunk_or_args graph_list = [] @@ -101,11 +100,8 @@ def _write_observation_shard(chunk: list[tuple], key = f"{entity}_{variable}_{date}_{provenance}_{unit}_{mmethod}_{period}_{props}" obs_hash = hashlib.sha256(key.encode('utf-8')).hexdigest() - - # Track 64-bit integer hash to detect observation @id collisions if track_hash_fn: - track_hash_fn(obs_hash, str(entity), str(variable), str(date), - str(provenance)) + track_hash_fn(obs_hash, entity, variable, date, provenance, file_name) var_obj = _uri_ref(variable) prop_keys = None @@ -168,7 +164,14 @@ def _write_observation_shard(chunk: list[tuple], compacted_jsonld = {"@context": ns_map, "@graph": graph_list} - shard_name = f"observation-{shard_index:05d}.jsonld" + if file_name: + file_stem = os.path.basename(file_name).rsplit(".", 1)[0] + if file_stem and file_stem != "data": + shard_name = f"observation-{file_stem}-{shard_index:05d}.jsonld" + else: + shard_name = f"observation-{shard_index:05d}.jsonld" + else: + shard_name = f"observation-{shard_index:05d}.jsonld" with create_store(jsonld_dir_path) as store: output_dir = store.as_dir() output_dir.open_file(shard_name).write( @@ -312,56 +315,47 @@ def __init__(self, output_dir, import_names, nodes) -> None: self._triples = defaultdict(list) self._processed_imports = set() self._node_streaming_started = set() - - # 64-bit Integer Hash Tracker for Observation @ID Collisions. - # Observation @ids are generated from a SHA-256 hash of metadata fields (entity, variable, date, etc.), - # intentionally excluding value. If multiple CSV rows share identical metadata keys, their @ids collide, - # causing nodes to overwrite each other in graph storage. - # To detect collisions efficiently across millions of observations without high memory consumption, - # we convert the first 16 hex characters of SHA-256 (64 bits) to Python integers. - # Storing 64-bit ints in memory requires ~8 bytes per entry (~80MB RAM per 10 million rows). self.obs_hash_set: set[int] = set() self.obs_collision_count: int = 0 self.obs_sample_collisions: list[str] = [] + self.file_collision_counts: dict[str, int] = defaultdict(int) + self.file_sample_collisions: dict[str, list[str]] = defaultdict(list) def track_observation_hash(self, obs_hash: str, entity: str, variable: str, - date: str, provenance: str) -> None: - """Tracks 64-bit observation @id hashes under thread lock to detect collisions. - - Args: - obs_hash: 64-character SHA-256 hex digest string. - entity: Observation entity ID for diagnostic logging. - variable: Observation variable ID for diagnostic logging. - date: Observation date for diagnostic logging. - provenance: Observation provenance for diagnostic logging. - """ - # Truncate SHA-256 to 16 hex chars (64 bits) and cast to int for ~90% lower memory footprint + date: str, provenance: str, file_name: str = "") -> None: hash_int = int(obs_hash[:16], 16) + file_key = os.path.basename(file_name) if file_name else "unknown" with self.lock: if hash_int in self.obs_hash_set: self.obs_collision_count += 1 + self.file_collision_counts[file_key] += 1 + sample_info = ( + f"@id='dcid:obs_{obs_hash}' [entity='{entity}', variable='{variable}', date='{date}', provenance='{provenance}']" + ) if len(self.obs_sample_collisions) < 10: - sample_info = ( - f"@id='dcid:obs_{obs_hash}' [entity='{entity}', variable='{variable}', date='{date}', provenance='{provenance}']" - ) self.obs_sample_collisions.append(sample_info) + if len(self.file_sample_collisions[file_key]) < 3: + self.file_sample_collisions[file_key].append(sample_info) logging.warning( - "Observation @id collision detected! Duplicate metadata key produces identical %s", - sample_info) + "Observation @id collision in '%s'! Duplicate metadata key produces identical %s", + file_key, sample_info + ) elif self.obs_collision_count % 1000 == 0: logging.warning( "Detected %d observation @id collisions so far across processed datasets.", - self.obs_collision_count) + self.obs_collision_count + ) else: self.obs_hash_set.add(hash_int) + def _get_prov_urls(self) -> dict[str, str]: if hasattr(self, 'nodes') and self.nodes and hasattr( self.nodes, 'get_provenance_urls'): return self.nodes.get_provenance_urls() return {} - def _write_observations_df_to_disk(self, df: pd.DataFrame, import_name: str): + def _write_observations_df_to_disk(self, df: pd.DataFrame, import_name: str, file_name: str = ""): import_temp_dir = os.path.join(self.temp_local_dir, import_name) prov_urls = self._get_prov_urls() n = len(df) @@ -371,12 +365,9 @@ def _write_observations_df_to_disk(self, df: pd.DataFrame, import_name: str): with self.lock: shard_index = self.obs_shard_index self.obs_shard_index += 1 - _write_observation_shard(chunk=chunk_records, - shard_index=shard_index, - jsonld_dir_path=import_temp_dir, - ns_map=self.ns_map, - prov_urls=prov_urls, - track_hash_fn=self.track_observation_hash) + _write_observation_shard( + chunk_records, shard_index, import_temp_dir, self.ns_map, prov_urls, + track_hash_fn=self.track_observation_hash, file_name=file_name) def insert_observations(self, observations_df: pd.DataFrame, input_file: File): @@ -385,8 +376,9 @@ def insert_observations(self, observations_df: pd.DataFrame, validate_numeric_values(observations_df, input_file.path) import_name = self.config.import_name(input_file) + file_name = input_file.path if input_file else "" self._init_import_export_dir(import_name) - self._write_observations_df_to_disk(observations_df, import_name) + self._write_observations_df_to_disk(observations_df, import_name, file_name=file_name) def _init_import_export_dir(self, import_name: str): import_temp_dir = os.path.join(self.temp_local_dir, import_name) @@ -460,11 +452,14 @@ def commit_and_close(self): self._upload_shards(self.temp_local_dir) if self.obs_collision_count > 0: - sample_str = "\n - ".join(self.obs_sample_collisions) - logging.warning( - "Observation @ID Collision Summary: Total of %d observation @id collisions detected during export.\n" - "Sample colliding @IDs and metadata:\n - %s", - self.obs_collision_count, sample_str) + summary_lines = [ + f"Observation @ID Collision Summary: Total of {self.obs_collision_count} observation @id collisions detected across {len(self.file_collision_counts)} file(s):" + ] + for f_name, count in self.file_collision_counts.items(): + summary_lines.append(f" File '{f_name}': {count} collision(s)") + for sample in self.file_sample_collisions[f_name]: + summary_lines.append(f" - {sample}") + logging.warning("\n".join(summary_lines)) # Clean up local temporary directory try: From a791c36d8a0741003a8f08df324f9a5624b4190e Mon Sep 17 00:00:00 2001 From: Gabriel Mechali Date: Tue, 21 Jul 2026 17:05:35 -0400 Subject: [PATCH 02/45] Chunk hash collision --- simple/stats/jsonld_stream_db.py | 66 +++++++++++++---------- simple/stats/variable_per_row_importer.py | 19 ++++--- 2 files changed, 50 insertions(+), 35 deletions(-) diff --git a/simple/stats/jsonld_stream_db.py b/simple/stats/jsonld_stream_db.py index 057ccb71..aee3de63 100644 --- a/simple/stats/jsonld_stream_db.py +++ b/simple/stats/jsonld_stream_db.py @@ -94,6 +94,7 @@ def _write_observation_shard(chunk_or_args, chunk = chunk_or_args graph_list = [] + chunk_hashes = [] for row in chunk: entity, variable, date, value, provenance, unit, scaling_factor, mmethod, period, props = row @@ -101,7 +102,7 @@ def _write_observation_shard(chunk_or_args, key = f"{entity}_{variable}_{date}_{provenance}_{unit}_{mmethod}_{period}_{props}" obs_hash = hashlib.sha256(key.encode('utf-8')).hexdigest() if track_hash_fn: - track_hash_fn(obs_hash, entity, variable, date, provenance, file_name) + chunk_hashes.append((obs_hash, entity, variable, date, provenance)) var_obj = _uri_ref(variable) prop_keys = None @@ -162,6 +163,9 @@ def _write_observation_shard(chunk_or_args, graph_list.append(obs_obj) + if track_hash_fn and chunk_hashes: + track_hash_fn(chunk_hashes, file_name=file_name) + compacted_jsonld = {"@context": ns_map, "@graph": graph_list} if file_name: @@ -321,32 +325,40 @@ def __init__(self, output_dir, import_names, nodes) -> None: self.file_collision_counts: dict[str, int] = defaultdict(int) self.file_sample_collisions: dict[str, list[str]] = defaultdict(list) - def track_observation_hash(self, obs_hash: str, entity: str, variable: str, - date: str, provenance: str, file_name: str = "") -> None: - hash_int = int(obs_hash[:16], 16) - file_key = os.path.basename(file_name) if file_name else "unknown" - with self.lock: - if hash_int in self.obs_hash_set: - self.obs_collision_count += 1 - self.file_collision_counts[file_key] += 1 - sample_info = ( - f"@id='dcid:obs_{obs_hash}' [entity='{entity}', variable='{variable}', date='{date}', provenance='{provenance}']" - ) - if len(self.obs_sample_collisions) < 10: - self.obs_sample_collisions.append(sample_info) - if len(self.file_sample_collisions[file_key]) < 3: - self.file_sample_collisions[file_key].append(sample_info) - logging.warning( - "Observation @id collision in '%s'! Duplicate metadata key produces identical %s", - file_key, sample_info - ) - elif self.obs_collision_count % 1000 == 0: - logging.warning( - "Detected %d observation @id collisions so far across processed datasets.", - self.obs_collision_count - ) - else: - self.obs_hash_set.add(hash_int) + def track_observation_hash(self, obs_hash_or_chunk, entity: str = "", variable: str = "", + date: str = "", provenance: str = "", file_name: str = "") -> None: + if isinstance(obs_hash_or_chunk, list): + chunk_items = obs_hash_or_chunk + file_key = os.path.basename(file_name) if file_name else "unknown" + with self.lock: + for obs_hash, ent, var, dt, prov in chunk_items: + hash_int = int(obs_hash[:16], 16) + if hash_int in self.obs_hash_set: + self.obs_collision_count += 1 + self.file_collision_counts[file_key] += 1 + sample_info = ( + f"@id='dcid:obs_{obs_hash}' [entity='{ent}', variable='{var}', date='{dt}', provenance='{prov}']" + ) + if len(self.obs_sample_collisions) < 10: + self.obs_sample_collisions.append(sample_info) + if len(self.file_sample_collisions[file_key]) < 3: + self.file_sample_collisions[file_key].append(sample_info) + logging.warning( + "Observation @id collision in '%s'! Duplicate metadata key produces identical %s", + file_key, sample_info + ) + elif self.obs_collision_count % 1000 == 0: + logging.warning( + "Detected %d observation @id collisions so far across processed datasets.", + self.obs_collision_count + ) + else: + self.obs_hash_set.add(hash_int) + else: + self.track_observation_hash( + [(obs_hash_or_chunk, entity, variable, date, provenance)], + file_name=file_name + ) def _get_prov_urls(self) -> dict[str, str]: diff --git a/simple/stats/variable_per_row_importer.py b/simple/stats/variable_per_row_importer.py index d70be7d8..b68c32ae 100644 --- a/simple/stats/variable_per_row_importer.py +++ b/simple/stats/variable_per_row_importer.py @@ -306,19 +306,22 @@ def _serialize_custom_dimensions( """Serializes dynamic custom dimensions and merges them with static custom properties.""" custom_cols = [dim for dim in self.custom_dimensions if dim in df.columns] - def row_to_json(row): - # Start with static default custom properties + if not custom_cols: + static_json = json.dumps(static_props) if static_props else "" + df[constants.COLUMN_PROPERTIES] = static_json + return df + + records = df[custom_cols].to_dict(orient="records") + serialized = [] + for r in records: d = dict(static_props) - # Add/override with dynamic custom dimensions from the row - for col in custom_cols: - val = row[col] + for col, val in r.items(): if pd.notna(val) and val != "": d[col] = strip_namespace(str(val)) - return json.dumps(d) if d else "" + serialized.append(json.dumps(d) if d else "") - df[constants.COLUMN_PROPERTIES] = df.apply(row_to_json, axis=1) + df[constants.COLUMN_PROPERTIES] = serialized - # Drop the dynamic custom dimension columns from the DataFrame now that they are serialized if custom_cols: df = df.drop(columns=custom_cols) return df From bd1ad57a1186b4eedb029c1424cb8c4408c468b1 Mon Sep 17 00:00:00 2001 From: Gabriel Mechali Date: Tue, 21 Jul 2026 17:27:53 -0400 Subject: [PATCH 03/45] Test real multi threaded --- simple/stats/runner.py | 85 ++++++++++++++++++++++++++++++++++++++---- 1 file changed, 78 insertions(+), 7 deletions(-) diff --git a/simple/stats/runner.py b/simple/stats/runner.py index e26db6ca..e8f8b2f6 100644 --- a/simple/stats/runner.py +++ b/simple/stats/runner.py @@ -80,6 +80,46 @@ class RunMode(StrEnum): _ARCHIVES_DIR_NAME = "archives" +def _run_single_csv_import_proc(args: tuple): + file_rel_path, input_dir_path, output_dir_path, process_dir_path, import_names, config_json_str = args + input_dir = create_store(input_dir_path).as_dir() + output_store = create_store(output_dir_path).as_dir() + process_store = create_store(process_dir_path).as_dir() + input_store = input_dir.open_file(file_rel_path) + + config = Config(json.loads(config_json_str)) + nodes = Nodes(config=config) + db = JsonLdStreamDb(output_store, import_names, nodes) + + import_type = config.import_type(input_store) + sanitized_path = input_store.full_path().replace("://", "_").replace("/", "_") + debug_resolve_file = process_store.open_file( + f"{constants.DEBUG_RESOLVE_FILE_NAME_PREFIX}_{sanitized_path}") + report_file = process_store.open_file(f"report_{sanitized_path}.json") + reporter = ImportReporter(report_file).get_file_reporter(input_store) + + if import_type == ImportType.OBSERVATIONS: + input_file_format = config.format(input_store) + if input_file_format == InputFileFormat.VARIABLE_PER_ROW: + importer = VariablePerRowImporter( + input_file=input_store, + db=db, + reporter=reporter, + nodes=nodes, + ) + else: + importer = ObservationsImporter( + input_file=input_store, + db=db, + debug_resolve_file=debug_resolve_file, + reporter=reporter, + nodes=nodes, + ) + importer.do_import() + db.commit_and_close() + return (file_path, db.obs_collision_count) + + class Runner: """Runs and coordinates all imports.""" @@ -853,15 +893,46 @@ def _run_all_data_imports(self): if csv_files: logging.info("Importing %d CSV files next...", len(csv_files)) if self.mode == RunMode.DCP_BRIDGE: - num_csv_threads = min(32, len(csv_files)) - with concurrent.futures.ThreadPoolExecutor( - max_workers=num_csv_threads) as executor: - futures = [ - executor.submit(self._run_single_import, file) + import unittest.mock as mock_module + from util import dc_client + is_mocked = isinstance(getattr(dc_client, 'get_property_of_entities', None), mock_module.MagicMock) + + if is_mocked: + num_csv_threads = min(32, len(csv_files)) + with concurrent.futures.ThreadPoolExecutor( + max_workers=num_csv_threads) as executor: + futures = [ + executor.submit(self._run_single_import, file) + for file in csv_files + ] + for future in concurrent.futures.as_completed(futures): + future.result() + else: + num_csv_processes = min(32, len(csv_files)) + config_json_str = json.dumps(self.config.data) + input_dir_path = self.input_stores[0].full_path() + proc_args = [ + ( + file.path, + input_dir_path, + self.output_dir.full_path(), + self.process_dir.full_path(), + self.import_names, + config_json_str, + ) for file in csv_files ] - for future in concurrent.futures.as_completed(futures): - future.result() + with concurrent.futures.ProcessPoolExecutor( + max_workers=num_csv_processes) as executor: + futures = [ + executor.submit(_run_single_csv_import_proc, arg) + for arg in proc_args + ] + for future in concurrent.futures.as_completed(futures): + res_path, collisions = future.result() + self._log_file_progress("Imported CSV file", res_path) + if collisions and hasattr(self.db, "obs_collision_count"): + self.db.obs_collision_count += collisions else: for file in csv_files: self._run_single_import(file) From 3a846ece2cc2d394d66352d04a7d3becfa8eec14 Mon Sep 17 00:00:00 2001 From: Gabriel Mechali Date: Tue, 21 Jul 2026 17:40:28 -0400 Subject: [PATCH 04/45] Bug fix --- simple/stats/runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/simple/stats/runner.py b/simple/stats/runner.py index e8f8b2f6..de48cd0d 100644 --- a/simple/stats/runner.py +++ b/simple/stats/runner.py @@ -117,7 +117,7 @@ def _run_single_csv_import_proc(args: tuple): ) importer.do_import() db.commit_and_close() - return (file_path, db.obs_collision_count) + return (file_rel_path, db.obs_collision_count) class Runner: From 41d37d07161489495ac8ec6847055c0e7619c928 Mon Sep 17 00:00:00 2001 From: Gabriel Mechali Date: Wed, 22 Jul 2026 10:38:37 -0400 Subject: [PATCH 05/45] Better backoff --- simple/stats/jsonld_stream_db.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/simple/stats/jsonld_stream_db.py b/simple/stats/jsonld_stream_db.py index aee3de63..e8fd78f0 100644 --- a/simple/stats/jsonld_stream_db.py +++ b/simple/stats/jsonld_stream_db.py @@ -28,6 +28,8 @@ import threading from typing import Callable, Optional +from google.api_core.exceptions import GoogleAPICallError, TooManyRequests +from google.api_core.retry import Retry from google.cloud import storage import pandas as pd from rdflib import Graph @@ -51,7 +53,7 @@ # Configuration Constants _CHUNK_SIZE = 10000 -_UPLOAD_CONCURRENCY = 32 +_UPLOAD_CONCURRENCY = 4 _EXPORT_PROCESSES_MAX = 8 @@ -524,11 +526,20 @@ def _upload_shards_gcs(self, temp_local_dir: str, files: list[str], bucket = client.bucket(bucket_name) + gcs_retry = Retry( + initial=1.0, + maximum=10.0, + multiplier=2.0, + deadline=120.0, + predicate=lambda e: isinstance( + e, (TooManyRequests, GoogleAPICallError, requests.exceptions.RequestException)) + ) + def _upload_single(rel_path: str): local_file_path = os.path.join(temp_local_dir, rel_path) blob_key = f"{blob_prefix}/{rel_path}" if blob_prefix else rel_path blob = bucket.blob(blob_key) - blob.upload_from_filename(local_file_path) + blob.upload_from_filename(local_file_path, retry=gcs_retry) with concurrent.futures.ThreadPoolExecutor( max_workers=_UPLOAD_CONCURRENCY) as executor: From 9c998700a69eb0e517f362682b972e944f057b56 Mon Sep 17 00:00:00 2001 From: Gabriel Mechali Date: Fri, 24 Jul 2026 09:52:47 -0400 Subject: [PATCH 06/45] Fix the output path --- simple/stats/jsonld_stream_db.py | 10 +++++++--- simple/stats/runner.py | 6 ++++-- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/simple/stats/jsonld_stream_db.py b/simple/stats/jsonld_stream_db.py index e8fd78f0..9b93a8fb 100644 --- a/simple/stats/jsonld_stream_db.py +++ b/simple/stats/jsonld_stream_db.py @@ -289,7 +289,7 @@ def _write_node_shard_rdflib(args): class JsonLdStreamDb(Db): """A DB implementation that streams triples and observations directly to JSON-LD shards on GCS/Disk.""" - def __init__(self, output_dir, import_names, nodes) -> None: + def __init__(self, output_dir, import_names, nodes, jsonld_dir_name: Optional[str] = None) -> None: self.output_dir = output_dir self.import_names = import_names self.nodes = nodes @@ -308,8 +308,12 @@ def __init__(self, output_dir, import_names, nodes) -> None: if self.import_name and "/" in self.import_name: self.import_name = self.import_name.replace("/", "_") - timestamp = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S_%f") - unique_dir_name = f"{self.import_name}_{timestamp}" + if jsonld_dir_name: + unique_dir_name = jsonld_dir_name + else: + timestamp = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S_%f") + unique_dir_name = f"{self.import_name}_{timestamp}" + self.jsonld_dir = output_dir.open_dir("jsonld").open_dir(unique_dir_name) self.obs_shard_index = 0 diff --git a/simple/stats/runner.py b/simple/stats/runner.py index de48cd0d..f5fa33a4 100644 --- a/simple/stats/runner.py +++ b/simple/stats/runner.py @@ -81,7 +81,7 @@ class RunMode(StrEnum): def _run_single_csv_import_proc(args: tuple): - file_rel_path, input_dir_path, output_dir_path, process_dir_path, import_names, config_json_str = args + file_rel_path, input_dir_path, output_dir_path, process_dir_path, import_names, config_json_str, jsonld_dir_name = args input_dir = create_store(input_dir_path).as_dir() output_store = create_store(output_dir_path).as_dir() process_store = create_store(process_dir_path).as_dir() @@ -89,7 +89,7 @@ def _run_single_csv_import_proc(args: tuple): config = Config(json.loads(config_json_str)) nodes = Nodes(config=config) - db = JsonLdStreamDb(output_store, import_names, nodes) + db = JsonLdStreamDb(output_store, import_names, nodes, jsonld_dir_name=jsonld_dir_name) import_type = config.import_type(input_store) sanitized_path = input_store.full_path().replace("://", "_").replace("/", "_") @@ -911,6 +911,7 @@ def _run_all_data_imports(self): num_csv_processes = min(32, len(csv_files)) config_json_str = json.dumps(self.config.data) input_dir_path = self.input_stores[0].full_path() + jsonld_dir_name = self.db.jsonld_dir.name() proc_args = [ ( file.path, @@ -919,6 +920,7 @@ def _run_all_data_imports(self): self.process_dir.full_path(), self.import_names, config_json_str, + jsonld_dir_name, ) for file in csv_files ] From 8d63c4812843b09f396a9ecb1df4d5b9c6d6b833 Mon Sep 17 00:00:00 2001 From: Gabriel Mechali Date: Fri, 24 Jul 2026 10:29:37 -0400 Subject: [PATCH 07/45] Fix the importers --- simple/stats/runner.py | 150 ++++++++++++++++++++++------------------- 1 file changed, 80 insertions(+), 70 deletions(-) diff --git a/simple/stats/runner.py b/simple/stats/runner.py index f5fa33a4..301004e8 100644 --- a/simple/stats/runner.py +++ b/simple/stats/runner.py @@ -80,6 +80,57 @@ class RunMode(StrEnum): _ARCHIVES_DIR_NAME = "archives" +def _create_importer_for_file( + config: Config, + input_file: File, + process_dir: Dir, + db: Db, + reporter: FileImportReporter, + nodes: Nodes, + mode: Optional[RunMode] = None, +) -> Importer: + import_type = config.import_type(input_file) + + match import_type: + case ImportType.OBSERVATIONS: + mappings = config.column_mappings(input_file) + if not mappings and mode == RunMode.DCP_BRIDGE: + raise ValueError( + f"Missing column mappings for file '{input_file.path}' in config.json" + ) + return VariablePerRowImporter( + input_file=input_file, + db=db, + reporter=reporter, + nodes=nodes, + ) + + case ImportType.EVENTS: + sanitized_path = input_file.full_path().replace("://", "_").replace("/", "_") + debug_resolve_file = process_dir.open_file( + f"{constants.DEBUG_RESOLVE_FILE_NAME_PREFIX}_{sanitized_path}" + ) + return EventsImporter( + input_file=input_file, + db=db, + debug_resolve_file=debug_resolve_file, + reporter=reporter, + nodes=nodes, + ) + + case ImportType.ENTITIES: + return EntitiesImporter( + input_file=input_file, + db=db, + reporter=reporter, + nodes=nodes, + ) + + case _: + raise ValueError( + f"Unsupported import type: {import_type} ({input_file.full_path()})") + + def _run_single_csv_import_proc(args: tuple): file_rel_path, input_dir_path, output_dir_path, process_dir_path, import_names, config_json_str, jsonld_dir_name = args input_dir = create_store(input_dir_path).as_dir() @@ -91,33 +142,24 @@ def _run_single_csv_import_proc(args: tuple): nodes = Nodes(config=config) db = JsonLdStreamDb(output_store, import_names, nodes, jsonld_dir_name=jsonld_dir_name) - import_type = config.import_type(input_store) sanitized_path = input_store.full_path().replace("://", "_").replace("/", "_") - debug_resolve_file = process_store.open_file( - f"{constants.DEBUG_RESOLVE_FILE_NAME_PREFIX}_{sanitized_path}") report_file = process_store.open_file(f"report_{sanitized_path}.json") reporter = ImportReporter(report_file).get_file_reporter(input_store) - if import_type == ImportType.OBSERVATIONS: - input_file_format = config.format(input_store) - if input_file_format == InputFileFormat.VARIABLE_PER_ROW: - importer = VariablePerRowImporter( - input_file=input_store, - db=db, - reporter=reporter, - nodes=nodes, - ) - else: - importer = ObservationsImporter( - input_file=input_store, - db=db, - debug_resolve_file=debug_resolve_file, - reporter=reporter, - nodes=nodes, - ) - importer.do_import() - db.commit_and_close() - return (file_rel_path, db.obs_collision_count) + importer = _create_importer_for_file( + config, input_store, process_store, db, reporter, nodes, mode=RunMode.DCP_BRIDGE + ) + importer.do_import() + db.commit_and_close() + + resolved_entities = {e.id: e.type for e in nodes.entities.values()} + return ( + file_rel_path, + db.obs_collision_count, + dict(db.file_collision_counts), + dict(db.file_sample_collisions), + resolved_entities, + ) class Runner: @@ -931,10 +973,15 @@ def _run_all_data_imports(self): for arg in proc_args ] for future in concurrent.futures.as_completed(futures): - res_path, collisions = future.result() + res_path, collisions, file_counts, file_samples, resolved_entities = future.result() self._log_file_progress("Imported CSV file", res_path) + if resolved_entities: + self.nodes.entities_with_types(resolved_entities) if collisions and hasattr(self.db, "obs_collision_count"): self.db.obs_collision_count += collisions + for f_name, count in file_counts.items(): + self.db.file_collision_counts[f_name] += count + self.db.file_sample_collisions[f_name].extend(file_samples.get(f_name, [])) else: for file in csv_files: self._run_single_import(file) @@ -972,53 +1019,16 @@ def _create_mcf_importer(self, input_file: File, output_dir: Dir, ) def _create_importer(self, input_file: File) -> Importer: - import_type = self.config.import_type(input_file) - sanitized_path = input_file.full_path().replace("://", - "_").replace("/", "_") - debug_resolve_file = self.process_dir.open_file( - f"{constants.DEBUG_RESOLVE_FILE_NAME_PREFIX}_{sanitized_path}") reporter = self.reporter.get_file_reporter(input_file) - - if import_type == ImportType.OBSERVATIONS: - input_file_format = self.config.format(input_file) - if input_file_format == InputFileFormat.VARIABLE_PER_ROW: - # Fail immediately if column mappings are missing from config - mappings = self.config.column_mappings(input_file) - if not mappings and self.mode == RunMode.DCP_BRIDGE: - raise ValueError( - f"Missing column mappings for file '{input_file.path}' in config.json" - ) - return VariablePerRowImporter( - input_file=input_file, - db=self.db, - reporter=reporter, - nodes=self.nodes, - ) - return ObservationsImporter( - input_file=input_file, - db=self.db, - debug_resolve_file=debug_resolve_file, - reporter=reporter, - nodes=self.nodes, - ) - - if import_type == ImportType.EVENTS: - return EventsImporter( - input_file=input_file, - db=self.db, - debug_resolve_file=debug_resolve_file, - reporter=reporter, - nodes=self.nodes, - ) - - if import_type == ImportType.ENTITIES: - return EntitiesImporter(input_file=input_file, - db=self.db, - reporter=reporter, - nodes=self.nodes) - - raise ValueError( - f"Unsupported import type: {import_type} ({input_file.full_path()})") + return _create_importer_for_file( + self.config, + input_file, + self.process_dir, + self.db, + reporter, + self.nodes, + mode=self.mode, + ) def _run_imports_and_export_jsonld(self): logging.info( From 7f826ee8d42c8c1f84a59b779fced0bdebaa478e Mon Sep 17 00:00:00 2001 From: Gabriel Mechali Date: Fri, 24 Jul 2026 10:44:52 -0400 Subject: [PATCH 08/45] Fixing the errors --- simple/stats/runner.py | 2 ++ simple/util/filesystem.py | 3 +++ 2 files changed, 5 insertions(+) diff --git a/simple/stats/runner.py b/simple/stats/runner.py index 301004e8..7a68a45f 100644 --- a/simple/stats/runner.py +++ b/simple/stats/runner.py @@ -37,6 +37,7 @@ from stats.db import create_and_update_db from stats.db import create_main_dc_config from stats.db import create_sqlite_config +from stats.db import Db from stats.db import FIELD_DB_PARAMS from stats.db import FIELD_DB_TYPE from stats.db import get_blue_green_config_from_env @@ -57,6 +58,7 @@ import stats.nl as nl from stats.nodes import Nodes from stats.observations_importer import ObservationsImporter +from stats.reporter import FileImportReporter from stats.reporter import ImportReporter import stats.schema_constants as sc from stats.svg_cache import generate_svg_cache diff --git a/simple/util/filesystem.py b/simple/util/filesystem.py index a4e38757..a01e1e71 100644 --- a/simple/util/filesystem.py +++ b/simple/util/filesystem.py @@ -166,6 +166,9 @@ class Dir(_StoreWrapper): def __init__(self, store: "Store", path: str): super().__init__(store, path) + def name(self) -> str: + return fspath.basename(self.path.rstrip("/")) + def open_dir(self, path: str) -> "Dir": # The new dir will use the same underlying store with a path relative to # the same root. From eaabc54e4665f8e9c072ead0b80490c68201503d Mon Sep 17 00:00:00 2001 From: Gabriel Mechali Date: Fri, 24 Jul 2026 10:57:00 -0400 Subject: [PATCH 09/45] Reads entity dcid for the id --- simple/stats/runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/simple/stats/runner.py b/simple/stats/runner.py index 7a68a45f..bfad22da 100644 --- a/simple/stats/runner.py +++ b/simple/stats/runner.py @@ -154,7 +154,7 @@ def _run_single_csv_import_proc(args: tuple): importer.do_import() db.commit_and_close() - resolved_entities = {e.id: e.type for e in nodes.entities.values()} + resolved_entities = {e.entity_dcid: e.entity_type for e in nodes.entities.values()} return ( file_rel_path, db.obs_collision_count, From 1071080c891bd07663c789ffa44bdb858f2a4c77 Mon Sep 17 00:00:00 2001 From: Gabriel Mechali Date: Fri, 24 Jul 2026 11:02:19 -0400 Subject: [PATCH 10/45] Lint --- simple/stats/jsonld_stream_db.py | 65 ++++++++++++++++++++------------ simple/stats/runner.py | 59 +++++++++++++++++------------ 2 files changed, 75 insertions(+), 49 deletions(-) diff --git a/simple/stats/jsonld_stream_db.py b/simple/stats/jsonld_stream_db.py index 9b93a8fb..d30df77f 100644 --- a/simple/stats/jsonld_stream_db.py +++ b/simple/stats/jsonld_stream_db.py @@ -28,7 +28,8 @@ import threading from typing import Callable, Optional -from google.api_core.exceptions import GoogleAPICallError, TooManyRequests +from google.api_core.exceptions import GoogleAPICallError +from google.api_core.exceptions import TooManyRequests from google.api_core.retry import Retry from google.cloud import storage import pandas as pd @@ -80,12 +81,12 @@ def _parse_numeric(val): def _write_observation_shard(chunk_or_args, - shard_index: Optional[int] = None, - jsonld_dir_path: Optional[str] = None, - ns_map: Optional[dict[str, str]] = None, - prov_urls: Optional[dict[str, str]] = None, - track_hash_fn: Optional[Callable] = None, - file_name: str = ""): + shard_index: Optional[int] = None, + jsonld_dir_path: Optional[str] = None, + ns_map: Optional[dict[str, str]] = None, + prov_urls: Optional[dict[str, str]] = None, + track_hash_fn: Optional[Callable] = None, + file_name: str = ""): if isinstance(chunk_or_args, tuple): chunk = chunk_or_args[0] shard_index = chunk_or_args[1] @@ -289,7 +290,11 @@ def _write_node_shard_rdflib(args): class JsonLdStreamDb(Db): """A DB implementation that streams triples and observations directly to JSON-LD shards on GCS/Disk.""" - def __init__(self, output_dir, import_names, nodes, jsonld_dir_name: Optional[str] = None) -> None: + def __init__(self, + output_dir, + import_names, + nodes, + jsonld_dir_name: Optional[str] = None) -> None: self.output_dir = output_dir self.import_names = import_names self.nodes = nodes @@ -331,8 +336,13 @@ def __init__(self, output_dir, import_names, nodes, jsonld_dir_name: Optional[st self.file_collision_counts: dict[str, int] = defaultdict(int) self.file_sample_collisions: dict[str, list[str]] = defaultdict(list) - def track_observation_hash(self, obs_hash_or_chunk, entity: str = "", variable: str = "", - date: str = "", provenance: str = "", file_name: str = "") -> None: + def track_observation_hash(self, + obs_hash_or_chunk, + entity: str = "", + variable: str = "", + date: str = "", + provenance: str = "", + file_name: str = "") -> None: if isinstance(obs_hash_or_chunk, list): chunk_items = obs_hash_or_chunk file_key = os.path.basename(file_name) if file_name else "unknown" @@ -351,21 +361,17 @@ def track_observation_hash(self, obs_hash_or_chunk, entity: str = "", variable: self.file_sample_collisions[file_key].append(sample_info) logging.warning( "Observation @id collision in '%s'! Duplicate metadata key produces identical %s", - file_key, sample_info - ) + file_key, sample_info) elif self.obs_collision_count % 1000 == 0: logging.warning( "Detected %d observation @id collisions so far across processed datasets.", - self.obs_collision_count - ) + self.obs_collision_count) else: self.obs_hash_set.add(hash_int) else: self.track_observation_hash( [(obs_hash_or_chunk, entity, variable, date, provenance)], - file_name=file_name - ) - + file_name=file_name) def _get_prov_urls(self) -> dict[str, str]: if hasattr(self, 'nodes') and self.nodes and hasattr( @@ -373,7 +379,10 @@ def _get_prov_urls(self) -> dict[str, str]: return self.nodes.get_provenance_urls() return {} - def _write_observations_df_to_disk(self, df: pd.DataFrame, import_name: str, file_name: str = ""): + def _write_observations_df_to_disk(self, + df: pd.DataFrame, + import_name: str, + file_name: str = ""): import_temp_dir = os.path.join(self.temp_local_dir, import_name) prov_urls = self._get_prov_urls() n = len(df) @@ -383,9 +392,13 @@ def _write_observations_df_to_disk(self, df: pd.DataFrame, import_name: str, fil with self.lock: shard_index = self.obs_shard_index self.obs_shard_index += 1 - _write_observation_shard( - chunk_records, shard_index, import_temp_dir, self.ns_map, prov_urls, - track_hash_fn=self.track_observation_hash, file_name=file_name) + _write_observation_shard(chunk_records, + shard_index, + import_temp_dir, + self.ns_map, + prov_urls, + track_hash_fn=self.track_observation_hash, + file_name=file_name) def insert_observations(self, observations_df: pd.DataFrame, input_file: File): @@ -396,7 +409,9 @@ def insert_observations(self, observations_df: pd.DataFrame, import_name = self.config.import_name(input_file) file_name = input_file.path if input_file else "" self._init_import_export_dir(import_name) - self._write_observations_df_to_disk(observations_df, import_name, file_name=file_name) + self._write_observations_df_to_disk(observations_df, + import_name, + file_name=file_name) def _init_import_export_dir(self, import_name: str): import_temp_dir = os.path.join(self.temp_local_dir, import_name) @@ -535,9 +550,9 @@ def _upload_shards_gcs(self, temp_local_dir: str, files: list[str], maximum=10.0, multiplier=2.0, deadline=120.0, - predicate=lambda e: isinstance( - e, (TooManyRequests, GoogleAPICallError, requests.exceptions.RequestException)) - ) + predicate=lambda e: isinstance(e, + (TooManyRequests, GoogleAPICallError, + requests.exceptions.RequestException))) def _upload_single(rel_path: str): local_file_path = os.path.join(temp_local_dir, rel_path) diff --git a/simple/stats/runner.py b/simple/stats/runner.py index bfad22da..7020b3ae 100644 --- a/simple/stats/runner.py +++ b/simple/stats/runner.py @@ -108,10 +108,10 @@ def _create_importer_for_file( ) case ImportType.EVENTS: - sanitized_path = input_file.full_path().replace("://", "_").replace("/", "_") + sanitized_path = input_file.full_path().replace("://", + "_").replace("/", "_") debug_resolve_file = process_dir.open_file( - f"{constants.DEBUG_RESOLVE_FILE_NAME_PREFIX}_{sanitized_path}" - ) + f"{constants.DEBUG_RESOLVE_FILE_NAME_PREFIX}_{sanitized_path}") return EventsImporter( input_file=input_file, db=db, @@ -142,19 +142,28 @@ def _run_single_csv_import_proc(args: tuple): config = Config(json.loads(config_json_str)) nodes = Nodes(config=config) - db = JsonLdStreamDb(output_store, import_names, nodes, jsonld_dir_name=jsonld_dir_name) + db = JsonLdStreamDb(output_store, + import_names, + nodes, + jsonld_dir_name=jsonld_dir_name) sanitized_path = input_store.full_path().replace("://", "_").replace("/", "_") report_file = process_store.open_file(f"report_{sanitized_path}.json") reporter = ImportReporter(report_file).get_file_reporter(input_store) - importer = _create_importer_for_file( - config, input_store, process_store, db, reporter, nodes, mode=RunMode.DCP_BRIDGE - ) + importer = _create_importer_for_file(config, + input_store, + process_store, + db, + reporter, + nodes, + mode=RunMode.DCP_BRIDGE) importer.do_import() db.commit_and_close() - resolved_entities = {e.entity_dcid: e.entity_type for e in nodes.entities.values()} + resolved_entities = { + e.entity_dcid: e.entity_type for e in nodes.entities.values() + } return ( file_rel_path, db.obs_collision_count, @@ -938,9 +947,12 @@ def _run_all_data_imports(self): logging.info("Importing %d CSV files next...", len(csv_files)) if self.mode == RunMode.DCP_BRIDGE: import unittest.mock as mock_module + from util import dc_client - is_mocked = isinstance(getattr(dc_client, 'get_property_of_entities', None), mock_module.MagicMock) - + is_mocked = isinstance( + getattr(dc_client, 'get_property_of_entities', None), + mock_module.MagicMock) + if is_mocked: num_csv_threads = min(32, len(csv_files)) with concurrent.futures.ThreadPoolExecutor( @@ -956,18 +968,15 @@ def _run_all_data_imports(self): config_json_str = json.dumps(self.config.data) input_dir_path = self.input_stores[0].full_path() jsonld_dir_name = self.db.jsonld_dir.name() - proc_args = [ - ( - file.path, - input_dir_path, - self.output_dir.full_path(), - self.process_dir.full_path(), - self.import_names, - config_json_str, - jsonld_dir_name, - ) - for file in csv_files - ] + proc_args = [( + file.path, + input_dir_path, + self.output_dir.full_path(), + self.process_dir.full_path(), + self.import_names, + config_json_str, + jsonld_dir_name, + ) for file in csv_files] with concurrent.futures.ProcessPoolExecutor( max_workers=num_csv_processes) as executor: futures = [ @@ -975,7 +984,8 @@ def _run_all_data_imports(self): for arg in proc_args ] for future in concurrent.futures.as_completed(futures): - res_path, collisions, file_counts, file_samples, resolved_entities = future.result() + res_path, collisions, file_counts, file_samples, resolved_entities = future.result( + ) self._log_file_progress("Imported CSV file", res_path) if resolved_entities: self.nodes.entities_with_types(resolved_entities) @@ -983,7 +993,8 @@ def _run_all_data_imports(self): self.db.obs_collision_count += collisions for f_name, count in file_counts.items(): self.db.file_collision_counts[f_name] += count - self.db.file_sample_collisions[f_name].extend(file_samples.get(f_name, [])) + self.db.file_sample_collisions[f_name].extend( + file_samples.get(f_name, [])) else: for file in csv_files: self._run_single_import(file) From 24253bdddad9b843edec7cd1b34b6156d7ea2b30 Mon Sep 17 00:00:00 2001 From: Gabriel Mechali Date: Fri, 24 Jul 2026 11:12:02 -0400 Subject: [PATCH 11/45] REal fix for id. --- simple/stats/events_importer.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/simple/stats/events_importer.py b/simple/stats/events_importer.py index c1574c29..7a391d59 100644 --- a/simple/stats/events_importer.py +++ b/simple/stats/events_importer.py @@ -196,11 +196,13 @@ def _write_event_triples(self) -> None: id_column_name = self.nodes.property( self.id_column).dcid if self.id_column else "" - triples: list[Triple] = [] for index, row in self.df.iterrows(): - # If id column is configured, use it as the event dcid else generate based on row index. - dcid = row[ - id_column_name] if id_column_name else f"{self.event_type}_{index}" + if id_column_name and id_column_name in row: + dcid = row[id_column_name] + elif self.id_column and self.id_column in row: + dcid = row[self.id_column] + else: + dcid = f"{self.event_type}_{index}" entity = row.iloc[0] date = row.iloc[1] From e8f9cbf1f29d6259b58c3564edd34fd3dbcd990d Mon Sep 17 00:00:00 2001 From: Gabriel Mechali Date: Fri, 24 Jul 2026 11:14:53 -0400 Subject: [PATCH 12/45] Add error for missing id col, if missing just make arbitrary ids --- simple/stats/events_importer.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/simple/stats/events_importer.py b/simple/stats/events_importer.py index 7a391d59..e5d4bb5d 100644 --- a/simple/stats/events_importer.py +++ b/simple/stats/events_importer.py @@ -197,10 +197,15 @@ def _write_event_triples(self) -> None: self.id_column).dcid if self.id_column else "" for index, row in self.df.iterrows(): - if id_column_name and id_column_name in row: - dcid = row[id_column_name] - elif self.id_column and self.id_column in row: - dcid = row[self.id_column] + if self.id_column: + if id_column_name and id_column_name in row: + dcid = row[id_column_name] + elif self.id_column in row: + dcid = row[self.id_column] + else: + raise ValueError( + f"Configured idColumn '{self.id_column}' not found in CSV file '{self.input_file.path}'" + ) else: dcid = f"{self.event_type}_{index}" From 16c1a40cb356ef016f8a6e3763e1ca7e27a6a113 Mon Sep 17 00:00:00 2001 From: Gabriel Mechali Date: Fri, 24 Jul 2026 11:29:07 -0400 Subject: [PATCH 13/45] Fix pickling --- simple/stats/importer.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/simple/stats/importer.py b/simple/stats/importer.py index 850be840..6919207e 100644 --- a/simple/stats/importer.py +++ b/simple/stats/importer.py @@ -16,11 +16,11 @@ class EntityResolutionError(ValueError): """Raised when entity resolution fails during import.""" - def __init__(self, file_path: str, unresolved_entities: list[str], - message: str) -> None: + def __init__(self, file_path: str = "", unresolved_entities: list[str] | None = None, + message: str = "") -> None: super().__init__(message) self.file_path = file_path - self.unresolved_entities = unresolved_entities + self.unresolved_entities = unresolved_entities if unresolved_entities is not None else [] class Importer: From 4b1a38818d0660d028461d9d2682b177c328fb9b Mon Sep 17 00:00:00 2001 From: Gabriel Mechali Date: Fri, 24 Jul 2026 11:45:42 -0400 Subject: [PATCH 14/45] Pushing small events fix --- simple/stats/events_importer.py | 1 + 1 file changed, 1 insertion(+) diff --git a/simple/stats/events_importer.py b/simple/stats/events_importer.py index e5d4bb5d..2b9cf51b 100644 --- a/simple/stats/events_importer.py +++ b/simple/stats/events_importer.py @@ -196,6 +196,7 @@ def _write_event_triples(self) -> None: id_column_name = self.nodes.property( self.id_column).dcid if self.id_column else "" + triples: list[Triple] = [] for index, row in self.df.iterrows(): if self.id_column: if id_column_name and id_column_name in row: From 8da14f970d38d80a4df52062d4dc0415720eda4e Mon Sep 17 00:00:00 2001 From: Gabriel Mechali Date: Fri, 24 Jul 2026 14:22:45 -0400 Subject: [PATCH 15/45] lint --- simple/stats/importer.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/simple/stats/importer.py b/simple/stats/importer.py index 6919207e..d27a1a60 100644 --- a/simple/stats/importer.py +++ b/simple/stats/importer.py @@ -16,7 +16,9 @@ class EntityResolutionError(ValueError): """Raised when entity resolution fails during import.""" - def __init__(self, file_path: str = "", unresolved_entities: list[str] | None = None, + def __init__(self, + file_path: str = "", + unresolved_entities: list[str] | None = None, message: str = "") -> None: super().__init__(message) self.file_path = file_path From 18494baf5fa70f6f21d94e0bae58a54878d4197b Mon Sep 17 00:00:00 2001 From: Gabriel Mechali Date: Fri, 24 Jul 2026 15:23:15 -0400 Subject: [PATCH 16/45] Move the MCF parsing in workers too. --- simple/stats/jsonld_stream_db.py | 7 +- simple/stats/runner.py | 155 ++++++++++++++++++------------- 2 files changed, 92 insertions(+), 70 deletions(-) diff --git a/simple/stats/jsonld_stream_db.py b/simple/stats/jsonld_stream_db.py index 2bbe74ff..591e19a8 100644 --- a/simple/stats/jsonld_stream_db.py +++ b/simple/stats/jsonld_stream_db.py @@ -196,11 +196,8 @@ def _write_observation_shard(chunk_or_args, compacted_jsonld = {"@context": ns_map, "@graph": graph_list} if file_name: - file_stem = os.path.basename(file_name).rsplit(".", 1)[0] - if file_stem and file_stem != "data": - shard_name = f"observation-{file_stem}-{shard_index:05d}.jsonld" - else: - shard_name = f"observation-{shard_index:05d}.jsonld" + sanitized_stem = file_name.replace("/", "_").rsplit(".", 1)[0] + shard_name = f"observation-{sanitized_stem}-{shard_index:05d}.jsonld" else: shard_name = f"observation-{shard_index:05d}.jsonld" with create_store(jsonld_dir_path) as store: diff --git a/simple/stats/runner.py b/simple/stats/runner.py index da410055..fb67adea 100644 --- a/simple/stats/runner.py +++ b/simple/stats/runner.py @@ -91,9 +91,21 @@ def _create_importer_for_file( nodes: Nodes, mode: Optional[RunMode] = None, ) -> Importer: + if input_file.path.endswith(".mcf"): + output_file = process_dir.open_file(input_file.path) if mode == RunMode.MAIN_DC else None + return McfImporter( + input_file=input_file, + output_file=output_file, + db=db, + reporter=reporter, + is_main_dc=(mode == RunMode.MAIN_DC), + nodes=nodes, + ) + import_type = config.import_type(input_file) match import_type: + case ImportType.OBSERVATIONS: mappings = config.column_mappings(input_file) if not mappings and mode == RunMode.DCP_BRIDGE: @@ -135,42 +147,57 @@ def _create_importer_for_file( def _run_single_csv_import_proc(args: tuple): file_rel_path, input_dir_path, output_dir_path, process_dir_path, import_names, config_json_str, jsonld_dir_name = args - input_dir = create_store(input_dir_path).as_dir() - output_store = create_store(output_dir_path).as_dir() - process_store = create_store(process_dir_path).as_dir() - input_store = input_dir.open_file(file_rel_path) - - config = Config(json.loads(config_json_str)) - nodes = Nodes(config=config) - db = JsonLdStreamDb(output_store, - import_names, - nodes, - jsonld_dir_name=jsonld_dir_name) - - sanitized_path = input_store.full_path().replace("://", "_").replace("/", "_") - report_file = process_store.open_file(f"report_{sanitized_path}.json") - reporter = ImportReporter(report_file).get_file_reporter(input_store) - - importer = _create_importer_for_file(config, - input_store, - process_store, - db, - reporter, - nodes, - mode=RunMode.DCP_BRIDGE) - importer.do_import() - db.commit_and_close() - - resolved_entities = { - e.entity_dcid: e.entity_type for e in nodes.entities.values() - } - return ( - file_rel_path, - db.obs_collision_count, - dict(db.file_collision_counts), - dict(db.file_sample_collisions), - resolved_entities, - ) + with create_store(input_dir_path) as input_store_obj, \ + create_store(output_dir_path) as output_store_obj, \ + create_store(process_dir_path) as process_store_obj: + input_dir = input_store_obj.as_dir() + output_store = output_store_obj.as_dir() + process_store = process_store_obj.as_dir() + input_store = input_dir.open_file(file_rel_path) + + config = Config(json.loads(config_json_str)) + nodes = Nodes(config=config) + db = JsonLdStreamDb(output_store, + import_names, + nodes, + jsonld_dir_name=jsonld_dir_name) + + sanitized_path = input_store.full_path().replace("://", "_").replace("/", "_") + report_file = process_store.open_file(f"report_{sanitized_path}.json") + reporter = ImportReporter(report_file).get_file_reporter(input_store) + + importer = _create_importer_for_file(config, + input_store, + process_store, + db, + reporter, + nodes, + mode=RunMode.DCP_BRIDGE) + importer.do_import() + db.commit_and_close() + + resolved_entities = { + e.entity_dcid: e.entity_type for e in nodes.entities.values() + } + event_types = dict(nodes.event_types) + entity_types = dict(nodes.entity_types) + variables = dict(nodes.variables) + sources = dict(nodes.sources) + provenances = dict(nodes.provenances) + groups = dict(nodes.groups) + return ( + file_rel_path, + db.obs_collision_count, + dict(db.file_collision_counts), + dict(db.file_sample_collisions), + resolved_entities, + event_types, + entity_types, + variables, + sources, + provenances, + groups, + ) class Runner: @@ -928,26 +955,9 @@ def _run_all_data_imports(self): self._total_files_count = len(csv_files) + len(mcf_files) self._counter_lock = threading.Lock() - # 1. Process MCF files first (contains schema/provenances) - if mcf_files: - logging.info("Importing %d MCF files first...", len(mcf_files)) - if self.mode == RunMode.DCP_BRIDGE: - num_mcf_threads = min(32, len(mcf_files)) - with concurrent.futures.ThreadPoolExecutor( - max_workers=num_mcf_threads) as executor: - futures = [ - executor.submit(self._run_single_mcf_import, file) - for file in mcf_files - ] - for future in concurrent.futures.as_completed(futures): - future.result() - else: - for file in mcf_files: - self._run_single_mcf_import(file) - - # 2. Process CSV files next (contains data observations/entities) - if csv_files: - logging.info("Importing %d CSV files next...", len(csv_files)) + all_files = list(mcf_files) + list(csv_files) + if all_files: + logging.info("Importing %d files (%d MCF, %d CSV)...", len(all_files), len(mcf_files), len(csv_files)) if self.mode == RunMode.DCP_BRIDGE: import unittest.mock as mock_module @@ -957,17 +967,17 @@ def _run_all_data_imports(self): mock_module.MagicMock) if is_mocked: - num_csv_threads = min(32, len(csv_files)) + num_threads = min(32, len(all_files)) with concurrent.futures.ThreadPoolExecutor( - max_workers=num_csv_threads) as executor: + max_workers=num_threads) as executor: futures = [ executor.submit(self._run_single_import, file) - for file in csv_files + for file in all_files ] for future in concurrent.futures.as_completed(futures): future.result() else: - num_csv_processes = min(32, len(csv_files)) + num_processes = min(32, len(all_files)) config_json_str = json.dumps(self.config.data) input_dir_path = self.input_stores[0].full_path() jsonld_dir_name = self.db.jsonld_dir.name() @@ -979,19 +989,32 @@ def _run_all_data_imports(self): self.import_names, config_json_str, jsonld_dir_name, - ) for file in csv_files] + ) for file in all_files] with concurrent.futures.ProcessPoolExecutor( - max_workers=num_csv_processes) as executor: + max_workers=num_processes) as executor: futures = [ executor.submit(_run_single_csv_import_proc, arg) for arg in proc_args ] for future in concurrent.futures.as_completed(futures): - res_path, collisions, file_counts, file_samples, resolved_entities = future.result( - ) - self._log_file_progress("Imported CSV file", res_path) + (res_path, collisions, file_counts, file_samples, + resolved_entities, event_types, entity_types, + variables, sources, provenances, groups) = future.result() + self._log_file_progress("Imported file", res_path) if resolved_entities: self.nodes.entities_with_types(resolved_entities) + if event_types: + self.nodes.event_types.update(event_types) + if entity_types: + self.nodes.entity_types.update(entity_types) + if variables: + self.nodes.variables.update(variables) + if sources: + self.nodes.sources.update(sources) + if provenances: + self.nodes.provenances.update(provenances) + if groups: + self.nodes.groups.update(groups) if collisions and hasattr(self.db, "obs_collision_count"): self.db.obs_collision_count += collisions for f_name, count in file_counts.items(): @@ -999,6 +1022,8 @@ def _run_all_data_imports(self): self.db.file_sample_collisions[f_name].extend( file_samples.get(f_name, [])) else: + for file in mcf_files: + self._run_single_mcf_import(file) for file in csv_files: self._run_single_import(file) From a765529c23a6dd136491f4f031d904fbfc5c7ff8 Mon Sep 17 00:00:00 2001 From: Gabriel Mechali Date: Fri, 24 Jul 2026 15:25:34 -0400 Subject: [PATCH 17/45] Lint --- simple/stats/runner.py | 36 ++++++++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/simple/stats/runner.py b/simple/stats/runner.py index fb67adea..a9504bce 100644 --- a/simple/stats/runner.py +++ b/simple/stats/runner.py @@ -92,7 +92,8 @@ def _create_importer_for_file( mode: Optional[RunMode] = None, ) -> Importer: if input_file.path.endswith(".mcf"): - output_file = process_dir.open_file(input_file.path) if mode == RunMode.MAIN_DC else None + output_file = process_dir.open_file( + input_file.path) if mode == RunMode.MAIN_DC else None return McfImporter( input_file=input_file, output_file=output_file, @@ -107,14 +108,27 @@ def _create_importer_for_file( match import_type: case ImportType.OBSERVATIONS: - mappings = config.column_mappings(input_file) - if not mappings and mode == RunMode.DCP_BRIDGE: - raise ValueError( - f"Missing column mappings for file '{input_file.path}' in config.json" + input_file_format = config.format(input_file) + if input_file_format == InputFileFormat.VARIABLE_PER_ROW: + mappings = config.column_mappings(input_file) + if not mappings and mode == RunMode.DCP_BRIDGE: + raise ValueError( + f"Missing column mappings for file '{input_file.path}' in config.json" + ) + return VariablePerRowImporter( + input_file=input_file, + db=db, + reporter=reporter, + nodes=nodes, ) - return VariablePerRowImporter( + sanitized_path = input_file.full_path().replace("://", "_").replace("/", "_") + debug_resolve_file = process_dir.open_file( + f"{constants.DEBUG_RESOLVE_FILE_NAME_PREFIX}_{sanitized_path}" + ) + return ObservationsImporter( input_file=input_file, db=db, + debug_resolve_file=debug_resolve_file, reporter=reporter, nodes=nodes, ) @@ -162,7 +176,8 @@ def _run_single_csv_import_proc(args: tuple): nodes, jsonld_dir_name=jsonld_dir_name) - sanitized_path = input_store.full_path().replace("://", "_").replace("/", "_") + sanitized_path = input_store.full_path().replace("://", + "_").replace("/", "_") report_file = process_store.open_file(f"report_{sanitized_path}.json") reporter = ImportReporter(report_file).get_file_reporter(input_store) @@ -957,7 +972,8 @@ def _run_all_data_imports(self): all_files = list(mcf_files) + list(csv_files) if all_files: - logging.info("Importing %d files (%d MCF, %d CSV)...", len(all_files), len(mcf_files), len(csv_files)) + logging.info("Importing %d files (%d MCF, %d CSV)...", len(all_files), + len(mcf_files), len(csv_files)) if self.mode == RunMode.DCP_BRIDGE: import unittest.mock as mock_module @@ -998,8 +1014,8 @@ def _run_all_data_imports(self): ] for future in concurrent.futures.as_completed(futures): (res_path, collisions, file_counts, file_samples, - resolved_entities, event_types, entity_types, - variables, sources, provenances, groups) = future.result() + resolved_entities, event_types, entity_types, variables, sources, + provenances, groups) = future.result() self._log_file_progress("Imported file", res_path) if resolved_entities: self.nodes.entities_with_types(resolved_entities) From d8d4b1b5226e5a0a0a880ec5cd0aa8a36252dcdd Mon Sep 17 00:00:00 2001 From: Gabriel Mechali Date: Fri, 24 Jul 2026 15:35:42 -0400 Subject: [PATCH 18/45] lint --- simple/stats/runner.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/simple/stats/runner.py b/simple/stats/runner.py index a9504bce..7883bc7f 100644 --- a/simple/stats/runner.py +++ b/simple/stats/runner.py @@ -121,10 +121,10 @@ def _create_importer_for_file( reporter=reporter, nodes=nodes, ) - sanitized_path = input_file.full_path().replace("://", "_").replace("/", "_") + sanitized_path = input_file.full_path().replace("://", + "_").replace("/", "_") debug_resolve_file = process_dir.open_file( - f"{constants.DEBUG_RESOLVE_FILE_NAME_PREFIX}_{sanitized_path}" - ) + f"{constants.DEBUG_RESOLVE_FILE_NAME_PREFIX}_{sanitized_path}") return ObservationsImporter( input_file=input_file, db=db, From e37bd412e2cc133f459f847f72b9e05873fef366 Mon Sep 17 00:00:00 2001 From: Gabriel Mechali Date: Fri, 24 Jul 2026 15:57:46 -0400 Subject: [PATCH 19/45] Fix validation to read from distrubited output --- simple/stats/validation.py | 9 ++++++++- simple/tests/stats/jsonld_stream_db_test.py | 7 ++++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/simple/stats/validation.py b/simple/stats/validation.py index 738a6e04..e93c22b4 100644 --- a/simple/stats/validation.py +++ b/simple/stats/validation.py @@ -83,10 +83,17 @@ def _collect_referenced_provenances(self) -> set[str]: return referenced def _collect_defined_nodes(self) -> tuple[set[str], dict[str, str]]: - """Gathers all defined Provenances and their links from the DB triples.""" + """Gathers all defined Provenances and their links from Nodes and DB triples.""" defined_provenances = set() provenance_to_source = {} + if hasattr(self.db, "nodes") and self.db.nodes: + for prov_id, prov in self.db.nodes.provenances.items(): + clean_prov_id = self._clean_dcid(prov.id) + defined_provenances.add(clean_prov_id) + if prov.source_id: + provenance_to_source[clean_prov_id] = self._clean_dcid(prov.source_id) + all_triples = [] db_triples = getattr(self.db, "_triples", {}) if isinstance(db_triples, dict): diff --git a/simple/tests/stats/jsonld_stream_db_test.py b/simple/tests/stats/jsonld_stream_db_test.py index eb4def7b..f813fff5 100644 --- a/simple/tests/stats/jsonld_stream_db_test.py +++ b/simple/tests/stats/jsonld_stream_db_test.py @@ -73,7 +73,7 @@ def test_insert_observations_and_triples(self): mock_file = mock.Mock(path="test_import/data.csv") db.insert_observations(df, mock_file) obs_shard = os.path.join(db.temp_local_dir, "test_import", - "observation-00000.jsonld") + "observation-test_import_data-00000.jsonld") self.assertTrue(os.path.exists(obs_shard)) # Insert triples @@ -161,7 +161,7 @@ def test_commit_and_close_local(self): # Shards should be written directly to the target unique directory target_dir_path = db.jsonld_dir.full_path() obs_shard = os.path.join(target_dir_path, "test_import", - "observation-00000.jsonld") + "observation-test_import_data-00000.jsonld") node_shard = os.path.join(target_dir_path, "test_import", "node-00000.jsonld") @@ -223,7 +223,8 @@ def test_commit_and_close_gcs(self, mock_storage_client): # Verify upload blob calls mock_bucket.blob.assert_called_with( - "ingestion/test/test_import/observation-00000.jsonld") + "ingestion/test/test_import/observation-test_import_data-00000.jsonld" + ) mock_blob.upload_from_filename.assert_called_once() def test_node_fast_vs_rdflib_parity(self): From e6b020b29625924897a3844ff5fb6ef18cc722b6 Mon Sep 17 00:00:00 2001 From: Gabriel Mechali Date: Fri, 24 Jul 2026 16:09:52 -0400 Subject: [PATCH 20/45] svg tiny fix --- simple/stats/config.py | 5 ++++- simple/stats/runner.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/simple/stats/config.py b/simple/stats/config.py index 17f286a4..6f8ff38e 100644 --- a/simple/stats/config.py +++ b/simple/stats/config.py @@ -216,7 +216,10 @@ def database(self) -> dict: return self.data.get(_DATABASE_FIELD) def generate_hierarchy(self) -> bool: - return self.data.get(_GROUP_STAT_VARS_BY_PROPERTY) or False + val = self.data.get(_GROUP_STAT_VARS_BY_PROPERTY) + if isinstance(val, str): + return val.lower() in ("true", "1", "yes") + return bool(val) def include_input_subdirs(self) -> bool: return self.data.get(_INCLUDE_INPUT_SUBDIRS_PROPERTY) or False diff --git a/simple/stats/runner.py b/simple/stats/runner.py index 7883bc7f..a2a4fe2a 100644 --- a/simple/stats/runner.py +++ b/simple/stats/runner.py @@ -436,7 +436,7 @@ def _handle_workflow_handoff(self) -> None: else: handshake_payload = { "importList": json.dumps(self.trigger_workflow_info), - "generateStatVarGroups": self.config.generate_hierarchy(), + "generateStatVarGroups": bool(self.config.generate_hierarchy()), } output_json_path = f"{temp_location.rstrip('/')}/datacommons/ingestion_records/{workflow_id}.json" From fd84819179a15e10605f6171623cdac007d2b53f Mon Sep 17 00:00:00 2001 From: Gabriel Mechali Date: Fri, 24 Jul 2026 16:22:30 -0400 Subject: [PATCH 21/45] Fix validation --- simple/stats/runner.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/simple/stats/runner.py b/simple/stats/runner.py index a2a4fe2a..4befd9a5 100644 --- a/simple/stats/runner.py +++ b/simple/stats/runner.py @@ -200,6 +200,7 @@ def _run_single_csv_import_proc(args: tuple): sources = dict(nodes.sources) provenances = dict(nodes.provenances) groups = dict(nodes.groups) + properties = dict(nodes.properties) return ( file_rel_path, db.obs_collision_count, @@ -212,6 +213,7 @@ def _run_single_csv_import_proc(args: tuple): sources, provenances, groups, + properties, ) @@ -995,11 +997,10 @@ def _run_all_data_imports(self): else: num_processes = min(32, len(all_files)) config_json_str = json.dumps(self.config.data) - input_dir_path = self.input_stores[0].full_path() jsonld_dir_name = self.db.jsonld_dir.name() proc_args = [( file.path, - input_dir_path, + file._store.root_path, self.output_dir.full_path(), self.process_dir.full_path(), self.import_names, @@ -1015,7 +1016,7 @@ def _run_all_data_imports(self): for future in concurrent.futures.as_completed(futures): (res_path, collisions, file_counts, file_samples, resolved_entities, event_types, entity_types, variables, sources, - provenances, groups) = future.result() + provenances, groups, properties) = future.result() self._log_file_progress("Imported file", res_path) if resolved_entities: self.nodes.entities_with_types(resolved_entities) @@ -1031,6 +1032,10 @@ def _run_all_data_imports(self): self.nodes.provenances.update(provenances) if groups: self.nodes.groups.update(groups) + for svg in groups.values(): + self.nodes.ids_to_groups[svg.id] = svg + if properties: + self.nodes.properties.update(properties) if collisions and hasattr(self.db, "obs_collision_count"): self.db.obs_collision_count += collisions for f_name, count in file_counts.items(): From 1ef9ccb28b331c3e58bb5c0fc0c532a0c86a61f6 Mon Sep 17 00:00:00 2001 From: Gabriel Mechali Date: Fri, 24 Jul 2026 16:40:13 -0400 Subject: [PATCH 22/45] Moving imports up --- simple/stats/runner.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/simple/stats/runner.py b/simple/stats/runner.py index 4befd9a5..6536f0f3 100644 --- a/simple/stats/runner.py +++ b/simple/stats/runner.py @@ -21,6 +21,7 @@ import os import threading from typing import Optional +import unittest.mock as mock_module import fs.path as fspath from stats import constants @@ -64,6 +65,7 @@ from stats.svg_cache import generate_svg_cache from stats.validation import MetadataValidator from stats.variable_per_row_importer import VariablePerRowImporter +from util import dc_client from util.file_match import match from util.filesystem import create_store from util.filesystem import Dir @@ -977,9 +979,7 @@ def _run_all_data_imports(self): logging.info("Importing %d files (%d MCF, %d CSV)...", len(all_files), len(mcf_files), len(csv_files)) if self.mode == RunMode.DCP_BRIDGE: - import unittest.mock as mock_module - - from util import dc_client + # TODO(gmechali): Remove thread fallback mock check once unit tests mock dc_client inside process pool. is_mocked = isinstance( getattr(dc_client, 'get_property_of_entities', None), mock_module.MagicMock) From 13e7f94c948e247b6842fb14ee52c0b74c9a50c0 Mon Sep 17 00:00:00 2001 From: Gabriel Mechali Date: Fri, 24 Jul 2026 16:44:43 -0400 Subject: [PATCH 23/45] lint --- simple/stats/runner.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/simple/stats/runner.py b/simple/stats/runner.py index 6536f0f3..03634309 100644 --- a/simple/stats/runner.py +++ b/simple/stats/runner.py @@ -65,7 +65,6 @@ from stats.svg_cache import generate_svg_cache from stats.validation import MetadataValidator from stats.variable_per_row_importer import VariablePerRowImporter -from util import dc_client from util.file_match import match from util.filesystem import create_store from util.filesystem import Dir @@ -73,6 +72,8 @@ from util.filesystem import join_path from util.filesystem import Store +from util import dc_client + class RunMode(StrEnum): CUSTOM_DC = "customdc" From 76a8bfc1b8bf9c8b7c24fc525ff55acb09f517e8 Mon Sep 17 00:00:00 2001 From: Gabriel Mechali Date: Fri, 24 Jul 2026 16:47:37 -0400 Subject: [PATCH 24/45] one more val fix --- simple/stats/runner.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/simple/stats/runner.py b/simple/stats/runner.py index 03634309..1f2d4c93 100644 --- a/simple/stats/runner.py +++ b/simple/stats/runner.py @@ -1028,9 +1028,17 @@ def _run_all_data_imports(self): if variables: self.nodes.variables.update(variables) if sources: - self.nodes.sources.update(sources) + for src in sources.values(): + self.nodes.register_source( + id=src.id, name=src.name, url=src.url) if provenances: - self.nodes.provenances.update(provenances) + for prov in provenances.values(): + self.nodes.register_provenance( + id=prov.id, + name=prov.name, + url=prov.url, + source_id=prov.source_id, + properties=prov.properties) if groups: self.nodes.groups.update(groups) for svg in groups.values(): From 7e6a0bbb9a8b4f6c89abba77a52921f7183116b7 Mon Sep 17 00:00:00 2001 From: Gabriel Mechali Date: Fri, 24 Jul 2026 16:56:41 -0400 Subject: [PATCH 25/45] lint --- simple/stats/runner.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/simple/stats/runner.py b/simple/stats/runner.py index 1f2d4c93..b20bda92 100644 --- a/simple/stats/runner.py +++ b/simple/stats/runner.py @@ -1029,16 +1029,16 @@ def _run_all_data_imports(self): self.nodes.variables.update(variables) if sources: for src in sources.values(): - self.nodes.register_source( - id=src.id, name=src.name, url=src.url) + self.nodes.register_source(id=src.id, + name=src.name, + url=src.url) if provenances: for prov in provenances.values(): - self.nodes.register_provenance( - id=prov.id, - name=prov.name, - url=prov.url, - source_id=prov.source_id, - properties=prov.properties) + self.nodes.register_provenance(id=prov.id, + name=prov.name, + url=prov.url, + source_id=prov.source_id, + properties=prov.properties) if groups: self.nodes.groups.update(groups) for svg in groups.values(): From 4370cc6fc1235a319f73901e14cad7d8f8f9d2ab Mon Sep 17 00:00:00 2001 From: Gabriel Mechali Date: Sat, 25 Jul 2026 10:57:09 -0400 Subject: [PATCH 26/45] Minor comments for readbility --- simple/stats/runner.py | 134 +++++++++++++++++------------- simple/tests/stats/runner_test.py | 3 +- 2 files changed, 78 insertions(+), 59 deletions(-) diff --git a/simple/stats/runner.py b/simple/stats/runner.py index b20bda92..c0f5fee6 100644 --- a/simple/stats/runner.py +++ b/simple/stats/runner.py @@ -13,6 +13,7 @@ # limitations under the License. import concurrent.futures +from dataclasses import dataclass from datetime import datetime from datetime import timezone from enum import StrEnum @@ -94,7 +95,7 @@ def _create_importer_for_file( nodes: Nodes, mode: Optional[RunMode] = None, ) -> Importer: - if input_file.path.endswith(".mcf"): + if input_file.path.lower().endswith(".mcf"): output_file = process_dir.open_file( input_file.path) if mode == RunMode.MAIN_DC else None return McfImporter( @@ -162,8 +163,31 @@ def _create_importer_for_file( f"Unsupported import type: {import_type} ({input_file.full_path()})") -def _run_single_csv_import_proc(args: tuple): - file_rel_path, input_dir_path, output_dir_path, process_dir_path, import_names, config_json_str, jsonld_dir_name = args +@dataclass +class ImportProcResult: + file_rel_path: str + obs_collision_count: int + file_collision_counts: dict + file_sample_collisions: dict + resolved_entities: dict + event_types: dict + entity_types: dict + variables: dict + sources: dict + provenances: dict + groups: dict + properties: dict + + +def _run_single_csv_import_proc( + file_rel_path: str, + input_dir_path: str, + output_dir_path: str, + process_dir_path: str, + import_names: dict, + config_json_str: str, + jsonld_dir_name: str, +) -> ImportProcResult: with create_store(input_dir_path) as input_store_obj, \ create_store(output_dir_path) as output_store_obj, \ create_store(process_dir_path) as process_store_obj: @@ -204,19 +228,19 @@ def _run_single_csv_import_proc(args: tuple): provenances = dict(nodes.provenances) groups = dict(nodes.groups) properties = dict(nodes.properties) - return ( - file_rel_path, - db.obs_collision_count, - dict(db.file_collision_counts), - dict(db.file_sample_collisions), - resolved_entities, - event_types, - entity_types, - variables, - sources, - provenances, - groups, - properties, + return ImportProcResult( + file_rel_path=file_rel_path, + obs_collision_count=db.obs_collision_count, + file_collision_counts=dict(db.file_collision_counts), + file_sample_collisions=dict(db.file_sample_collisions), + resolved_entities=resolved_entities, + event_types=event_types, + entity_types=entity_types, + variables=variables, + sources=sources, + provenances=provenances, + groups=groups, + properties=properties, ) @@ -230,6 +254,7 @@ def __init__( output_dir_path: str, mode: RunMode = RunMode.CUSTOM_DC, import_names: Optional[list[str]] = None, + use_multiprocessing: bool = True, ) -> None: assert (config_file_path or input_dir_path), "One of config_file or input_dir must be specified" @@ -237,6 +262,7 @@ def __init__( self.mode = mode self.import_names = import_names + self.use_multiprocessing = use_multiprocessing self.active_import_prefixes = None # File systems, both input and output. Must be closed when run finishes. @@ -980,12 +1006,7 @@ def _run_all_data_imports(self): logging.info("Importing %d files (%d MCF, %d CSV)...", len(all_files), len(mcf_files), len(csv_files)) if self.mode == RunMode.DCP_BRIDGE: - # TODO(gmechali): Remove thread fallback mock check once unit tests mock dc_client inside process pool. - is_mocked = isinstance( - getattr(dc_client, 'get_property_of_entities', None), - mock_module.MagicMock) - - if is_mocked: + if not self.use_multiprocessing: num_threads = min(32, len(all_files)) with concurrent.futures.ThreadPoolExecutor( max_workers=num_threads) as executor: @@ -999,58 +1020,55 @@ def _run_all_data_imports(self): num_processes = min(32, len(all_files)) config_json_str = json.dumps(self.config.data) jsonld_dir_name = self.db.jsonld_dir.name() - proc_args = [( - file.path, - file._store.root_path, - self.output_dir.full_path(), - self.process_dir.full_path(), - self.import_names, - config_json_str, - jsonld_dir_name, - ) for file in all_files] with concurrent.futures.ProcessPoolExecutor( max_workers=num_processes) as executor: futures = [ - executor.submit(_run_single_csv_import_proc, arg) - for arg in proc_args + executor.submit( + _run_single_csv_import_proc, + file.path, + file._store.root_path, + self.output_dir.full_path(), + self.process_dir.full_path(), + self.import_names, + config_json_str, + jsonld_dir_name, + ) for file in all_files ] for future in concurrent.futures.as_completed(futures): - (res_path, collisions, file_counts, file_samples, - resolved_entities, event_types, entity_types, variables, sources, - provenances, groups, properties) = future.result() - self._log_file_progress("Imported file", res_path) - if resolved_entities: - self.nodes.entities_with_types(resolved_entities) - if event_types: - self.nodes.event_types.update(event_types) - if entity_types: - self.nodes.entity_types.update(entity_types) - if variables: - self.nodes.variables.update(variables) - if sources: - for src in sources.values(): + res = future.result() + self._log_file_progress("Imported file", res.file_rel_path) + if res.resolved_entities: + self.nodes.entities_with_types(res.resolved_entities) + if res.event_types: + self.nodes.event_types.update(res.event_types) + if res.entity_types: + self.nodes.entity_types.update(res.entity_types) + if res.variables: + self.nodes.variables.update(res.variables) + if res.sources: + for src in res.sources.values(): self.nodes.register_source(id=src.id, name=src.name, url=src.url) - if provenances: - for prov in provenances.values(): + if res.provenances: + for prov in res.provenances.values(): self.nodes.register_provenance(id=prov.id, name=prov.name, url=prov.url, source_id=prov.source_id, properties=prov.properties) - if groups: - self.nodes.groups.update(groups) - for svg in groups.values(): + if res.groups: + self.nodes.groups.update(res.groups) + for svg in res.groups.values(): self.nodes.ids_to_groups[svg.id] = svg - if properties: - self.nodes.properties.update(properties) - if collisions and hasattr(self.db, "obs_collision_count"): - self.db.obs_collision_count += collisions - for f_name, count in file_counts.items(): + if res.properties: + self.nodes.properties.update(res.properties) + if res.obs_collision_count and hasattr(self.db, "obs_collision_count"): + self.db.obs_collision_count += res.obs_collision_count + for f_name, count in res.file_collision_counts.items(): self.db.file_collision_counts[f_name] += count self.db.file_sample_collisions[f_name].extend( - file_samples.get(f_name, [])) + res.file_sample_collisions.get(f_name, [])) else: for file in mcf_files: self._run_single_mcf_import(file) diff --git a/simple/tests/stats/runner_test.py b/simple/tests/stats/runner_test.py index 0903debe..3f266941 100644 --- a/simple/tests/stats/runner_test.py +++ b/simple/tests/stats/runner_test.py @@ -93,7 +93,8 @@ def _test_runner(test: unittest.TestCase, Runner(config_file_path=config_path, input_dir_path=input_dir, output_dir_path=temp_dir, - mode=run_mode).run() + mode=run_mode, + use_multiprocessing=False).run() if is_write_mode(): write_full_db_to_file(db_path=db_path, output_path=expected_db_path) From 25b6bc13c4ce458df6d906762be2d3deaf4d6b70 Mon Sep 17 00:00:00 2001 From: Gabriel Mechali Date: Sat, 25 Jul 2026 11:07:25 -0400 Subject: [PATCH 27/45] Test fix --- simple/tests/stats/runner_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/simple/tests/stats/runner_test.py b/simple/tests/stats/runner_test.py index 3f266941..89282d0c 100644 --- a/simple/tests/stats/runner_test.py +++ b/simple/tests/stats/runner_test.py @@ -546,7 +546,8 @@ def side_effect(path, *args, **kwargs): input_dir_path=input_dir, output_dir_path=output_dir, mode=RunMode.DCP_BRIDGE, - import_names=[constants.ALL_IMPORTS]) + import_names=[constants.ALL_IMPORTS], + use_multiprocessing=False) runner.run() # Verify GCS client calls were made From 681de0121eb5c3f875308f369e02b478803637f4 Mon Sep 17 00:00:00 2001 From: Gabriel Mechali Date: Sat, 25 Jul 2026 11:07:42 -0400 Subject: [PATCH 28/45] Fix lint --- simple/stats/runner.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/simple/stats/runner.py b/simple/stats/runner.py index c0f5fee6..39f2053b 100644 --- a/simple/stats/runner.py +++ b/simple/stats/runner.py @@ -1063,7 +1063,8 @@ def _run_all_data_imports(self): self.nodes.ids_to_groups[svg.id] = svg if res.properties: self.nodes.properties.update(res.properties) - if res.obs_collision_count and hasattr(self.db, "obs_collision_count"): + if res.obs_collision_count and hasattr(self.db, + "obs_collision_count"): self.db.obs_collision_count += res.obs_collision_count for f_name, count in res.file_collision_counts.items(): self.db.file_collision_counts[f_name] += count From 345c23b85e8053db65ff8b1ca9f3550c45670257 Mon Sep 17 00:00:00 2001 From: Gabriel Mechali Date: Fri, 24 Jul 2026 18:10:34 -0400 Subject: [PATCH 29/45] Namespace presercation through preprocessor, first try --- simple/stats/data.py | 6 +++- simple/stats/mcf_importer.py | 4 +-- simple/stats/observations_importer.py | 8 +++-- simple/stats/util.py | 28 ++++++++++++++- simple/stats/validation.py | 14 ++++---- .../tests/stats/observations_importer_test.py | 36 +++++++++++++++++++ simple/tests/stats/scratch_test_floats.py | 33 +++++++++++++++++ 7 files changed, 116 insertions(+), 13 deletions(-) create mode 100644 simple/tests/stats/scratch_test_floats.py diff --git a/simple/stats/data.py b/simple/stats/data.py index 22c02636..f7959587 100644 --- a/simple/stats/data.py +++ b/simple/stats/data.py @@ -28,6 +28,7 @@ from stats import schema_constants as sc from stats.util import base64_decode_and_gunzip_json from stats.util import gzip_and_base64_encode_json +from stats.util import has_namespace_prefix from stats.util import is_uri_or_namespace _PREDICATE_TYPE_OF = "typeOf" @@ -546,7 +547,10 @@ def add_triple(self, triple: Triple) -> Self: def to_mcf(self) -> str: parts: list[str] = [] - parts.append(f"Node: dcid:{self.id}") + if has_namespace_prefix(self.id): + parts.append(f"Node: {self.id}") + else: + parts.append(f"Node: dcid:{self.id}") parts.extend([f"{p}: {v}" for p, v in self.properties.items()]) return "\n".join(parts) diff --git a/simple/stats/mcf_importer.py b/simple/stats/mcf_importer.py index 0987d319..792f0b52 100644 --- a/simple/stats/mcf_importer.py +++ b/simple/stats/mcf_importer.py @@ -191,8 +191,8 @@ def _register_metadata_nodes(triples: list[Triple], nodes: Nodes) -> None: val = triple.object_value or triple.object_id or "" subject_properties[sub_id]["name"] = _clean_literal(val) elif pred == "source": - # Map source to the source ID - subject_properties[sub_id]["source"] = triple.object_id + val = triple.object_id or triple.object_value or "" + subject_properties[sub_id]["source"] = _clean_literal(val) for sub_id, props in subject_properties.items(): node_type = props.get("typeOf") diff --git a/simple/stats/observations_importer.py b/simple/stats/observations_importer.py index f635aa81..dc5ab0ec 100644 --- a/simple/stats/observations_importer.py +++ b/simple/stats/observations_importer.py @@ -24,6 +24,8 @@ from stats.importer import Importer from stats.nodes import Nodes from stats.reporter import FileImportReporter +from stats.util import get_namespace_prefix_and_suffix +from stats.util import has_namespace_prefix from util.filesystem import File from util import dc_client as dc @@ -176,9 +178,9 @@ def _resolve_entities(self) -> None: pre_resolved_entities = {} def remove_pre_resolved(entity: str) -> bool: - if entity.startswith(constants.DCID_OVERRIDE_PREFIX): - pre_resolved_entities[entity] = entity[ - len(constants.DCID_OVERRIDE_PREFIX):].strip() + if has_namespace_prefix(entity): + prefix, suffix = get_namespace_prefix_and_suffix(entity) + pre_resolved_entities[entity] = suffix.strip() return False return True diff --git a/simple/stats/util.py b/simple/stats/util.py index a5f3c0b4..16f200f4 100644 --- a/simple/stats/util.py +++ b/simple/stats/util.py @@ -55,6 +55,33 @@ def base64_decode_and_gunzip_json(encoded_data: str) -> dict: return json.loads(base64_decode_and_gunzip(encoded_data)) +_STANDARD_PREFIXES = {"http", "https", "dcid", "schema", "dcs"} +_NAMESPACE_PREFIX_RE = re.compile(r"^[a-zA-Z0-9_\-]+:") + + +def has_namespace_prefix(val: str) -> bool: + """Returns True if string starts with a namespace prefix (e.g. 'dcid:', 'undata:', 'acme:').""" + if not isinstance(val, str) or not val: + return False + return bool(_NAMESPACE_PREFIX_RE.match(val.strip())) + + +def get_namespace_prefix_and_suffix(val: str) -> tuple[str, str]: + """Extracts (prefix, suffix) from a namespaced string (e.g. 'undata:place/1' -> ('undata', 'place/1')).""" + if isinstance(val, str) and ":" in val and " " not in val: + parts = val.split(":", 1) + return parts[0], parts[1] + return "", val or "" + + +def is_custom_namespace_prefix(prefix: str) -> bool: + """Returns True if the prefix represents a custom instance namespace (not standard 'dcid', 'http', 'schema').""" + if not prefix: + return False + p_lower = prefix.lower() + return p_lower not in _STANDARD_PREFIXES + + def is_uri_or_namespace(val: str) -> bool: """Returns True if the value is a full URL, standard DCID, or valid custom namespace.""" if not isinstance(val, str): @@ -65,7 +92,6 @@ def is_uri_or_namespace(val: str) -> bool: return True if ":" in val and " " not in val: prefix = val.split(":", 1)[0] - # A valid namespace prefix must be purely alphanumeric (e.g. 'custom', 'un', 'myorg') return prefix.isalnum() return False diff --git a/simple/stats/validation.py b/simple/stats/validation.py index e93c22b4..68186fb7 100644 --- a/simple/stats/validation.py +++ b/simple/stats/validation.py @@ -15,8 +15,10 @@ import logging from stats.config import Config +from stats.data import strip_namespace from stats.data import ValidationErrorType from stats.db import Db +from stats.util import has_namespace_prefix from stats.util import is_uri_or_namespace @@ -102,7 +104,7 @@ def _collect_defined_nodes(self) -> tuple[set[str], dict[str, str]]: for triple in all_triples: sub = self._clean_dcid(triple.subject_id) - pred = triple.predicate + pred = strip_namespace(triple.predicate) if pred == "typeOf": obj = triple.object_id or "" @@ -110,7 +112,7 @@ def _collect_defined_nodes(self) -> tuple[set[str], dict[str, str]]: defined_provenances.add(sub) elif pred == "source": - obj = triple.object_id or "" + obj = triple.object_id or triple.object_value or "" if obj: provenance_to_source[sub] = self._clean_dcid(obj) @@ -151,11 +153,11 @@ def _validate_source_links(self, defined_provs: set[str], raise ex def _clean_dcid(self, val: str) -> str: - """Normalizes a DCID value by ensuring it starts with 'dcid:' and has no prefix namespaces.""" + """Normalizes a DCID value by ensuring it starts with a namespace prefix.""" + if not val: + return "" if val.startswith(("http://", "https://")): return val - if val.startswith("dcid:"): + if has_namespace_prefix(val): return val - if ":" in val: - return f"dcid:{val.split(':', 1)[1]}" return f"dcid:{val}" diff --git a/simple/tests/stats/observations_importer_test.py b/simple/tests/stats/observations_importer_test.py index 037af453..be06f4b1 100644 --- a/simple/tests/stats/observations_importer_test.py +++ b/simple/tests/stats/observations_importer_test.py @@ -105,3 +105,39 @@ def test_countryalpha3codes(self): def test_obs_props(self): _test_import(self, "obs_props") + + def test_custom_namespace_pre_resolution(self): + config = Config({ + "inputFiles": [{ + "pattern": "data.csv", + "provenance": "undata:provenance/WHO" + }] + }) + nodes = Nodes(config) + mock_input_file = MagicMock() + mock_input_file.path = "data.csv" + importer = ObservationsImporter( + input_file=mock_input_file, + db=MagicMock(), + debug_resolve_file=MagicMock(), + reporter=MagicMock(), + nodes=nodes, + ) + importer.entity_type = "State" + importer.entity_column_name = "entity" + importer.df = pd.DataFrame({ + "dcid": ["undata:place/custom_1", "dcid:geoId/06", "California"] + }) + + dc_client.resolve_entities = MagicMock(return_value={"California": "geoId/06"}) + dc_client.get_property_of_entities = MagicMock(return_value={}) + + importer._resolve_entities() + + dc_client.resolve_entities.assert_called_once_with( + entities=["California"], entity_type="State", property_name="description" + ) + self.assertEqual( + importer.df["dcid"].tolist(), + ["place/custom_1", "geoId/06", "geoId/06"] + ) diff --git a/simple/tests/stats/scratch_test_floats.py b/simple/tests/stats/scratch_test_floats.py new file mode 100644 index 00000000..7982eb89 --- /dev/null +++ b/simple/tests/stats/scratch_test_floats.py @@ -0,0 +1,33 @@ +import numpy as np +import pandas as pd + + +def convert_old(col): + is_int_value = col.notna() & (col == col.round()) + is_na = col.isna() + return np.where( + is_int_value, + col.round().astype("Int64").astype(str), + np.where(is_na, "", col.astype(str)), + ) + + +def convert_new_str(col): + # Convert to string and remove trailing .0 + s = col.astype(str) + # Only replace .0 if it's at the end of the string + return s.str.replace(r'\.0$', '', regex=True) + + +test_vals = [ + 100.0, 100.5, 100000000000000.0, 100000000000000.0001, 0.0000000000001, + np.nan +] + +df = pd.DataFrame({"val": test_vals}) +print("INPUTS:") +print(df["val"]) +print("\nOLD CONVERSION:") +print(convert_old(df["val"])) +print("\nNEW STR CONVERSION:") +print(convert_new_str(df["val"])) From 8b87dbd7fd4515c76525c9beb438b19962a0a727 Mon Sep 17 00:00:00 2001 From: Gabriel Mechali Date: Fri, 24 Jul 2026 18:46:50 -0400 Subject: [PATCH 30/45] More tweaks --- simple/stats/events_importer.py | 8 +++++--- simple/stats/jsonld_stream_db.py | 5 ++++- simple/util/filesystem.py | 10 +++++----- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/simple/stats/events_importer.py b/simple/stats/events_importer.py index 2b9cf51b..5df86099 100644 --- a/simple/stats/events_importer.py +++ b/simple/stats/events_importer.py @@ -27,6 +27,8 @@ from stats.importer import Importer from stats.nodes import Nodes from stats.reporter import FileImportReporter +from stats.util import get_namespace_prefix_and_suffix +from stats.util import has_namespace_prefix from util.filesystem import File from util import dc_client as dc @@ -237,9 +239,9 @@ def _resolve_entities(self) -> None: pre_resolved_entities = {} def remove_pre_resolved(entity: str) -> bool: - if entity.startswith(constants.DCID_OVERRIDE_PREFIX): - pre_resolved_entities[entity] = entity[ - len(constants.DCID_OVERRIDE_PREFIX):].strip() + if has_namespace_prefix(entity): + prefix, suffix = get_namespace_prefix_and_suffix(entity) + pre_resolved_entities[entity] = suffix.strip() return False return True diff --git a/simple/stats/jsonld_stream_db.py b/simple/stats/jsonld_stream_db.py index 591e19a8..160ea597 100644 --- a/simple/stats/jsonld_stream_db.py +++ b/simple/stats/jsonld_stream_db.py @@ -591,7 +591,10 @@ def _upload_shards_local(self, temp_local_dir: str, files: list[str]): parent_dirs = set(os.path.dirname(f) for f in files if os.path.dirname(f)) for d in sorted(parent_dirs): - target_store.open_dir(d) + try: + target_store.open_dir(d) + except Exception: + pass def _copy_single(rel_path: str): local_file_path = os.path.join(temp_local_dir, rel_path) diff --git a/simple/util/filesystem.py b/simple/util/filesystem.py index 0ea1d0a2..0b6a721c 100644 --- a/simple/util/filesystem.py +++ b/simple/util/filesystem.py @@ -204,7 +204,10 @@ def __init__(self, store: "Store", path: str, create_if_missing: bool): # Make parent dir if needed. parent_dir_path = fspath.dirname(path) if not self.fs().isdir(parent_dir_path): - self.fs().makedirs(parent_dir_path) + try: + self.fs().makedirs(parent_dir_path) + except Exception: + pass # Make empty file. self.fs().touch(path) else: @@ -232,10 +235,7 @@ def read_string_io(self) -> io.StringIO: return io.StringIO(self.read()) def open_stream(self, max_retries: int = 3): - """Returns an open text stream for streaming line-by-line without downloading full contents upfront. - - Includes retries for transient network drops. - """ + """Returns an open text stream for streaming line-by-line with retries for transient network drops.""" for attempt in range(max_retries): try: return self.fs().open(self.path, "r") From fcb111eb15267b95ec5b53daba3be98fd0c7f233 Mon Sep 17 00:00:00 2001 From: Gabriel Mechali Date: Sat, 25 Jul 2026 11:42:16 -0400 Subject: [PATCH 31/45] Remove test file --- simple/tests/stats/scratch_test_floats.py | 33 ----------------------- 1 file changed, 33 deletions(-) delete mode 100644 simple/tests/stats/scratch_test_floats.py diff --git a/simple/tests/stats/scratch_test_floats.py b/simple/tests/stats/scratch_test_floats.py deleted file mode 100644 index 7982eb89..00000000 --- a/simple/tests/stats/scratch_test_floats.py +++ /dev/null @@ -1,33 +0,0 @@ -import numpy as np -import pandas as pd - - -def convert_old(col): - is_int_value = col.notna() & (col == col.round()) - is_na = col.isna() - return np.where( - is_int_value, - col.round().astype("Int64").astype(str), - np.where(is_na, "", col.astype(str)), - ) - - -def convert_new_str(col): - # Convert to string and remove trailing .0 - s = col.astype(str) - # Only replace .0 if it's at the end of the string - return s.str.replace(r'\.0$', '', regex=True) - - -test_vals = [ - 100.0, 100.5, 100000000000000.0, 100000000000000.0001, 0.0000000000001, - np.nan -] - -df = pd.DataFrame({"val": test_vals}) -print("INPUTS:") -print(df["val"]) -print("\nOLD CONVERSION:") -print(convert_old(df["val"])) -print("\nNEW STR CONVERSION:") -print(convert_new_str(df["val"])) From 0778e4ab23915ed7207d412ff9cf53b10f80d821 Mon Sep 17 00:00:00 2001 From: Gabriel Mechali Date: Sat, 25 Jul 2026 19:44:48 -0400 Subject: [PATCH 32/45] Fix the per import --- simple/stats/runner.py | 8 ++++++-- simple/stats/util.py | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/simple/stats/runner.py b/simple/stats/runner.py index 39f2053b..428d00e2 100644 --- a/simple/stats/runner.py +++ b/simple/stats/runner.py @@ -177,6 +177,7 @@ class ImportProcResult: provenances: dict groups: dict properties: dict + processed_imports: set def _run_single_csv_import_proc( @@ -241,6 +242,7 @@ def _run_single_csv_import_proc( provenances=provenances, groups=groups, properties=properties, + processed_imports=set(db._processed_imports), ) @@ -1063,6 +1065,8 @@ def _run_all_data_imports(self): self.nodes.ids_to_groups[svg.id] = svg if res.properties: self.nodes.properties.update(res.properties) + if res.processed_imports: + self.db._processed_imports.update(res.processed_imports) if res.obs_collision_count and hasattr(self.db, "obs_collision_count"): self.db.obs_collision_count += res.obs_collision_count @@ -1139,12 +1143,12 @@ def _run_imports_and_export_jsonld(self): output_path = self.db.jsonld_dir.full_path() import_name = self.db.import_name if os.getenv("WORKFLOW_EXECUTION_ID") and output_path.startswith("gs://"): - processed_imports = list(self.db._processed_imports) + processed_imports = sorted(list(self.db._processed_imports)) if not processed_imports: processed_imports = [import_name] import_list = [] for imp in processed_imports: - gcs_pattern = f"{output_path.rstrip('/')}/{imp}/*.jsonld" + gcs_pattern = f"{output_path.rstrip('/')}/{imp}/**/*.jsonld" import_list.append({"importName": imp, "graphPath": gcs_pattern}) self.trigger_workflow_info = import_list else: diff --git a/simple/stats/util.py b/simple/stats/util.py index 16f200f4..6b74423c 100644 --- a/simple/stats/util.py +++ b/simple/stats/util.py @@ -68,7 +68,7 @@ def has_namespace_prefix(val: str) -> bool: def get_namespace_prefix_and_suffix(val: str) -> tuple[str, str]: """Extracts (prefix, suffix) from a namespaced string (e.g. 'undata:place/1' -> ('undata', 'place/1')).""" - if isinstance(val, str) and ":" in val and " " not in val: + if isinstance(val, str) and ":" in val and " " not in val.split(":", 1)[0]: parts = val.split(":", 1) return parts[0], parts[1] return "", val or "" From 039c9f91503b5d11a498c405f4b38cda261c1234 Mon Sep 17 00:00:00 2001 From: Gabriel Mechali Date: Sat, 25 Jul 2026 20:09:46 -0400 Subject: [PATCH 33/45] Fixing per provenance imports --- simple/stats/data.py | 3 + simple/stats/db.py | 20 +++++-- simple/stats/jsonld_stream_db.py | 15 +++-- simple/stats/mcf_importer.py | 19 +++++-- simple/stats/nodes.py | 82 +++++++++++++++++++++++---- simple/stats/observations_importer.py | 7 ++- simple/stats/runner.py | 18 ++++-- 7 files changed, 134 insertions(+), 30 deletions(-) diff --git a/simple/stats/data.py b/simple/stats/data.py index f7959587..77244f67 100644 --- a/simple/stats/data.py +++ b/simple/stats/data.py @@ -182,6 +182,7 @@ def triples(self) -> list[Triple]: class Entity: entity_dcid: str entity_type: str + provenance_dir: str = "" def triples(self) -> list[Triple]: # Currently only 1 triple is generated but could be more in the future (e.g. name) @@ -197,6 +198,7 @@ class Provenance: name: str url: str = "" properties: dict[str, str] = field(default_factory=dict) + provenance_dir: str = "" def triples(self) -> list[Triple]: triples: list[Triple] = [] @@ -228,6 +230,7 @@ class Source: url: str = "" domain: str = field(init=False) properties: dict[str, str] = field(default_factory=dict) + provenance_dir: str = "" def __post_init__(self): self.domain = urlparse(self.url).netloc diff --git a/simple/stats/db.py b/simple/stats/db.py index d5812c39..18e0d0d5 100644 --- a/simple/stats/db.py +++ b/simple/stats/db.py @@ -230,7 +230,10 @@ class Db: def maybe_clear_before_import(self): pass - def insert_triples(self, triples: list[Triple], input_file: File = None): + def insert_triples(self, + triples: list[Triple], + input_file: File = None, + provenance_dir: str = None): pass def insert_observations(self, observations_df: pd.DataFrame, @@ -281,7 +284,10 @@ def __init__(self, db_params: dict) -> None: # dcid to node dict self.nodes: dict[str, McfNode] = {} - def insert_triples(self, triples: list[Triple], input_file: File = None): + def insert_triples(self, + triples: list[Triple], + input_file: File = None, + provenance_dir: str = None): for triple in triples: self._add_triple(triple) @@ -351,7 +357,10 @@ def __init__(self, config: dict) -> None: def maybe_clear_before_import(self): self.engine.clear_tables_and_indexes() - def insert_triples(self, triples: list[Triple], input_file: File = None): + def insert_triples(self, + triples: list[Triple], + input_file: File = None, + provenance_dir: str = None): logging.info("Writing %s triples to [%s]", len(triples), self.engine) if triples: self.engine.executemany(_INSERT_TRIPLES_STATEMENT, @@ -425,7 +434,10 @@ def maybe_clear_before_import(self): # Not applicable for Data Commons Platform. pass - def insert_triples(self, triples: list[Triple], input_file: File = None): + def insert_triples(self, + triples: list[Triple], + input_file: File = None, + provenance_dir: str = None): """ Convert triples to a jsonld graph and writes the graph to the Data Commons Platform instance. """ diff --git a/simple/stats/jsonld_stream_db.py b/simple/stats/jsonld_stream_db.py index 160ea597..7d831e0f 100644 --- a/simple/stats/jsonld_stream_db.py +++ b/simple/stats/jsonld_stream_db.py @@ -465,18 +465,23 @@ def _write_triples_to_disk(self, triples: list[Triple], import_name: str): (chunk, self.node_shard_index, import_temp_dir, self.ns_map)) self.node_shard_index += 1 - def insert_triples(self, triples: list[Triple], input_file: File = None): + def insert_triples(self, + triples: list[Triple], + input_file: File = None, + provenance_dir: str = None): if not triples: return - if not input_file: + if not provenance_dir and input_file: + provenance_dir = self.config.import_name(input_file) + + if not provenance_dir: with self.lock: self._triples["_global"].extend(triples) return - import_name = self.config.import_name(input_file) - self._init_import_export_dir(import_name) - self._write_triples_to_disk(triples, import_name) + self._init_import_export_dir(provenance_dir) + self._write_triples_to_disk(triples, provenance_dir) def commit(self): pass diff --git a/simple/stats/mcf_importer.py b/simple/stats/mcf_importer.py index 792f0b52..0dee3237 100644 --- a/simple/stats/mcf_importer.py +++ b/simple/stats/mcf_importer.py @@ -105,7 +105,14 @@ def do_import(self) -> None: # Register all collected metadata nodes at the end if all_metadata_triples: - _register_metadata_nodes(all_metadata_triples, self.nodes) + prov_dir = "" + try: + prov_dir = self.config.import_name(self.input_file) + except Exception: + pass + _register_metadata_nodes(all_metadata_triples, + self.nodes, + provenance_dir=prov_dir) self.reporter.report_success() except Exception as e: @@ -161,7 +168,9 @@ def _to_triple(parser_triple: list[str], local2dcid: dict[str, str]) -> Triple: return Triple(resolved_subject, predicate, object_value=value) -def _register_metadata_nodes(triples: list[Triple], nodes: Nodes) -> None: +def _register_metadata_nodes(triples: list[Triple], + nodes: Nodes, + provenance_dir: str = "") -> None: """Extracts and registers Provenance and Source nodes from parsed MCF triples. This helper is robust against: @@ -200,11 +209,13 @@ def _register_metadata_nodes(triples: list[Triple], nodes: Nodes) -> None: nodes.register_provenance(id=sub_id, name=props.get("name", ""), url=props.get("url", ""), - source_id=props.get("source", "")) + source_id=props.get("source", ""), + provenance_dir=provenance_dir) elif node_type == "Source": nodes.register_source(id=sub_id, name=props.get("name", ""), - url=props.get("url", "")) + url=props.get("url", ""), + provenance_dir=provenance_dir) def _clean_literal(val: str) -> str: diff --git a/simple/stats/nodes.py b/simple/stats/nodes.py index 6f596b68..241b996f 100644 --- a/simple/stats/nodes.py +++ b/simple/stats/nodes.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +from collections import defaultdict from functools import wraps import logging import re @@ -146,7 +147,8 @@ def register_provenance(self, name: str = "", url: str = "", source_id: str = "", - properties: dict[str, str] = None) -> Provenance: + properties: dict[str, str] = None, + provenance_dir: str = "") -> Provenance: self.has_custom_mcf_nodes = True clean_id = _clean_metadata_id(id) clean_source_id = _clean_metadata_id(source_id) if source_id else "" @@ -157,7 +159,8 @@ def register_provenance(self, source_id=clean_source_id, name=name or clean_id, url=url, - properties=properties or {}) + properties=properties or {}, + provenance_dir=provenance_dir or strip_namespace(clean_id)) self.provenances[clean_id] = prov if name: self.provenances[name] = prov @@ -172,6 +175,8 @@ def register_provenance(self, prov.source_id = clean_source_id if properties: prov.properties.update(properties) + if provenance_dir and not prov.provenance_dir: + prov.provenance_dir = provenance_dir return prov @thread_safe @@ -179,7 +184,8 @@ def register_source(self, id: str, name: str = "", url: str = "", - properties: dict[str, str] = None) -> Source: + properties: dict[str, str] = None, + provenance_dir: str = "") -> Source: self.has_custom_mcf_nodes = True clean_id = _clean_metadata_id(id) @@ -188,7 +194,8 @@ def register_source(self, src = Source(id=clean_id, name=name or clean_id, url=url, - properties=properties or {}) + properties=properties or {}, + provenance_dir=provenance_dir or "") self.sources[clean_id] = src if name: self.sources[name] = src @@ -201,6 +208,8 @@ def register_source(self, src.url = url if properties: src.properties.update(properties) + if provenance_dir and not src.provenance_dir: + src.provenance_dir = provenance_dir return src @thread_safe @@ -357,9 +366,17 @@ def _default_custom_group(self) -> StatVarGroup: return self.groups[_DEFAULT_CUSTOM_GROUP_PATH] @thread_safe - def entity_with_type(self, entity_dcid: str, entity_type: str): + def entity_with_type(self, + entity_dcid: str, + entity_type: str, + provenance_dir: str = ""): if entity_dcid not in self.entities: - self.entities[entity_dcid] = Entity(entity_dcid, entity_type) + self.entities[entity_dcid] = Entity(entity_dcid, + entity_type, + provenance_dir=provenance_dir) + elif provenance_dir and not getattr(self.entities[entity_dcid], + "provenance_dir", ""): + self.entities[entity_dcid].provenance_dir = provenance_dir @thread_safe def has_entity(self, entity_dcid: str) -> bool: @@ -377,18 +394,27 @@ def get_provenance_urls(self) -> dict[str, str]: return prov_urls @thread_safe - def entities_with_type(self, entity_dcids: list[str], entity_type: str): + def entities_with_type(self, + entity_dcids: list[str], + entity_type: str, + provenance_dir: str = ""): for entity_dcid in entity_dcids: - self.entity_with_type(entity_dcid, entity_type) + self.entity_with_type(entity_dcid, + entity_type, + provenance_dir=provenance_dir) @thread_safe - def entities_with_types(self, dcid2type: dict[str, str]): + def entities_with_types(self, + dcid2type: dict[str, str], + provenance_dir: str = ""): """ Adds each dcid2type mapping to the list of entities with their types. The full list will be inserted into the DB in the final stages of the import. """ for entity_dcid, entity_type in dcid2type.items(): - self.entity_with_type(entity_dcid, entity_type) + self.entity_with_type(entity_dcid, + entity_type, + provenance_dir=provenance_dir) @thread_safe def triples(self, triples_file: File | None = None) -> list[Triple]: @@ -420,6 +446,42 @@ def triples(self, triples_file: File | None = None) -> list[Triple]: return triples + @thread_safe + def triples_by_provenance_dir(self) -> dict[str, list[Triple]]: + result: dict[str, list[Triple]] = defaultdict(list) + for source in self.sources.values(): + if self.has_custom_mcf_nodes and source.id == _DEFAULT_SOURCE.id and _DEFAULT_SOURCE.id not in self._used_source_ids: + continue + imp = getattr(source, "provenance_dir", "") or "_global" + result[imp].extend(source.triples()) + for provenance in self.provenances.values(): + if self.has_custom_mcf_nodes and provenance.id == _DEFAULT_PROVENANCE.id and _DEFAULT_PROVENANCE.id not in self._used_provenance_ids: + continue + imp = getattr(provenance, + "provenance_dir", "") or (strip_namespace(provenance.id) + if provenance.id else "_global") + result[imp].extend(provenance.triples()) + for group in self.groups.values(): + result["_global"].extend(group.triples()) + for variable in self.variables.values(): + imp = strip_namespace(variable.provenance_ids[0]) if getattr( + variable, "provenance_ids", None) else "_global" + result[imp].extend(variable.triples()) + for event_type in self.event_types.values(): + imp = strip_namespace(event_type.provenance_ids[0]) if getattr( + event_type, "provenance_ids", None) else "_global" + result[imp].extend(event_type.triples()) + for entity_type in self.entity_types.values(): + imp = strip_namespace(entity_type.provenance_ids[0]) if getattr( + entity_type, "provenance_ids", None) else "_global" + result[imp].extend(entity_type.triples()) + for property in self.properties.values(): + result["_global"].extend(property.triples()) + for entity in self.entities.values(): + imp = getattr(entity, "provenance_dir", "") or "_global" + result[imp].extend(entity.triples()) + return result + def _clean_metadata_id(id: str) -> str: """Only strips standard Data Commons system prefixes, preserving custom namespaces.""" diff --git a/simple/stats/observations_importer.py b/simple/stats/observations_importer.py index dc5ab0ec..483e0393 100644 --- a/simple/stats/observations_importer.py +++ b/simple/stats/observations_importer.py @@ -164,11 +164,14 @@ def _add_entity_nodes(self) -> None: if dcid2type: logging.info("Importing %s of %s entities.", len(dcid2type), len(new_entity_dcids)) - self.nodes.entities_with_types(dcid2type) + self.nodes.entities_with_types(dcid2type, + provenance_dir=self.import_name) elif self.entity_type: logging.info("Importing %s entities with type %s.", len(new_entity_dcids), self.entity_type) - self.nodes.entities_with_type(new_entity_dcids, self.entity_type) + self.nodes.entities_with_type(new_entity_dcids, + self.entity_type, + provenance_dir=self.import_name) def _resolve_entities(self) -> None: df = self.df diff --git a/simple/stats/runner.py b/simple/stats/runner.py index 428d00e2..4baecf5b 100644 --- a/simple/stats/runner.py +++ b/simple/stats/runner.py @@ -1051,14 +1051,19 @@ def _run_all_data_imports(self): for src in res.sources.values(): self.nodes.register_source(id=src.id, name=src.name, - url=src.url) + url=src.url, + provenance_dir=getattr( + src, "provenance_dir", "")) if res.provenances: for prov in res.provenances.values(): self.nodes.register_provenance(id=prov.id, name=prov.name, url=prov.url, source_id=prov.source_id, - properties=prov.properties) + properties=prov.properties, + provenance_dir=getattr( + prov, "provenance_dir", + "")) if res.groups: self.nodes.groups.update(res.groups) for svg in res.groups.values(): @@ -1132,9 +1137,12 @@ def _run_imports_and_export_jsonld(self): # Run data imports (CSV and MCF) self._run_all_data_imports() - # Generate triples from nodes and write directly - triples = self.nodes.triples() - self.db.insert_triples(triples) + # Generate triples from nodes grouped by provenance directory and write directly + for prov_dir, triples in self.nodes.triples_by_provenance_dir().items(): + if prov_dir == "_global": + self.db.insert_triples(triples) + else: + self.db.insert_triples(triples, provenance_dir=prov_dir) # Perform strict metadata validation before committing and closing MetadataValidator(self.config, self.db).validate() From 1031695efbcaae3dffb3eac34cd78f3893c5b5af Mon Sep 17 00:00:00 2001 From: Gabriel Mechali Date: Sat, 25 Jul 2026 20:13:41 -0400 Subject: [PATCH 34/45] test fix --- simple/stats/observations_importer.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/simple/stats/observations_importer.py b/simple/stats/observations_importer.py index 483e0393..a40f6e8a 100644 --- a/simple/stats/observations_importer.py +++ b/simple/stats/observations_importer.py @@ -164,14 +164,15 @@ def _add_entity_nodes(self) -> None: if dcid2type: logging.info("Importing %s of %s entities.", len(dcid2type), len(new_entity_dcids)) - self.nodes.entities_with_types(dcid2type, - provenance_dir=self.import_name) + self.nodes.entities_with_types( + dcid2type, provenance_dir=self.config.import_name(self.input_file)) elif self.entity_type: logging.info("Importing %s entities with type %s.", len(new_entity_dcids), self.entity_type) - self.nodes.entities_with_type(new_entity_dcids, - self.entity_type, - provenance_dir=self.import_name) + self.nodes.entities_with_type( + new_entity_dcids, + self.entity_type, + provenance_dir=self.config.import_name(self.input_file)) def _resolve_entities(self) -> None: df = self.df From e7449388300b17bd909ee50a6bc9087933c50a0b Mon Sep 17 00:00:00 2001 From: Gabriel Mechali Date: Sun, 26 Jul 2026 09:23:18 -0400 Subject: [PATCH 35/45] Preserve import of entities --- simple/stats/runner.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/simple/stats/runner.py b/simple/stats/runner.py index 4baecf5b..fd020d93 100644 --- a/simple/stats/runner.py +++ b/simple/stats/runner.py @@ -220,7 +220,9 @@ def _run_single_csv_import_proc( db.commit_and_close() resolved_entities = { - e.entity_dcid: e.entity_type for e in nodes.entities.values() + e.entity_dcid: + (e.entity_type, getattr(e, "provenance_dir", "")) + for e in nodes.entities.values() } event_types = dict(nodes.event_types) entity_types = dict(nodes.entity_types) @@ -1040,7 +1042,14 @@ def _run_all_data_imports(self): res = future.result() self._log_file_progress("Imported file", res.file_rel_path) if res.resolved_entities: - self.nodes.entities_with_types(res.resolved_entities) + for dcid, val in res.resolved_entities.items(): + if isinstance(val, tuple): + t, p_dir = val + else: + t, p_dir = val, "" + self.nodes.entity_with_type(dcid, + t, + provenance_dir=p_dir) if res.event_types: self.nodes.event_types.update(res.event_types) if res.entity_types: From b82d1e392334503bd48826e9ca1e3d32c8502b18 Mon Sep 17 00:00:00 2001 From: Gabriel Mechali Date: Sun, 26 Jul 2026 09:35:49 -0400 Subject: [PATCH 36/45] Bring in nodes to all references provs --- simple/stats/data.py | 2 +- simple/stats/jsonld_stream_db.py | 20 ++++++++++---------- simple/stats/nodes.py | 17 +++++++++-------- simple/stats/runner.py | 18 ++++++++++++------ 4 files changed, 32 insertions(+), 25 deletions(-) diff --git a/simple/stats/data.py b/simple/stats/data.py index 77244f67..ba1f80a7 100644 --- a/simple/stats/data.py +++ b/simple/stats/data.py @@ -182,7 +182,7 @@ def triples(self) -> list[Triple]: class Entity: entity_dcid: str entity_type: str - provenance_dir: str = "" + provenance_dirs: set[str] = field(default_factory=set) def triples(self) -> list[Triple]: # Currently only 1 triple is generated but could be more in the future (e.g. name) diff --git a/simple/stats/jsonld_stream_db.py b/simple/stats/jsonld_stream_db.py index 7d831e0f..16ac409d 100644 --- a/simple/stats/jsonld_stream_db.py +++ b/simple/stats/jsonld_stream_db.py @@ -487,21 +487,21 @@ def commit(self): pass def commit_and_close(self): - # Add global triples to every processed import's triples global_triples = self._triples.pop("_global", []) if not self._processed_imports: self._processed_imports.add(self.import_name) - # Write global triples as node shards to the local temp directory for each import - for import_name in self._processed_imports: - import_temp_dir = os.path.join(self.temp_local_dir, import_name) + # Write any remaining untagged triples once to the primary import directory (no duplication across provenances) + if global_triples: + target_import = self.import_name if self.import_name in self._processed_imports else next( + iter(self._processed_imports)) + import_temp_dir = os.path.join(self.temp_local_dir, target_import) os.makedirs(import_temp_dir, exist_ok=True) - if global_triples: - for i in range(0, len(global_triples), _CHUNK_SIZE): - chunk = global_triples[i:i + _CHUNK_SIZE] - _write_node_shard( - (chunk, self.node_shard_index, import_temp_dir, self.ns_map)) - self.node_shard_index += 1 + for i in range(0, len(global_triples), _CHUNK_SIZE): + chunk = global_triples[i:i + _CHUNK_SIZE] + _write_node_shard( + (chunk, self.node_shard_index, import_temp_dir, self.ns_map)) + self.node_shard_index += 1 has_local_files = any(os.scandir(self.temp_local_dir)) if os.path.exists( self.temp_local_dir) else False diff --git a/simple/stats/nodes.py b/simple/stats/nodes.py index 241b996f..7b8899bf 100644 --- a/simple/stats/nodes.py +++ b/simple/stats/nodes.py @@ -371,12 +371,9 @@ def entity_with_type(self, entity_type: str, provenance_dir: str = ""): if entity_dcid not in self.entities: - self.entities[entity_dcid] = Entity(entity_dcid, - entity_type, - provenance_dir=provenance_dir) - elif provenance_dir and not getattr(self.entities[entity_dcid], - "provenance_dir", ""): - self.entities[entity_dcid].provenance_dir = provenance_dir + self.entities[entity_dcid] = Entity(entity_dcid, entity_type) + if provenance_dir: + self.entities[entity_dcid].provenance_dirs.add(provenance_dir) @thread_safe def has_entity(self, entity_dcid: str) -> bool: @@ -478,8 +475,12 @@ def triples_by_provenance_dir(self) -> dict[str, list[Triple]]: for property in self.properties.values(): result["_global"].extend(property.triples()) for entity in self.entities.values(): - imp = getattr(entity, "provenance_dir", "") or "_global" - result[imp].extend(entity.triples()) + dirs = getattr(entity, "provenance_dirs", set()) + if not dirs: + result["_global"].extend(entity.triples()) + else: + for p_dir in dirs: + result[p_dir].extend(entity.triples()) return result diff --git a/simple/stats/runner.py b/simple/stats/runner.py index fd020d93..67980ffa 100644 --- a/simple/stats/runner.py +++ b/simple/stats/runner.py @@ -221,7 +221,7 @@ def _run_single_csv_import_proc( resolved_entities = { e.entity_dcid: - (e.entity_type, getattr(e, "provenance_dir", "")) + (e.entity_type, getattr(e, "provenance_dirs", set())) for e in nodes.entities.values() } event_types = dict(nodes.event_types) @@ -1044,12 +1044,18 @@ def _run_all_data_imports(self): if res.resolved_entities: for dcid, val in res.resolved_entities.items(): if isinstance(val, tuple): - t, p_dir = val + t, p_dirs = val else: - t, p_dir = val, "" - self.nodes.entity_with_type(dcid, - t, - provenance_dir=p_dir) + t, p_dirs = val, set() + if isinstance(p_dirs, set): + for p_dir in (p_dirs or [""]): + self.nodes.entity_with_type(dcid, + t, + provenance_dir=p_dir) + else: + self.nodes.entity_with_type(dcid, + t, + provenance_dir=p_dirs) if res.event_types: self.nodes.event_types.update(res.event_types) if res.entity_types: From d70f64912b71af76fa50dac22d06901479c15da5 Mon Sep 17 00:00:00 2001 From: Gabriel Mechali Date: Sun, 26 Jul 2026 09:46:15 -0400 Subject: [PATCH 37/45] single prov fix --- simple/stats/data.py | 1 + simple/stats/nodes.py | 40 +++++++++++++++++++++++++++++----------- simple/stats/runner.py | 7 ++++++- 3 files changed, 36 insertions(+), 12 deletions(-) diff --git a/simple/stats/data.py b/simple/stats/data.py index ba1f80a7..3569526f 100644 --- a/simple/stats/data.py +++ b/simple/stats/data.py @@ -335,6 +335,7 @@ def _get_flattened_dataclass_field_names(cls) -> list[str]: class Property: dcid: str name: str + provenance_dirs: set[str] = field(default_factory=set) def triples(self) -> list[Triple]: return [ diff --git a/simple/stats/nodes.py b/simple/stats/nodes.py index 7b8899bf..28479c6f 100644 --- a/simple/stats/nodes.py +++ b/simple/stats/nodes.py @@ -247,10 +247,14 @@ def variable(self, sv_column_name: str, input_file: File) -> StatVar: self.provenance(input_file)) @thread_safe - def property(self, property_column_name: str) -> Property: + def property(self, + property_column_name: str, + provenance_dir: str = "") -> Property: if not property_column_name in self.properties: self.properties[property_column_name] = Property( self._property_id(property_column_name), property_column_name) + if provenance_dir: + self.properties[property_column_name].provenance_dirs.add(provenance_dir) return self.properties[property_column_name] @@ -461,19 +465,33 @@ def triples_by_provenance_dir(self) -> dict[str, list[Triple]]: for group in self.groups.values(): result["_global"].extend(group.triples()) for variable in self.variables.values(): - imp = strip_namespace(variable.provenance_ids[0]) if getattr( - variable, "provenance_ids", None) else "_global" - result[imp].extend(variable.triples()) + p_ids = getattr(variable, "provenance_ids", []) + if not p_ids: + result["_global"].extend(variable.triples()) + else: + for p_id in p_ids: + result[strip_namespace(p_id)].extend(variable.triples()) for event_type in self.event_types.values(): - imp = strip_namespace(event_type.provenance_ids[0]) if getattr( - event_type, "provenance_ids", None) else "_global" - result[imp].extend(event_type.triples()) + p_ids = getattr(event_type, "provenance_ids", []) + if not p_ids: + result["_global"].extend(event_type.triples()) + else: + for p_id in p_ids: + result[strip_namespace(p_id)].extend(event_type.triples()) for entity_type in self.entity_types.values(): - imp = strip_namespace(entity_type.provenance_ids[0]) if getattr( - entity_type, "provenance_ids", None) else "_global" - result[imp].extend(entity_type.triples()) + p_ids = getattr(entity_type, "provenance_ids", []) + if not p_ids: + result["_global"].extend(entity_type.triples()) + else: + for p_id in p_ids: + result[strip_namespace(p_id)].extend(entity_type.triples()) for property in self.properties.values(): - result["_global"].extend(property.triples()) + dirs = getattr(property, "provenance_dirs", set()) + if not dirs: + result["_global"].extend(property.triples()) + else: + for p_dir in dirs: + result[p_dir].extend(property.triples()) for entity in self.entities.values(): dirs = getattr(entity, "provenance_dirs", set()) if not dirs: diff --git a/simple/stats/runner.py b/simple/stats/runner.py index 67980ffa..5ba91e51 100644 --- a/simple/stats/runner.py +++ b/simple/stats/runner.py @@ -1084,7 +1084,12 @@ def _run_all_data_imports(self): for svg in res.groups.values(): self.nodes.ids_to_groups[svg.id] = svg if res.properties: - self.nodes.properties.update(res.properties) + for prop_name, prop_obj in res.properties.items(): + if prop_name not in self.nodes.properties: + self.nodes.properties[prop_name] = prop_obj + else: + self.nodes.properties[prop_name].provenance_dirs.update( + getattr(prop_obj, "provenance_dirs", set())) if res.processed_imports: self.db._processed_imports.update(res.processed_imports) if res.obs_collision_count and hasattr(self.db, From 413fa7ed21da25d060338c95b9f0ccbefef071f2 Mon Sep 17 00:00:00 2001 From: Gabriel Mechali Date: Sun, 26 Jul 2026 10:14:24 -0400 Subject: [PATCH 38/45] Fixing the per import for country res --- simple/stats/events_importer.py | 11 +++++++++++ simple/stats/observations_importer.py | 5 +++++ simple/stats/variable_per_row_importer.py | 10 ++++++++-- 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/simple/stats/events_importer.py b/simple/stats/events_importer.py index 5df86099..9cd9cc33 100644 --- a/simple/stats/events_importer.py +++ b/simple/stats/events_importer.py @@ -20,6 +20,7 @@ from stats.data import AggregationConfig from stats.data import Event from stats.data import filter_invalid_observation_values +from stats.data import strip_namespace from stats.data import strip_namespace_series from stats.data import TimePeriod from stats.data import Triple @@ -247,6 +248,16 @@ def remove_pre_resolved(entity: str) -> bool: entities = list(filter(remove_pre_resolved, column.tolist())) + prov_dir = self.config.import_name(self.input_file) + for dcid in column.tolist(): + clean_dcid = strip_namespace(dcid) + if self.nodes.has_entity(clean_dcid): + self.nodes.entities[clean_dcid].provenance_dirs.add(prov_dir) + else: + self.nodes.entity_with_type(clean_dcid, + "Thing", + provenance_dir=prov_dir) + logging.info("Found %s entities pre-resolved.", len(pre_resolved_entities)) logging.info("Resolving %s entities of type %s.", len(entities), diff --git a/simple/stats/observations_importer.py b/simple/stats/observations_importer.py index a40f6e8a..b2ec9662 100644 --- a/simple/stats/observations_importer.py +++ b/simple/stats/observations_importer.py @@ -149,6 +149,11 @@ def _add_entity_nodes(self) -> None: dcid for dcid in entity_dcids if not self.nodes.has_entity(dcid) ] + prov_dir = self.config.import_name(self.input_file) + for dcid in entity_dcids: + if self.nodes.has_entity(dcid): + self.nodes.entities[dcid].provenance_dirs.add(prov_dir) + logging.info("Found %s total entities, of which %s are already imported.", len(entity_dcids), len(entity_dcids) - len(new_entity_dcids)) diff --git a/simple/stats/variable_per_row_importer.py b/simple/stats/variable_per_row_importer.py index b68c32ae..9d7482fd 100644 --- a/simple/stats/variable_per_row_importer.py +++ b/simple/stats/variable_per_row_importer.py @@ -337,9 +337,15 @@ def _add_entity_nodes(self) -> None: new_entity_dcids = [ strip_namespace(dcid) for dcid in self.entity_dcids - if not self.nodes.has_entity(dcid) + if not self.nodes.has_entity(strip_namespace(dcid)) ] + prov_dir = self.config.import_name(self.input_file) + for dcid in self.entity_dcids: + clean_dcid = strip_namespace(dcid) + if self.nodes.has_entity(clean_dcid): + self.nodes.entities[clean_dcid].provenance_dirs.add(prov_dir) + logging.info("Found %s total entities, of which %s are already imported.", len(self.entity_dcids), len(self.entity_dcids) - len(new_entity_dcids)) @@ -357,4 +363,4 @@ def _add_entity_nodes(self) -> None: if dcid2type: logging.info("Importing %s of %s entities.", len(dcid2type), len(new_entity_dcids)) - self.nodes.entities_with_types(dcid2type) + self.nodes.entities_with_types(dcid2type, provenance_dir=prov_dir) From ac9800ccb6283cb1488d81869f5b60a788e2a48d Mon Sep 17 00:00:00 2001 From: Gabriel Mechali Date: Mon, 27 Jul 2026 11:35:38 -0400 Subject: [PATCH 39/45] Error catching - small nits --- simple/stats/events_importer.py | 9 ++++++--- simple/stats/observations_importer.py | 7 +++++-- simple/stats/variable_per_row_importer.py | 7 +++++-- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/simple/stats/events_importer.py b/simple/stats/events_importer.py index 9cd9cc33..e83c28a6 100644 --- a/simple/stats/events_importer.py +++ b/simple/stats/events_importer.py @@ -248,12 +248,15 @@ def remove_pre_resolved(entity: str) -> bool: entities = list(filter(remove_pre_resolved, column.tolist())) - prov_dir = self.config.import_name(self.input_file) + try: + prov_dir = self.config.import_name(self.input_file) + except Exception: + prov_dir = "" for dcid in column.tolist(): clean_dcid = strip_namespace(dcid) - if self.nodes.has_entity(clean_dcid): + if self.nodes.has_entity(clean_dcid) and prov_dir: self.nodes.entities[clean_dcid].provenance_dirs.add(prov_dir) - else: + elif prov_dir: self.nodes.entity_with_type(clean_dcid, "Thing", provenance_dir=prov_dir) diff --git a/simple/stats/observations_importer.py b/simple/stats/observations_importer.py index b2ec9662..57b8c66f 100644 --- a/simple/stats/observations_importer.py +++ b/simple/stats/observations_importer.py @@ -149,9 +149,12 @@ def _add_entity_nodes(self) -> None: dcid for dcid in entity_dcids if not self.nodes.has_entity(dcid) ] - prov_dir = self.config.import_name(self.input_file) + try: + prov_dir = self.config.import_name(self.input_file) + except Exception: + prov_dir = "" for dcid in entity_dcids: - if self.nodes.has_entity(dcid): + if self.nodes.has_entity(dcid) and prov_dir: self.nodes.entities[dcid].provenance_dirs.add(prov_dir) logging.info("Found %s total entities, of which %s are already imported.", diff --git a/simple/stats/variable_per_row_importer.py b/simple/stats/variable_per_row_importer.py index 9d7482fd..69108819 100644 --- a/simple/stats/variable_per_row_importer.py +++ b/simple/stats/variable_per_row_importer.py @@ -340,10 +340,13 @@ def _add_entity_nodes(self) -> None: if not self.nodes.has_entity(strip_namespace(dcid)) ] - prov_dir = self.config.import_name(self.input_file) + try: + prov_dir = self.config.import_name(self.input_file) + except Exception: + prov_dir = "" for dcid in self.entity_dcids: clean_dcid = strip_namespace(dcid) - if self.nodes.has_entity(clean_dcid): + if self.nodes.has_entity(clean_dcid) and prov_dir: self.nodes.entities[clean_dcid].provenance_dirs.add(prov_dir) logging.info("Found %s total entities, of which %s are already imported.", From 7618385fbdedc9ecbfeb1838350f7296c00e16cb Mon Sep 17 00:00:00 2001 From: Gabriel Mechali Date: Mon, 27 Jul 2026 12:02:12 -0400 Subject: [PATCH 40/45] Lint + adds comment for name collisions --- simple/stats/jsonld_stream_db.py | 15 ++++++--------- simple/stats/nodes.py | 8 ++++---- simple/stats/observations_importer.py | 13 +++++++------ simple/stats/runner.py | 11 +++-------- .../tests/stats/observations_importer_test.py | 19 +++++++++---------- 5 files changed, 29 insertions(+), 37 deletions(-) diff --git a/simple/stats/jsonld_stream_db.py b/simple/stats/jsonld_stream_db.py index 16ac409d..973bcfa8 100644 --- a/simple/stats/jsonld_stream_db.py +++ b/simple/stats/jsonld_stream_db.py @@ -102,21 +102,13 @@ def _parse_numeric(val): return str(val) -def _write_observation_shard(chunk_or_args, +def _write_observation_shard(chunk, shard_index: Optional[int] = None, jsonld_dir_path: Optional[str] = None, ns_map: Optional[dict[str, str]] = None, prov_urls: Optional[dict[str, str]] = None, track_hash_fn: Optional[Callable] = None, file_name: str = ""): - if isinstance(chunk_or_args, tuple): - chunk = chunk_or_args[0] - shard_index = chunk_or_args[1] - jsonld_dir_path = chunk_or_args[2] - ns_map = chunk_or_args[3] - prov_urls = chunk_or_args[4] - else: - chunk = chunk_or_args graph_list = [] chunk_hashes = [] @@ -207,6 +199,9 @@ def _write_observation_shard(chunk_or_args, logging.info(f"Saved JSON-LD shard to {shard_name}") +# TODO(gmechali): When parallelizing node shard exports (MCF and Events CSV files), +# include the sanitized file_name in the shard name (e.g., node-{sanitized_stem}-{shard_index:05d}.jsonld) +# to prevent shard filename collisions across parallel workers, similar to observation shards. def _write_node_shard(args): # TODO(gmechali): Get rid of this and keep only the "fast" mode. fast_export = os.getenv("FAST_NODE_EXPORT", @@ -268,6 +263,8 @@ def _write_node_shard_fast(args): graph_list = sorted(list(subjects.values()), key=lambda x: x["@id"]) compacted_jsonld = {"@context": ns_map, "@graph": graph_list} + # TODO(gmechali): When parallelizing node shard exports, include file_name in shard_name + # (e.g., f"node-{sanitized_stem}-{shard_index:05d}.jsonld") to prevent collisions across workers. shard_name = f"node-{shard_index:05d}.jsonld" with create_store(jsonld_dir_path) as store: output_dir = store.as_dir() diff --git a/simple/stats/nodes.py b/simple/stats/nodes.py index 28479c6f..7d00c8cf 100644 --- a/simple/stats/nodes.py +++ b/simple/stats/nodes.py @@ -160,7 +160,8 @@ def register_provenance(self, name=name or clean_id, url=url, properties=properties or {}, - provenance_dir=provenance_dir or strip_namespace(clean_id)) + provenance_dir=provenance_dir or + strip_namespace(clean_id)) self.provenances[clean_id] = prov if name: self.provenances[name] = prov @@ -458,9 +459,8 @@ def triples_by_provenance_dir(self) -> dict[str, list[Triple]]: for provenance in self.provenances.values(): if self.has_custom_mcf_nodes and provenance.id == _DEFAULT_PROVENANCE.id and _DEFAULT_PROVENANCE.id not in self._used_provenance_ids: continue - imp = getattr(provenance, - "provenance_dir", "") or (strip_namespace(provenance.id) - if provenance.id else "_global") + imp = getattr(provenance, "provenance_dir", "") or (strip_namespace( + provenance.id) if provenance.id else "_global") result[imp].extend(provenance.triples()) for group in self.groups.values(): result["_global"].extend(group.triples()) diff --git a/simple/stats/observations_importer.py b/simple/stats/observations_importer.py index 57b8c66f..0d18a8ff 100644 --- a/simple/stats/observations_importer.py +++ b/simple/stats/observations_importer.py @@ -172,15 +172,16 @@ def _add_entity_nodes(self) -> None: if dcid2type: logging.info("Importing %s of %s entities.", len(dcid2type), len(new_entity_dcids)) - self.nodes.entities_with_types( - dcid2type, provenance_dir=self.config.import_name(self.input_file)) + self.nodes.entities_with_types(dcid2type, + provenance_dir=self.config.import_name( + self.input_file)) elif self.entity_type: logging.info("Importing %s entities with type %s.", len(new_entity_dcids), self.entity_type) - self.nodes.entities_with_type( - new_entity_dcids, - self.entity_type, - provenance_dir=self.config.import_name(self.input_file)) + self.nodes.entities_with_type(new_entity_dcids, + self.entity_type, + provenance_dir=self.config.import_name( + self.input_file)) def _resolve_entities(self) -> None: df = self.df diff --git a/simple/stats/runner.py b/simple/stats/runner.py index 5ba91e51..93a7a65e 100644 --- a/simple/stats/runner.py +++ b/simple/stats/runner.py @@ -220,8 +220,7 @@ def _run_single_csv_import_proc( db.commit_and_close() resolved_entities = { - e.entity_dcid: - (e.entity_type, getattr(e, "provenance_dirs", set())) + e.entity_dcid: (e.entity_type, getattr(e, "provenance_dirs", set())) for e in nodes.entities.values() } event_types = dict(nodes.event_types) @@ -1049,13 +1048,9 @@ def _run_all_data_imports(self): t, p_dirs = val, set() if isinstance(p_dirs, set): for p_dir in (p_dirs or [""]): - self.nodes.entity_with_type(dcid, - t, - provenance_dir=p_dir) + self.nodes.entity_with_type(dcid, t, provenance_dir=p_dir) else: - self.nodes.entity_with_type(dcid, - t, - provenance_dir=p_dirs) + self.nodes.entity_with_type(dcid, t, provenance_dir=p_dirs) if res.event_types: self.nodes.event_types.update(res.event_types) if res.entity_types: diff --git a/simple/tests/stats/observations_importer_test.py b/simple/tests/stats/observations_importer_test.py index be06f4b1..1a7934ab 100644 --- a/simple/tests/stats/observations_importer_test.py +++ b/simple/tests/stats/observations_importer_test.py @@ -125,19 +125,18 @@ def test_custom_namespace_pre_resolution(self): ) importer.entity_type = "State" importer.entity_column_name = "entity" - importer.df = pd.DataFrame({ - "dcid": ["undata:place/custom_1", "dcid:geoId/06", "California"] - }) + importer.df = pd.DataFrame( + {"dcid": ["undata:place/custom_1", "dcid:geoId/06", "California"]}) - dc_client.resolve_entities = MagicMock(return_value={"California": "geoId/06"}) + dc_client.resolve_entities = MagicMock( + return_value={"California": "geoId/06"}) dc_client.get_property_of_entities = MagicMock(return_value={}) importer._resolve_entities() dc_client.resolve_entities.assert_called_once_with( - entities=["California"], entity_type="State", property_name="description" - ) - self.assertEqual( - importer.df["dcid"].tolist(), - ["place/custom_1", "geoId/06", "geoId/06"] - ) + entities=["California"], + entity_type="State", + property_name="description") + self.assertEqual(importer.df["dcid"].tolist(), + ["place/custom_1", "geoId/06", "geoId/06"]) From 0e9f8c65af06dc1fb16dbe273f645a26f9cd0c8b Mon Sep 17 00:00:00 2001 From: Gabriel Mechali Date: Mon, 27 Jul 2026 12:56:56 -0400 Subject: [PATCH 41/45] Rename provenance_dir to just using the prov id --- simple/stats/data.py | 8 +-- simple/stats/events_importer.py | 18 +++---- simple/stats/mcf_importer.py | 15 ++---- simple/stats/nodes.py | 61 +++++++++++------------ simple/stats/observations_importer.py | 19 +++---- simple/stats/runner.py | 27 +++++----- simple/stats/variable_per_row_importer.py | 16 +++--- 7 files changed, 73 insertions(+), 91 deletions(-) diff --git a/simple/stats/data.py b/simple/stats/data.py index 3569526f..e9c5524e 100644 --- a/simple/stats/data.py +++ b/simple/stats/data.py @@ -182,7 +182,7 @@ def triples(self) -> list[Triple]: class Entity: entity_dcid: str entity_type: str - provenance_dirs: set[str] = field(default_factory=set) + provenance_ids: set[str] = field(default_factory=set) def triples(self) -> list[Triple]: # Currently only 1 triple is generated but could be more in the future (e.g. name) @@ -198,7 +198,7 @@ class Provenance: name: str url: str = "" properties: dict[str, str] = field(default_factory=dict) - provenance_dir: str = "" + provenance_id: str = "" def triples(self) -> list[Triple]: triples: list[Triple] = [] @@ -230,7 +230,7 @@ class Source: url: str = "" domain: str = field(init=False) properties: dict[str, str] = field(default_factory=dict) - provenance_dir: str = "" + provenance_id: str = "" def __post_init__(self): self.domain = urlparse(self.url).netloc @@ -335,7 +335,7 @@ def _get_flattened_dataclass_field_names(cls) -> list[str]: class Property: dcid: str name: str - provenance_dirs: set[str] = field(default_factory=set) + provenance_ids: set[str] = field(default_factory=set) def triples(self) -> list[Triple]: return [ diff --git a/simple/stats/events_importer.py b/simple/stats/events_importer.py index e83c28a6..ccb19cd7 100644 --- a/simple/stats/events_importer.py +++ b/simple/stats/events_importer.py @@ -248,18 +248,16 @@ def remove_pre_resolved(entity: str) -> bool: entities = list(filter(remove_pre_resolved, column.tolist())) - try: - prov_dir = self.config.import_name(self.input_file) - except Exception: - prov_dir = "" + prov_id = getattr(self, "provenance", "") for dcid in column.tolist(): clean_dcid = strip_namespace(dcid) - if self.nodes.has_entity(clean_dcid) and prov_dir: - self.nodes.entities[clean_dcid].provenance_dirs.add(prov_dir) - elif prov_dir: - self.nodes.entity_with_type(clean_dcid, - "Thing", - provenance_dir=prov_dir) + if self.nodes.has_entity(clean_dcid): + if prov_id: + self.nodes.entities[clean_dcid].provenance_ids.add(prov_id) + else: + self.nodes.entity_with_type( + clean_dcid, "Thing", provenance_id=prov_id + ) logging.info("Found %s entities pre-resolved.", len(pre_resolved_entities)) diff --git a/simple/stats/mcf_importer.py b/simple/stats/mcf_importer.py index 0dee3237..5780fe6b 100644 --- a/simple/stats/mcf_importer.py +++ b/simple/stats/mcf_importer.py @@ -105,14 +105,10 @@ def do_import(self) -> None: # Register all collected metadata nodes at the end if all_metadata_triples: - prov_dir = "" - try: - prov_dir = self.config.import_name(self.input_file) - except Exception: - pass + prov_id = getattr(self, "provenance", "") _register_metadata_nodes(all_metadata_triples, self.nodes, - provenance_dir=prov_dir) + provenance_id=prov_id) self.reporter.report_success() except Exception as e: @@ -170,7 +166,7 @@ def _to_triple(parser_triple: list[str], local2dcid: dict[str, str]) -> Triple: def _register_metadata_nodes(triples: list[Triple], nodes: Nodes, - provenance_dir: str = "") -> None: + provenance_id: str = "") -> None: """Extracts and registers Provenance and Source nodes from parsed MCF triples. This helper is robust against: @@ -209,13 +205,12 @@ def _register_metadata_nodes(triples: list[Triple], nodes.register_provenance(id=sub_id, name=props.get("name", ""), url=props.get("url", ""), - source_id=props.get("source", ""), - provenance_dir=provenance_dir) + source_id=props.get("source", "")) elif node_type == "Source": nodes.register_source(id=sub_id, name=props.get("name", ""), url=props.get("url", ""), - provenance_dir=provenance_dir) + provenance_id=provenance_id) def _clean_literal(val: str) -> str: diff --git a/simple/stats/nodes.py b/simple/stats/nodes.py index 7d00c8cf..79573655 100644 --- a/simple/stats/nodes.py +++ b/simple/stats/nodes.py @@ -147,8 +147,7 @@ def register_provenance(self, name: str = "", url: str = "", source_id: str = "", - properties: dict[str, str] = None, - provenance_dir: str = "") -> Provenance: + properties: dict[str, str] = None) -> Provenance: self.has_custom_mcf_nodes = True clean_id = _clean_metadata_id(id) clean_source_id = _clean_metadata_id(source_id) if source_id else "" @@ -160,8 +159,7 @@ def register_provenance(self, name=name or clean_id, url=url, properties=properties or {}, - provenance_dir=provenance_dir or - strip_namespace(clean_id)) + provenance_id=clean_id) self.provenances[clean_id] = prov if name: self.provenances[name] = prov @@ -176,8 +174,8 @@ def register_provenance(self, prov.source_id = clean_source_id if properties: prov.properties.update(properties) - if provenance_dir and not prov.provenance_dir: - prov.provenance_dir = provenance_dir + if not prov.provenance_id: + prov.provenance_id = clean_id return prov @thread_safe @@ -186,7 +184,7 @@ def register_source(self, name: str = "", url: str = "", properties: dict[str, str] = None, - provenance_dir: str = "") -> Source: + provenance_id: str = "") -> Source: self.has_custom_mcf_nodes = True clean_id = _clean_metadata_id(id) @@ -196,7 +194,7 @@ def register_source(self, name=name or clean_id, url=url, properties=properties or {}, - provenance_dir=provenance_dir or "") + provenance_id=provenance_id or "") self.sources[clean_id] = src if name: self.sources[name] = src @@ -209,8 +207,8 @@ def register_source(self, src.url = url if properties: src.properties.update(properties) - if provenance_dir and not src.provenance_dir: - src.provenance_dir = provenance_dir + if provenance_id and not src.provenance_id: + src.provenance_id = provenance_id return src @thread_safe @@ -250,12 +248,12 @@ def variable(self, sv_column_name: str, input_file: File) -> StatVar: @thread_safe def property(self, property_column_name: str, - provenance_dir: str = "") -> Property: + provenance_id: str = "") -> Property: if not property_column_name in self.properties: self.properties[property_column_name] = Property( self._property_id(property_column_name), property_column_name) - if provenance_dir: - self.properties[property_column_name].provenance_dirs.add(provenance_dir) + if provenance_id: + self.properties[property_column_name].provenance_ids.add(provenance_id) return self.properties[property_column_name] @@ -374,11 +372,11 @@ def _default_custom_group(self) -> StatVarGroup: def entity_with_type(self, entity_dcid: str, entity_type: str, - provenance_dir: str = ""): + provenance_id: str = ""): if entity_dcid not in self.entities: self.entities[entity_dcid] = Entity(entity_dcid, entity_type) - if provenance_dir: - self.entities[entity_dcid].provenance_dirs.add(provenance_dir) + if provenance_id: + self.entities[entity_dcid].provenance_ids.add(provenance_id) @thread_safe def has_entity(self, entity_dcid: str) -> bool: @@ -399,16 +397,16 @@ def get_provenance_urls(self) -> dict[str, str]: def entities_with_type(self, entity_dcids: list[str], entity_type: str, - provenance_dir: str = ""): + provenance_id: str = ""): for entity_dcid in entity_dcids: self.entity_with_type(entity_dcid, entity_type, - provenance_dir=provenance_dir) + provenance_id=provenance_id) @thread_safe def entities_with_types(self, dcid2type: dict[str, str], - provenance_dir: str = ""): + provenance_id: str = ""): """ Adds each dcid2type mapping to the list of entities with their types. The full list will be inserted into the DB in the final stages of the import. @@ -416,7 +414,7 @@ def entities_with_types(self, for entity_dcid, entity_type in dcid2type.items(): self.entity_with_type(entity_dcid, entity_type, - provenance_dir=provenance_dir) + provenance_id=provenance_id) @thread_safe def triples(self, triples_file: File | None = None) -> list[Triple]: @@ -454,13 +452,14 @@ def triples_by_provenance_dir(self) -> dict[str, list[Triple]]: for source in self.sources.values(): if self.has_custom_mcf_nodes and source.id == _DEFAULT_SOURCE.id and _DEFAULT_SOURCE.id not in self._used_source_ids: continue - imp = getattr(source, "provenance_dir", "") or "_global" + raw = getattr(source, "provenance_id", "") or "_global" + imp = strip_namespace(raw) if raw != "_global" else "_global" result[imp].extend(source.triples()) for provenance in self.provenances.values(): if self.has_custom_mcf_nodes and provenance.id == _DEFAULT_PROVENANCE.id and _DEFAULT_PROVENANCE.id not in self._used_provenance_ids: continue - imp = getattr(provenance, "provenance_dir", "") or (strip_namespace( - provenance.id) if provenance.id else "_global") + raw = getattr(provenance, "provenance_id", "") or provenance.id or "_global" + imp = strip_namespace(raw) if raw != "_global" else "_global" result[imp].extend(provenance.triples()) for group in self.groups.values(): result["_global"].extend(group.triples()) @@ -486,19 +485,19 @@ def triples_by_provenance_dir(self) -> dict[str, list[Triple]]: for p_id in p_ids: result[strip_namespace(p_id)].extend(entity_type.triples()) for property in self.properties.values(): - dirs = getattr(property, "provenance_dirs", set()) - if not dirs: + p_ids = getattr(property, "provenance_ids", set()) + if not p_ids: result["_global"].extend(property.triples()) else: - for p_dir in dirs: - result[p_dir].extend(property.triples()) + for p_id in p_ids: + result[strip_namespace(p_id)].extend(property.triples()) for entity in self.entities.values(): - dirs = getattr(entity, "provenance_dirs", set()) - if not dirs: + p_ids = getattr(entity, "provenance_ids", set()) + if not p_ids: result["_global"].extend(entity.triples()) else: - for p_dir in dirs: - result[p_dir].extend(entity.triples()) + for p_id in p_ids: + result[strip_namespace(p_id)].extend(entity.triples()) return result diff --git a/simple/stats/observations_importer.py b/simple/stats/observations_importer.py index 0d18a8ff..7079e9d1 100644 --- a/simple/stats/observations_importer.py +++ b/simple/stats/observations_importer.py @@ -149,13 +149,11 @@ def _add_entity_nodes(self) -> None: dcid for dcid in entity_dcids if not self.nodes.has_entity(dcid) ] - try: - prov_dir = self.config.import_name(self.input_file) - except Exception: - prov_dir = "" - for dcid in entity_dcids: - if self.nodes.has_entity(dcid) and prov_dir: - self.nodes.entities[dcid].provenance_dirs.add(prov_dir) + prov_id = getattr(self, "provenance", "") + if prov_id: + for dcid in entity_dcids: + if self.nodes.has_entity(dcid): + self.nodes.entities[dcid].provenance_ids.add(prov_id) logging.info("Found %s total entities, of which %s are already imported.", len(entity_dcids), @@ -172,16 +170,13 @@ def _add_entity_nodes(self) -> None: if dcid2type: logging.info("Importing %s of %s entities.", len(dcid2type), len(new_entity_dcids)) - self.nodes.entities_with_types(dcid2type, - provenance_dir=self.config.import_name( - self.input_file)) + self.nodes.entities_with_types(dcid2type, provenance_id=prov_id) elif self.entity_type: logging.info("Importing %s entities with type %s.", len(new_entity_dcids), self.entity_type) self.nodes.entities_with_type(new_entity_dcids, self.entity_type, - provenance_dir=self.config.import_name( - self.input_file)) + provenance_id=prov_id) def _resolve_entities(self) -> None: df = self.df diff --git a/simple/stats/runner.py b/simple/stats/runner.py index 93a7a65e..64648d52 100644 --- a/simple/stats/runner.py +++ b/simple/stats/runner.py @@ -220,7 +220,7 @@ def _run_single_csv_import_proc( db.commit_and_close() resolved_entities = { - e.entity_dcid: (e.entity_type, getattr(e, "provenance_dirs", set())) + e.entity_dcid: (e.entity_type, getattr(e, "provenance_ids", set())) for e in nodes.entities.values() } event_types = dict(nodes.event_types) @@ -1043,14 +1043,14 @@ def _run_all_data_imports(self): if res.resolved_entities: for dcid, val in res.resolved_entities.items(): if isinstance(val, tuple): - t, p_dirs = val + t, p_ids = val else: - t, p_dirs = val, set() - if isinstance(p_dirs, set): - for p_dir in (p_dirs or [""]): - self.nodes.entity_with_type(dcid, t, provenance_dir=p_dir) + t, p_ids = val, set() + if isinstance(p_ids, set): + for p_id in (p_ids or [""]): + self.nodes.entity_with_type(dcid, t, provenance_id=p_id) else: - self.nodes.entity_with_type(dcid, t, provenance_dir=p_dirs) + self.nodes.entity_with_type(dcid, t, provenance_id=p_ids) if res.event_types: self.nodes.event_types.update(res.event_types) if res.entity_types: @@ -1062,18 +1062,15 @@ def _run_all_data_imports(self): self.nodes.register_source(id=src.id, name=src.name, url=src.url, - provenance_dir=getattr( - src, "provenance_dir", "")) + provenance_id=getattr( + src, "provenance_id", "")) if res.provenances: for prov in res.provenances.values(): self.nodes.register_provenance(id=prov.id, name=prov.name, url=prov.url, source_id=prov.source_id, - properties=prov.properties, - provenance_dir=getattr( - prov, "provenance_dir", - "")) + properties=prov.properties) if res.groups: self.nodes.groups.update(res.groups) for svg in res.groups.values(): @@ -1083,8 +1080,8 @@ def _run_all_data_imports(self): if prop_name not in self.nodes.properties: self.nodes.properties[prop_name] = prop_obj else: - self.nodes.properties[prop_name].provenance_dirs.update( - getattr(prop_obj, "provenance_dirs", set())) + self.nodes.properties[prop_name].provenance_ids.update( + getattr(prop_obj, "provenance_ids", set())) if res.processed_imports: self.db._processed_imports.update(res.processed_imports) if res.obs_collision_count and hasattr(self.db, diff --git a/simple/stats/variable_per_row_importer.py b/simple/stats/variable_per_row_importer.py index 69108819..d01f8602 100644 --- a/simple/stats/variable_per_row_importer.py +++ b/simple/stats/variable_per_row_importer.py @@ -340,14 +340,12 @@ def _add_entity_nodes(self) -> None: if not self.nodes.has_entity(strip_namespace(dcid)) ] - try: - prov_dir = self.config.import_name(self.input_file) - except Exception: - prov_dir = "" - for dcid in self.entity_dcids: - clean_dcid = strip_namespace(dcid) - if self.nodes.has_entity(clean_dcid) and prov_dir: - self.nodes.entities[clean_dcid].provenance_dirs.add(prov_dir) + prov_id = getattr(self, "provenance", "") + if prov_id: + for dcid in self.entity_dcids: + clean_dcid = strip_namespace(dcid) + if self.nodes.has_entity(clean_dcid): + self.nodes.entities[clean_dcid].provenance_ids.add(prov_id) logging.info("Found %s total entities, of which %s are already imported.", len(self.entity_dcids), @@ -366,4 +364,4 @@ def _add_entity_nodes(self) -> None: if dcid2type: logging.info("Importing %s of %s entities.", len(dcid2type), len(new_entity_dcids)) - self.nodes.entities_with_types(dcid2type, provenance_dir=prov_dir) + self.nodes.entities_with_types(dcid2type, provenance_id=prov_id) From dc7587dd9c05a566ed1cafb8cc504adf9cb801d3 Mon Sep 17 00:00:00 2001 From: Gabriel Mechali Date: Mon, 27 Jul 2026 13:06:49 -0400 Subject: [PATCH 42/45] Lint --- simple/stats/events_importer.py | 4 +--- simple/stats/nodes.py | 3 ++- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/simple/stats/events_importer.py b/simple/stats/events_importer.py index ccb19cd7..b72a6eb4 100644 --- a/simple/stats/events_importer.py +++ b/simple/stats/events_importer.py @@ -255,9 +255,7 @@ def remove_pre_resolved(entity: str) -> bool: if prov_id: self.nodes.entities[clean_dcid].provenance_ids.add(prov_id) else: - self.nodes.entity_with_type( - clean_dcid, "Thing", provenance_id=prov_id - ) + self.nodes.entity_with_type(clean_dcid, "Thing", provenance_id=prov_id) logging.info("Found %s entities pre-resolved.", len(pre_resolved_entities)) diff --git a/simple/stats/nodes.py b/simple/stats/nodes.py index 79573655..47b45390 100644 --- a/simple/stats/nodes.py +++ b/simple/stats/nodes.py @@ -458,7 +458,8 @@ def triples_by_provenance_dir(self) -> dict[str, list[Triple]]: for provenance in self.provenances.values(): if self.has_custom_mcf_nodes and provenance.id == _DEFAULT_PROVENANCE.id and _DEFAULT_PROVENANCE.id not in self._used_provenance_ids: continue - raw = getattr(provenance, "provenance_id", "") or provenance.id or "_global" + raw = getattr(provenance, "provenance_id", + "") or provenance.id or "_global" imp = strip_namespace(raw) if raw != "_global" else "_global" result[imp].extend(provenance.triples()) for group in self.groups.values(): From 9e1b161f920a0a05ac722e8a3ec07363c7c58591 Mon Sep 17 00:00:00 2001 From: Gabriel Mechali Date: Mon, 27 Jul 2026 14:09:04 -0400 Subject: [PATCH 43/45] Allow events to register the referred entiies --- simple/stats/events_importer.py | 22 +++++++----- .../tests/stats/observations_importer_test.py | 35 +++++++++++-------- .../countryalpha3codes.triples.db.csv | 3 ++ .../expected/idcolumns.triples.db.csv | 3 ++ 4 files changed, 40 insertions(+), 23 deletions(-) diff --git a/simple/stats/events_importer.py b/simple/stats/events_importer.py index b72a6eb4..f36a3ebb 100644 --- a/simple/stats/events_importer.py +++ b/simple/stats/events_importer.py @@ -248,15 +248,6 @@ def remove_pre_resolved(entity: str) -> bool: entities = list(filter(remove_pre_resolved, column.tolist())) - prov_id = getattr(self, "provenance", "") - for dcid in column.tolist(): - clean_dcid = strip_namespace(dcid) - if self.nodes.has_entity(clean_dcid): - if prov_id: - self.nodes.entities[clean_dcid].provenance_ids.add(prov_id) - else: - self.nodes.entity_with_type(clean_dcid, "Thing", provenance_id=prov_id) - logging.info("Found %s entities pre-resolved.", len(pre_resolved_entities)) logging.info("Resolving %s entities of type %s.", len(entities), @@ -289,6 +280,19 @@ def remove_pre_resolved(entity: str) -> bool: unresolved=unresolved_list, ) + prov_id = getattr(self, "provenance", "") + for dcid in df[constants.COLUMN_DCID].dropna().unique(): + clean_dcid = strip_namespace(dcid) + if self.nodes.has_entity(clean_dcid): + if prov_id: + self.nodes.entities[clean_dcid].provenance_ids.add(prov_id) + elif prov_id: + self.nodes.entity_with_type( + clean_dcid, + self.entity_type or "Thing", + provenance_id=prov_id, + ) + def _resolve(self, entities: list[str]) -> dict[str, str]: lower_case_entity_name = self.entity_column_name.lower() diff --git a/simple/tests/stats/observations_importer_test.py b/simple/tests/stats/observations_importer_test.py index 1a7934ab..4f9debb1 100644 --- a/simple/tests/stats/observations_importer_test.py +++ b/simple/tests/stats/observations_importer_test.py @@ -17,6 +17,7 @@ import shutil import tempfile import unittest +from unittest import mock from unittest.mock import MagicMock import pandas as pd @@ -126,17 +127,23 @@ def test_custom_namespace_pre_resolution(self): importer.entity_type = "State" importer.entity_column_name = "entity" importer.df = pd.DataFrame( - {"dcid": ["undata:place/custom_1", "dcid:geoId/06", "California"]}) - - dc_client.resolve_entities = MagicMock( - return_value={"California": "geoId/06"}) - dc_client.get_property_of_entities = MagicMock(return_value={}) - - importer._resolve_entities() - - dc_client.resolve_entities.assert_called_once_with( - entities=["California"], - entity_type="State", - property_name="description") - self.assertEqual(importer.df["dcid"].tolist(), - ["place/custom_1", "geoId/06", "geoId/06"]) + {"dcid": ["undata:place/custom_1", "dcid:geoId/06", "California"]} + ) + with mock.patch.object( + dc_client, + "resolve_entities", + return_value={"California": "geoId/06"}, + ) as mock_resolve, mock.patch.object( + dc_client, "get_property_of_entities", return_value={} + ): + importer._resolve_entities() + + mock_resolve.assert_called_once_with( + entities=["California"], + entity_type="State", + property_name="description", + ) + self.assertEqual( + importer.df["dcid"].tolist(), + ["place/custom_1", "geoId/06", "geoId/06"], + ) diff --git a/simple/tests/stats/test_data/events_importer/expected/countryalpha3codes.triples.db.csv b/simple/tests/stats/test_data/events_importer/expected/countryalpha3codes.triples.db.csv index 3b1640a5..9692c7c2 100644 --- a/simple/tests/stats/test_data/events_importer/expected/countryalpha3codes.triples.db.csv +++ b/simple/tests/stats/test_data/events_importer/expected/countryalpha3codes.triples.db.csv @@ -94,3 +94,6 @@ PRIMARY_DESCRIPTION,typeOf,Property, PRIMARY_DESCRIPTION,name,,PRIMARY DESCRIPTION SECONDARY_DESCRIPTION,typeOf,Property, SECONDARY_DESCRIPTION,name,,SECONDARY DESCRIPTION +country/USA,typeOf,Country, +country/BRA,typeOf,Country, +country/CHN,typeOf,Country, diff --git a/simple/tests/stats/test_data/events_importer/expected/idcolumns.triples.db.csv b/simple/tests/stats/test_data/events_importer/expected/idcolumns.triples.db.csv index bd28712a..e87b0bcb 100644 --- a/simple/tests/stats/test_data/events_importer/expected/idcolumns.triples.db.csv +++ b/simple/tests/stats/test_data/events_importer/expected/idcolumns.triples.db.csv @@ -93,3 +93,6 @@ PRIMARY_DESCRIPTION,typeOf,Property, PRIMARY_DESCRIPTION,name,,PRIMARY DESCRIPTION SECONDARY_DESCRIPTION,typeOf,Property, SECONDARY_DESCRIPTION,name,,SECONDARY DESCRIPTION +country/USA,typeOf,Country, +country/BRA,typeOf,Country, +country/CHN,typeOf,Country, From 7db606274d4572a6033fe7fab518e7a9d35a4164 Mon Sep 17 00:00:00 2001 From: Gabriel Mechali Date: Mon, 27 Jul 2026 22:15:53 -0400 Subject: [PATCH 44/45] Christie comment fixes --- simple/stats/nodes.py | 61 +++++++++++++------------------- simple/stats/validation.py | 2 -- simple/tests/stats/nodes_test.py | 11 ++++++ simple/util/filesystem.py | 7 +--- 4 files changed, 37 insertions(+), 44 deletions(-) diff --git a/simple/stats/nodes.py b/simple/stats/nodes.py index 47b45390..b376d69c 100644 --- a/simple/stats/nodes.py +++ b/simple/stats/nodes.py @@ -464,42 +464,31 @@ def triples_by_provenance_dir(self) -> dict[str, list[Triple]]: result[imp].extend(provenance.triples()) for group in self.groups.values(): result["_global"].extend(group.triples()) - for variable in self.variables.values(): - p_ids = getattr(variable, "provenance_ids", []) - if not p_ids: - result["_global"].extend(variable.triples()) - else: - for p_id in p_ids: - result[strip_namespace(p_id)].extend(variable.triples()) - for event_type in self.event_types.values(): - p_ids = getattr(event_type, "provenance_ids", []) - if not p_ids: - result["_global"].extend(event_type.triples()) - else: - for p_id in p_ids: - result[strip_namespace(p_id)].extend(event_type.triples()) - for entity_type in self.entity_types.values(): - p_ids = getattr(entity_type, "provenance_ids", []) - if not p_ids: - result["_global"].extend(entity_type.triples()) - else: - for p_id in p_ids: - result[strip_namespace(p_id)].extend(entity_type.triples()) - for property in self.properties.values(): - p_ids = getattr(property, "provenance_ids", set()) - if not p_ids: - result["_global"].extend(property.triples()) - else: - for p_id in p_ids: - result[strip_namespace(p_id)].extend(property.triples()) - for entity in self.entities.values(): - p_ids = getattr(entity, "provenance_ids", set()) - if not p_ids: - result["_global"].extend(entity.triples()) - else: - for p_id in p_ids: - result[strip_namespace(p_id)].extend(entity.triples()) - return result + for collection in ( + self.variables, + self.event_types, + self.entity_types, + self.properties, + self.entities, + ): + for node in collection.values(): + p_ids = getattr(node, "provenance_ids", None) + target_dirs = ( + {strip_namespace(p) for p in p_ids} if p_ids else {"_global"} + ) + for dir_name in target_dirs: + result[dir_name].extend(node.triples()) + deduped_result = {} + for k, triples in result.items(): + seen = set() + deduped = [] + for t in triples: + key = (t.subject_id, t.predicate, t.object_id, t.object_value) + if key not in seen: + seen.add(key) + deduped.append(t) + deduped_result[k] = deduped + return deduped_result def _clean_metadata_id(id: str) -> str: diff --git a/simple/stats/validation.py b/simple/stats/validation.py index 68186fb7..61a35a26 100644 --- a/simple/stats/validation.py +++ b/simple/stats/validation.py @@ -156,8 +156,6 @@ def _clean_dcid(self, val: str) -> str: """Normalizes a DCID value by ensuring it starts with a namespace prefix.""" if not val: return "" - if val.startswith(("http://", "https://")): - return val if has_namespace_prefix(val): return val return f"dcid:{val}" diff --git a/simple/tests/stats/nodes_test.py b/simple/tests/stats/nodes_test.py index 88fcba10..01325880 100644 --- a/simple/tests/stats/nodes_test.py +++ b/simple/tests/stats/nodes_test.py @@ -407,3 +407,14 @@ def test_default_custom_root_group_name_override(self): ] self.assertTrue(default_groups, "Default custom root SVG should exist") self.assertEqual(default_groups[0].name, "ONE Data") + + def test_triples_by_provenance_dir_and_deduplication(self): + nodes = Nodes(Config(CONFIG_DATA)) + var = nodes.variable("Variable 1", self.a) + # Add duplicate provenance IDs to verify deduplication + var.provenance_ids = ["prov/1", "prov/1"] + shards = nodes.triples_by_provenance_dir() + self.assertIn("prov/1", shards) + triples = shards["prov/1"] + db_tuples = [t.db_tuple() for t in triples] + self.assertEqual(len(db_tuples), len(set(db_tuples))) diff --git a/simple/util/filesystem.py b/simple/util/filesystem.py index 0b6a721c..27db18ab 100644 --- a/simple/util/filesystem.py +++ b/simple/util/filesystem.py @@ -201,13 +201,8 @@ def __init__(self, store: "Store", path: str, create_if_missing: bool): super().__init__(store, path) if not self.fs().exists(self.path): if create_if_missing: - # Make parent dir if needed. parent_dir_path = fspath.dirname(path) - if not self.fs().isdir(parent_dir_path): - try: - self.fs().makedirs(parent_dir_path) - except Exception: - pass + self.fs().makedirs(parent_dir_path, recreate=True) # Make empty file. self.fs().touch(path) else: From 91b33bad86f077aca27c0ffe42851f82a505ce23 Mon Sep 17 00:00:00 2001 From: Gabriel Mechali Date: Mon, 27 Jul 2026 22:17:55 -0400 Subject: [PATCH 45/45] lint --- simple/stats/nodes.py | 5 ++--- simple/tests/stats/observations_importer_test.py | 9 ++++----- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/simple/stats/nodes.py b/simple/stats/nodes.py index b376d69c..0daceb9a 100644 --- a/simple/stats/nodes.py +++ b/simple/stats/nodes.py @@ -473,9 +473,8 @@ def triples_by_provenance_dir(self) -> dict[str, list[Triple]]: ): for node in collection.values(): p_ids = getattr(node, "provenance_ids", None) - target_dirs = ( - {strip_namespace(p) for p in p_ids} if p_ids else {"_global"} - ) + target_dirs = ({strip_namespace(p) for p in p_ids} + if p_ids else {"_global"}) for dir_name in target_dirs: result[dir_name].extend(node.triples()) deduped_result = {} diff --git a/simple/tests/stats/observations_importer_test.py b/simple/tests/stats/observations_importer_test.py index 4f9debb1..54816893 100644 --- a/simple/tests/stats/observations_importer_test.py +++ b/simple/tests/stats/observations_importer_test.py @@ -127,15 +127,14 @@ def test_custom_namespace_pre_resolution(self): importer.entity_type = "State" importer.entity_column_name = "entity" importer.df = pd.DataFrame( - {"dcid": ["undata:place/custom_1", "dcid:geoId/06", "California"]} - ) + {"dcid": ["undata:place/custom_1", "dcid:geoId/06", "California"]}) with mock.patch.object( dc_client, "resolve_entities", return_value={"California": "geoId/06"}, - ) as mock_resolve, mock.patch.object( - dc_client, "get_property_of_entities", return_value={} - ): + ) as mock_resolve, mock.patch.object(dc_client, + "get_property_of_entities", + return_value={}): importer._resolve_entities() mock_resolve.assert_called_once_with(