easylink 0.1.17__py3-none-any.whl → 0.1.19__py3-none-any.whl

easylink/steps/cascading/exclude_clustered.py

@@ -0,0 +1,76 @@
+# STEP_NAME: determining_exclusions
+
+# REQUIREMENTS: pandas==2.1.2 pyarrow pyyaml
+
+# PIPELINE_SCHEMA: main
+
+import logging
+import os
+from pathlib import Path
+
+import pandas as pd
+
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s %(message)s",
+    handlers=[logging.StreamHandler()],
+)
+
+
+def load_file(file_path, file_format=None):
+    logging.info(f"Loading file {file_path} with format {file_format}")
+    if file_format is None:
+        file_format = file_path.split(".")[-1]
+    if file_format == "parquet":
+        return pd.read_parquet(file_path)
+    raise ValueError(f"Unknown file format {file_format}")
+
+
+# LOAD INPUTS
+
+# INPUT_DATASETS_AND_INPUT_KNOWN_CLUSTERS_FILE_PATHS is list of filepaths which includes
+# the known_clusters filepath due to workaround
+dataset_paths = os.environ["INPUT_DATASETS_AND_INPUT_KNOWN_CLUSTERS_FILE_PATHS"].split(",")
+dataset_paths = [path for path in dataset_paths if "clusters" not in Path(path).stem]
+
+# for workaround, choose path based on INPUT_DATASET configuration
+splitter_choice = os.environ["INPUT_DATASET"]
+dataset_path = None
+for path in dataset_paths:
+    if splitter_choice == Path(path).stem:
+        dataset_path = path
+        break
+if dataset_path is None:
+    raise ValueError(f"No dataset matching {splitter_choice} found")
+
+# KNOWN_CLUSTERS_AND_MAYBE_INPUT_DATASETS_FILE_PATHS is a list of file paths.
+# There is one item in it that is a file with "clusters" in the filename.
+# That's the only item we're interested in here.
+# The other items may be there if this is coming from the user's input,
+# due to our workaround for only having one slot of user input.
+clusters_filepaths = [
+    path
+    for path in os.environ["KNOWN_CLUSTERS_AND_MAYBE_INPUT_DATASETS_FILE_PATHS"].split(",")
+    if "clusters" in Path(path).stem
+]
+if len(clusters_filepaths) > 1:
+    raise ValueError("Multiple known clusters files found")
+if len(clusters_filepaths) == 0:
+    raise ValueError("No known clusters file found")
+
+clusters_filepath = clusters_filepaths[0]
+
+# Exclude records that have been clustered
+clusters_df = load_file(clusters_filepath)
+dataset_df = load_file(dataset_path)
+clustered_record_ids = set(dataset_df["Record ID"].unique()) & set(
+    clusters_df["Input Record ID"].unique()
+)
+
+IDS_TO_REMOVE = pd.DataFrame({"Record ID": list(clustered_record_ids)})
+
+# DUMMY_CONTAINER_OUTPUT_PATHS is a single path to a file (results.parquet)
+results_filepath = os.environ["DUMMY_CONTAINER_OUTPUT_PATHS"]
+
+logging.info(f"Writing output for dataset from input {dataset_path} to {results_filepath}")
+IDS_TO_REMOVE.to_parquet(results_filepath)

easylink/steps/cascading/exclude_none.def

@@ -0,0 +1,22 @@
+
+Bootstrap: docker
+From: python@sha256:1c26c25390307b64e8ff73e7edf34b4fbeac59d41da41c08da28dc316a721899
+
+%files
+    ./exclude_none.py /exclude_none.py
+
+%post
+    # Create directories
+    mkdir -p /input_data
+    mkdir -p /extra_implementation_specific_input_data
+    mkdir -p /results
+    mkdir -p /diagnostics
+
+    # Install Python packages with specific versions
+    pip install pandas==2.1.2 pyarrow pyyaml
+
+%environment
+    export LC_ALL=C
+
+%runscript
+    python /exclude_none.py '$@'

easylink/steps/cascading/exclude_none.py

@@ -0,0 +1,76 @@
+# STEP_NAME: determining_exclusions
+
+# REQUIREMENTS: pandas==2.1.2 pyarrow pyyaml
+
+# PIPELINE_SCHEMA: main
+
+import logging
+import os
+from pathlib import Path
+
+import pandas as pd
+
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s %(message)s",
+    handlers=[logging.StreamHandler()],
+)
+
+
+def load_file(file_path, file_format=None):
+    logging.info(f"Loading file {file_path} with format {file_format}")
+    if file_format is None:
+        file_format = file_path.split(".")[-1]
+    if file_format == "parquet":
+        return pd.read_parquet(file_path)
+    raise ValueError(f"Unknown file format {file_format}")
+
+
+# LOAD INPUTS
+
+# INPUT_DATASETS_AND_INPUT_KNOWN_CLUSTERS_FILE_PATHS is list of filepaths which includes
+# the known_clusters filepath due to workaround
+dataset_paths = os.environ["INPUT_DATASETS_AND_INPUT_KNOWN_CLUSTERS_FILE_PATHS"].split(",")
+dataset_paths = [path for path in dataset_paths if "clusters" not in Path(path).stem]
+
+# for workaround, choose path based on INPUT_DATASET configuration
+splitter_choice = os.environ["INPUT_DATASET"]
+dataset_path = None
+for path in dataset_paths:
+    if splitter_choice == Path(path).stem:
+        dataset_path = path
+        break
+if dataset_path is None:
+    raise ValueError(f"No dataset matching {splitter_choice} found")
+
+# KNOWN_CLUSTERS_AND_MAYBE_INPUT_DATASETS_FILE_PATHS is a list of file paths.
+# There is one item in it that is a file with "clusters" in the filename.
+# That's the only item we're interested in here.
+# The other items may be there if this is coming from the user's input,
+# due to our workaround for only having one slot of user input.
+clusters_filepaths = [
+    path
+    for path in os.environ["KNOWN_CLUSTERS_AND_MAYBE_INPUT_DATASETS_FILE_PATHS"].split(",")
+    if "clusters" in Path(path).stem
+]
+if len(clusters_filepaths) > 1:
+    raise ValueError("Multiple known clusters files found")
+if len(clusters_filepaths) == 0:
+    raise ValueError("No known clusters file found")
+
+clusters_filepath = clusters_filepaths[0]
+
+clusters_df = load_file(clusters_filepath)
+
+# don't need to actually load the dataset,
+# since we will just save an empty ids_to_remove
+
+# SAVE OUTPUTS
+
+IDS_TO_REMOVE = pd.DataFrame(columns=["Record ID"])
+
+# DUMMY_CONTAINER_OUTPUT_PATHS is a single path to a file (results.parquet)
+results_filepath = os.environ["DUMMY_CONTAINER_OUTPUT_PATHS"]
+
+logging.info(f"Writing output for dataset from input {dataset_path} to {results_filepath}")
+IDS_TO_REMOVE.to_parquet(results_filepath)

easylink/steps/cascading/update_clusters_by_connected_components.def

@@ -0,0 +1,22 @@
+
+Bootstrap: docker
+From: python@sha256:1c26c25390307b64e8ff73e7edf34b4fbeac59d41da41c08da28dc316a721899
+
+%files
+    ./update_clusters_by_connected_components.py /update_clusters_by_connected_components.py
+
+%post
+    # Create directories
+    mkdir -p /input_data
+    mkdir -p /extra_implementation_specific_input_data
+    mkdir -p /results
+    mkdir -p /diagnostics
+
+    # Install Python packages with specific versions
+    pip install pandas==2.1.2 pyarrow pyyaml networkx
+
+%environment
+    export LC_ALL=C
+
+%runscript
+    python /update_clusters_by_connected_components.py '$@'

easylink/steps/cascading/update_clusters_by_connected_components.py

@@ -0,0 +1,101 @@
+# STEP_NAME: updating_clusters
+
+# REQUIREMENTS: pandas==2.1.2 pyarrow pyyaml networkx
+
+# PIPELINE_SCHEMA: main
+
+import logging
+import os
+from pathlib import Path
+
+import networkx as nx
+import pandas as pd
+
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s %(message)s",
+    handlers=[logging.StreamHandler()],
+)
+
+
+def load_file(file_path, file_format=None):
+    logging.info(f"Loading file {file_path} with format {file_format}")
+    if file_format is None:
+        file_format = file_path.split(".")[-1]
+    if file_format == "parquet":
+        return pd.read_parquet(file_path)
+    raise ValueError(f"Unknown file format {file_format}")
+
+
+# LOAD INPUTS and SAVE OUTPUTS
+
+# NEW_CLUSTERS_FILE_PATH is a path to a single file
+new_clusters_filepath = os.environ["NEW_CLUSTERS_FILE_PATH"]
+
+# KNOWN_CLUSTERS_AND_MAYBE_INPUT_DATASETS_FILE_PATHS is a list of file paths.
+# There is one item in it that is a file with "clusters" in the filename.
+# That's the only item we're interested in here.
+# The other items may be there if this is coming from the user's input,
+# due to our workaround for only having one slot of user input.
+known_clusters_filepaths = [
+    path
+    for path in os.environ["KNOWN_CLUSTERS_AND_MAYBE_INPUT_DATASETS_FILE_PATHS"].split(",")
+    if "clusters" in Path(path).stem
+]
+if len(known_clusters_filepaths) > 1:
+    raise ValueError("Multiple known clusters files found")
+if len(known_clusters_filepaths) == 0:
+    raise ValueError("No known clusters file found")
+
+known_clusters_filepath = known_clusters_filepaths[0]
+known_clusters_df = load_file(known_clusters_filepath)
+
+# DUMMY_CONTAINER_OUTPUT_PATHS is a path to a single file (clusters.parquet)
+results_filepath = os.environ["DUMMY_CONTAINER_OUTPUT_PATHS"]
+Path(results_filepath).parent.mkdir(exist_ok=True, parents=True)
+
+new_clusters_df = load_file(new_clusters_filepath)
+
+
+def merge_clusters(known_clusters_df, new_clusters_df):
+    # Combine both dataframes
+    combined_df = pd.concat([known_clusters_df, new_clusters_df], ignore_index=True)
+
+    # Drop records with missing cluster IDs
+    combined_df = combined_df.dropna(subset=["Cluster ID"])
+
+    # Group by Cluster ID to get connected records
+    cluster_groups = combined_df.groupby("Cluster ID")["Input Record ID"].apply(list)
+
+    # Build a graph of all connections implied by cluster IDs
+    G = nx.Graph()
+    for group in cluster_groups:
+        for i in range(len(group)):
+            for j in range(i + 1, len(group)):
+                G.add_edge(group[i], group[j])
+
+    # Add isolated nodes (records with unique clusters)
+    all_ids = set(combined_df["Input Record ID"])
+    G.add_nodes_from(all_ids)
+
+    # Compute connected components
+    components = list(nx.connected_components(G))
+
+    # Assign new cluster IDs
+    merged_data = []
+    for cluster_id, records in enumerate(components, start=1):
+        for record_id in records:
+            merged_data.append((record_id, cluster_id))
+
+    # Build the final DataFrame
+    merged_df = pd.DataFrame(merged_data, columns=["Input Record ID", "Cluster ID"])
+
+    return merged_df
+
+
+output_df = merge_clusters(known_clusters_df, new_clusters_df)
+
+logging.info(
+    f"Writing output for dataset from input {new_clusters_filepath} to {results_filepath}"
+)
+output_df.to_parquet(results_filepath)

easylink/steps/default/default_clusters_to_links.def

@@ -0,0 +1,22 @@
+
+Bootstrap: docker
+From: python@sha256:1c26c25390307b64e8ff73e7edf34b4fbeac59d41da41c08da28dc316a721899
+
+%files
+    ./default_clusters_to_links.py /default_clusters_to_links.py
+
+%post
+    # Create directories
+    mkdir -p /input_data
+    mkdir -p /extra_implementation_specific_input_data
+    mkdir -p /results
+    mkdir -p /diagnostics
+
+    # Install Python packages with specific versions
+    pip install pandas==2.1.2 pyarrow pyyaml
+
+%environment
+    export LC_ALL=C
+
+%runscript
+    python /default_clusters_to_links.py '$@'

easylink/steps/default/default_clusters_to_links.py

@@ -0,0 +1,91 @@
+# STEP_NAME: clusters_to_links
+
+# REQUIREMENTS: pandas==2.1.2 pyarrow pyyaml
+
+# PIPELINE_SCHEMA: main
+
+import logging
+import os
+from pathlib import Path
+
+import pandas as pd
+
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s %(message)s",
+    handlers=[logging.StreamHandler()],
+)
+
+
+def load_file(file_path, file_format=None):
+    logging.info(f"Loading file {file_path} with format {file_format}")
+    if file_format is None:
+        file_format = file_path.split(".")[-1]
+    if file_format == "parquet":
+        return pd.read_parquet(file_path)
+    raise ValueError(f"Unknown file format {file_format}")
+
+
+# code example from pipeline schema docs
+def clusters_to_links(clusters_df):
+    # Merge the dataframe with itself on Cluster ID to get all pairs within each cluster
+    merged = clusters_df.merge(
+        clusters_df,
+        on="Cluster ID",
+        suffixes=("_left", "_right"),
+    )
+
+    # Compare tuples row-wise to keep only unique pairs (left < right)
+    mask = (merged["Input Record Dataset_left"] < merged["Input Record Dataset_right"]) | (
+        (merged["Input Record Dataset_left"] == merged["Input Record Dataset_right"])
+        & (merged["Input Record ID_left"] < merged["Input Record ID_right"])
+    )
+    filtered = merged[mask]
+
+    # Build the output DataFrame
+    links_df = filtered[
+        [
+            "Input Record Dataset_left",
+            "Input Record ID_left",
+            "Input Record Dataset_right",
+            "Input Record ID_right",
+        ]
+    ].copy()
+    links_df.columns = [
+        "Left Record Dataset",
+        "Left Record ID",
+        "Right Record Dataset",
+        "Right Record ID",
+    ]
+    links_df["Probability"] = 1.0
+    return links_df
+
+
+# LOAD INPUTS and SAVE OUTPUTS
+
+# KNOWN_CLUSTERS_AND_MAYBE_INPUT_DATASETS_FILE_PATHS is a list of file paths.
+# There is one item in it that is a file with "clusters" in the filename.
+# That's the only item we're interested in here.
+# The other items may be there if this is coming from the user's input,
+# due to our workaround for only having one slot of user input.
+clusters_filepaths = [
+    path
+    for path in os.environ["KNOWN_CLUSTERS_AND_MAYBE_INPUT_DATASETS_FILE_PATHS"].split(",")
+    if "clusters" in Path(path).stem
+]
+if len(clusters_filepaths) > 1:
+    raise ValueError("Multiple known clusters files found")
+if len(clusters_filepaths) == 0:
+    raise ValueError("No known clusters file found")
+
+clusters_filepath = clusters_filepaths[0]
+
+# DUMMY_CONTAINER_OUTPUT_PATHS is a path to a single file (results.parquet)
+results_filepath = os.environ["DUMMY_CONTAINER_OUTPUT_PATHS"]
+
+clusters_df = load_file(clusters_filepath)
+links_df = clusters_to_links(clusters_df)
+logging.info(
+    f"Writing output for dataset from input {clusters_filepath} to {results_filepath}"
+)
+links_df.to_parquet(results_filepath)

easylink/steps/default/default_determining_exclusions.def

@@ -0,0 +1,22 @@
+
+Bootstrap: docker
+From: python@sha256:1c26c25390307b64e8ff73e7edf34b4fbeac59d41da41c08da28dc316a721899
+
+%files
+    ./default_determining_exclusions.py /default_determining_exclusions.py
+
+%post
+    # Create directories
+    mkdir -p /input_data
+    mkdir -p /extra_implementation_specific_input_data
+    mkdir -p /results
+    mkdir -p /diagnostics
+
+    # Install Python packages with specific versions
+    pip install pandas==2.1.2 pyarrow pyyaml
+
+%environment
+    export LC_ALL=C
+
+%runscript
+    python /default_determining_exclusions.py '$@'

easylink/steps/default/default_determining_exclusions.py

@@ -0,0 +1,81 @@
+# STEP_NAME: determining_exclusions
+
+# REQUIREMENTS: pandas==2.1.2 pyarrow pyyaml
+
+# PIPELINE_SCHEMA: main
+
+import logging
+import os
+from pathlib import Path
+
+import pandas as pd
+
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s %(message)s",
+    handlers=[logging.StreamHandler()],
+)
+
+
+def load_file(file_path, file_format=None):
+    logging.info(f"Loading file {file_path} with format {file_format}")
+    if file_format is None:
+        file_format = file_path.split(".")[-1]
+    if file_format == "parquet":
+        return pd.read_parquet(file_path)
+    raise ValueError(f"Unknown file format {file_format}")
+
+
+# LOAD INPUTS
+
+# INPUT_DATASETS_AND_INPUT_KNOWN_CLUSTERS_FILE_PATHS is list of filepaths which includes
+# the known_clusters filepath due to workaround
+dataset_paths = os.environ["INPUT_DATASETS_AND_INPUT_KNOWN_CLUSTERS_FILE_PATHS"].split(",")
+dataset_paths = [path for path in dataset_paths if "clusters" not in Path(path).stem]
+
+# for workaround, choose path based on INPUT_DATASET configuration
+splitter_choice = os.environ["INPUT_DATASET"]
+dataset_path = None
+for path in dataset_paths:
+    if splitter_choice == Path(path).stem:
+        dataset_path = path
+        break
+if dataset_path is None:
+    raise ValueError(f"No dataset matching {splitter_choice} found")
+
+# KNOWN_CLUSTERS_AND_MAYBE_INPUT_DATASETS_FILE_PATHS is a list of file paths.
+# There is one item in it that is a file with "clusters" in the filename.
+# That's the only item we're interested in here.
+# The other items may be there if this is coming from the user's input,
+# due to our workaround for only having one slot of user input.
+clusters_filepaths = [
+    path
+    for path in os.environ["KNOWN_CLUSTERS_AND_MAYBE_INPUT_DATASETS_FILE_PATHS"].split(",")
+    if "clusters" in Path(path).stem
+]
+if len(clusters_filepaths) > 1:
+    raise ValueError("Multiple known clusters files found")
+if len(clusters_filepaths) == 0:
+    raise ValueError("No known clusters file found")
+
+clusters_filepath = clusters_filepaths[0]
+
+clusters_df = load_file(clusters_filepath)
+
+if len(clusters_df) > 0:
+    raise ValueError(
+        "Default implementation of determining_exclusions passed a non-empty set of known clusters"
+    )
+
+# don't need to actually load the dataset,
+# since we will just save an empty ids_to_remove
+
+# SAVE OUTPUTS
+
+IDS_TO_REMOVE = pd.DataFrame(columns=["Record ID"])
+
+# DUMMY_CONTAINER_OUTPUT_PATHS is a single path to a file (results.parquet)
+results_filepath = os.environ["DUMMY_CONTAINER_OUTPUT_PATHS"]
+
+logging.info(f"Writing output for dataset from input {dataset_path} to {results_filepath}")
+IDS_TO_REMOVE.to_parquet(results_filepath)

easylink/steps/default/default_removing_records.def

@@ -0,0 +1,22 @@
+
+Bootstrap: docker
+From: python@sha256:1c26c25390307b64e8ff73e7edf34b4fbeac59d41da41c08da28dc316a721899
+
+%files
+    ./default_removing_records.py /default_removing_records.py
+
+%post
+    # Create directories
+    mkdir -p /input_data
+    mkdir -p /extra_implementation_specific_input_data
+    mkdir -p /results
+    mkdir -p /diagnostics
+
+    # Install Python packages with specific versions
+    pip install pandas==2.1.2 pyarrow pyyaml
+
+%environment
+    export LC_ALL=C
+
+%runscript
+    python /default_removing_records.py '$@'

easylink/steps/default/default_removing_records.py

@@ -0,0 +1,59 @@
+# STEP_NAME: removing_records
+
+# REQUIREMENTS: pandas==2.1.2 pyarrow pyyaml
+
+# PIPELINE_SCHEMA: main
+
+import logging
+import os
+from pathlib import Path
+
+import pandas as pd
+
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s %(message)s",
+    handlers=[logging.StreamHandler()],
+)
+
+
+def load_file(file_path, file_format=None):
+    logging.info(f"Loading file {file_path} with format {file_format}")
+    if file_format is None:
+        file_format = file_path.split(".")[-1]
+    if file_format == "parquet":
+        return pd.read_parquet(file_path)
+    raise ValueError(f"Unknown file format {file_format}")
+
+
+# LOAD INPUTS and SAVE OUTPUTS
+
+# INPUT_DATASETS_AND_INPUT_KNOWN_CLUSTERS_FILE_PATHS is list of filepaths which includes
+# the known_clusters filepath due to workaround
+dataset_paths = os.environ["INPUT_DATASETS_AND_INPUT_KNOWN_CLUSTERS_FILE_PATHS"].split(",")
+dataset_paths = [path for path in dataset_paths if "clusters.parquet" not in Path(path).stem]
+
+# for workaround, choose path based on INPUT_DATASET configuration
+splitter_choice = os.environ["INPUT_DATASET"]
+dataset_path = None
+for path in dataset_paths:
+    if splitter_choice == Path(path).stem:
+        dataset_path = path
+        break
+if dataset_path is None:
+    raise ValueError(f"No dataset matching {splitter_choice} found")
+
+# IDS_TO_REMOVE_FILE_PATH is a single filepath (Cloneable section)
+ids_filepath = os.environ["IDS_TO_REMOVE_FILE_PATH"]
+# DUMMY_CONTAINER_OUTPUT_PATHS is a single path to a directory ('dataset')
+results_dir = Path(os.environ["DUMMY_CONTAINER_OUTPUT_PATHS"])
+results_dir.mkdir(exist_ok=True, parents=True)
+
+dataset = load_file(dataset_path)
+ids_to_remove = load_file(ids_filepath)
+
+dataset = dataset[~dataset["Record ID"].isin(ids_to_remove)]
+
+output_path = results_dir / Path(dataset_path).name
+logging.info(f"Writing output for dataset from input {dataset_path} to {output_path}")
+dataset.to_parquet(output_path)

easylink/steps/default/default_schema_alignment.def

@@ -0,0 +1,22 @@
+
+Bootstrap: docker
+From: python@sha256:1c26c25390307b64e8ff73e7edf34b4fbeac59d41da41c08da28dc316a721899
+
+%files
+    ./default_schema_alignment.py /default_schema_alignment.py
+
+%post
+    # Create directories
+    mkdir -p /input_data
+    mkdir -p /extra_implementation_specific_input_data
+    mkdir -p /results
+    mkdir -p /diagnostics
+
+    # Install Python packages with specific versions
+    pip install pandas==2.1.2 pyarrow pyyaml
+
+%environment
+    export LC_ALL=C
+
+%runscript
+    python /default_schema_alignment.py '$@'

easylink/steps/default/default_schema_alignment.py

@@ -0,0 +1,53 @@
+# STEP_NAME: schema_alignment
+# REQUIREMENTS: pandas==2.1.2 pyarrow pyyaml
+
+import logging
+import os
+from pathlib import Path
+
+import pandas as pd
+
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s %(message)s",
+    handlers=[logging.StreamHandler()],
+)
+
+
+def load_file(file_path, file_format=None):
+    logging.info(f"Loading file {file_path} with format {file_format}")
+    if file_format is None:
+        file_format = file_path.split(".")[-1]
+    if file_format == "parquet":
+        return pd.read_parquet(file_path)
+    raise ValueError(f"Unknown file format {file_format}")
+
+
+# LOAD INPUTS and SAVE OUTPUTS
+
+# DATASETS_DIR_PATHS is list of directories
+dataset_dirs = os.environ["DATASETS_DIR_PATHS"].split(",")
+
+datasets = {}
+
+for dir in dataset_dirs:
+    for root, dirs, files in os.walk(dir):
+        for file in files:
+            if file.startswith("."):
+                continue
+            datasets[Path(file).stem] = load_file(os.path.join(root, file))
+
+records = pd.concat(
+    [df.assign(**{"Input Record Dataset": dataset}) for dataset, df in datasets.items()],
+    ignore_index=True,
+    sort=False,
+)
+
+records = records.rename(columns={"Record ID": "Input Record ID"})
+
+# DUMMY_CONTAINER_OUTPUT_PATHS is a single filepath
+output_path = os.environ["DUMMY_CONTAINER_OUTPUT_PATHS"]
+Path(output_path).parent.mkdir(exist_ok=True, parents=True)
+
+logging.info(f"Writing output to {output_path}")
+records.to_parquet(output_path)