python-jack-knife 0.5.0__py3-none-any.whl

pjk/sinks/factory.py

@@ -0,0 +1,108 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2024 Mike Schultz
+
+from typing import Any, List, Callable
+import os
+from pjk.base import Source, Sink, ParsedToken
+from pjk.common import ComponentFactory
+from pjk.sinks.stdout import StdoutSink
+from pjk.sinks.json_sink import JsonSink
+from pjk.sinks.devnull import DevNullSink
+from pjk.sinks.graph import GraphSink
+from pjk.sinks.csv_sink import CSVSink
+from pjk.sinks.tsv_sink import TSVSink
+from pjk.sinks.ddb import DDBSink
+from pjk.sinks.dir_sink import DirSink
+from pjk.sinks.expect import ExpectSink
+from pjk.sinks.user_sink_factory import UserSinkFactory
+
+COMPONENTS = {
+        '-': StdoutSink,
+        'devnull': DevNullSink,
+        'graph': GraphSink,
+        'ddb': DDBSink,
+        'json': JsonSink,
+        'csv': CSVSink,
+        'tsv': TSVSink,
+        }
+
+class SinkFactory(ComponentFactory):
+    def __init__(self):
+        super().__init__(COMPONENTS, 'sink')   
+
+    def create(self, token: str) -> Callable[[Source], Sink]:
+        token = token.strip()
+        ptok = ParsedToken(token)
+
+        # non-usage sink (bind incompatible)
+        if ptok.pre_colon == 'expect':
+            return ExpectSink(ptok, None)
+
+        if ptok.pre_colon.endswith('.py'):
+            sink = UserSinkFactory.create(ptok)
+            if sink:
+                return sink
+        
+        #if ptok.all_but_params.startswith('s3'):
+        #    return S3Sink.create(ptok, get_format_class_gz=self.get_format_class_gz)
+
+        # check for format sinks
+        sink = self._attempt_format(ptok)
+        if sink:
+            return sink
+
+        sink_cls = self.components.get(ptok.pre_colon)
+        if not sink_cls:
+            return None
+        
+        usage = sink_cls.usage()
+        usage.bind(ptok)
+        return sink_cls(ptok, usage)
+
+        #raise TokenError.from_list(['pjk <source> [<pipe> ...] <sink>',
+        #                            "Expression must end in a sink (e.g. '-', 'out.json')"]
+        #                            )
+        
+    def _attempt_format(self, ptok: ParsedToken):
+        format = ptok.pre_colon
+        is_gz = False
+        if format.endswith('.gz'):
+            format = format[:-3]
+            is_gz = True
+
+        sink_cls = self.components.get(format) # <format>: directory case
+        if not sink_cls:
+            # attempt case -> myfile.<format>
+            return self._attempt_format_file(ptok)
+        
+        # case -> <format>:<path> local dir
+        if sink_cls.is_format: 
+            dir_usage = DirSink.usage()
+            dir_usage.bind(ptok)
+            return DirSink(ptok, dir_usage, sink_cls, is_gz)
+
+        return None
+
+
+    def _attempt_format_file(self, ptok: ParsedToken):
+        is_gz = False
+        path, ext = os.path.splitext(ptok.all_but_params)
+        if '.gz' in ext:
+            is_gz = True
+            path, ext = os.path.splitext(path)
+        
+        file_ext = ext.lstrip('.')  # removes the leading dot
+
+        sink_cls = self.components.get(file_ext)
+        if not sink_cls:
+            return None
+        
+        file_token = f'{path}:{is_gz}' # hack so user can do .json.gz
+        file_ptok = ParsedToken(file_token)
+        
+        usage = sink_cls.usage()
+        usage.bind(file_ptok) # not sure we'll ever use since we're hacking above
+
+        return sink_cls(file_ptok, usage)
+        
+

pjk/sinks/graph.py

@@ -0,0 +1,57 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2024 Mike Schultz
+
+from pjk.base import Sink, ParsedToken, Usage, TokenError
+
+from pjk.sinks.graph_cumulative import graph_cumulative
+from pjk.sinks.graph_hist import graph_hist
+from pjk.sinks.graph_scatter import graph_scatter
+from pjk.sinks.graph_bar_line import graph_bar_line
+
+class GraphSink(Sink):
+    @classmethod
+    def usage(cls):
+        usage = Usage(
+            name='graph',
+            desc='Display various kinds of graphs from streamed records',
+            component_class=cls
+        )
+        usage.def_arg(name='kind', usage='hist|scatter|bar|line|cumulative')
+        usage.def_param(name='x', usage='Name of x-axis field', default='x')
+        usage.def_param(name='y', usage='Name of y-axis field', default='y')
+        usage.def_param(name='pause', usage='Seconds to show graph', is_num=True, default='-1')
+        return usage
+
+    def __init__(self, ptok: ParsedToken, usage: Usage):
+        super().__init__(ptok, usage)
+        self.records = []
+        self.kind = usage.get_arg('kind')
+        self.x_field = usage.get_param('x')
+        self.y_field = usage.get_param('y')
+        self.pause = usage.get_param('pause')
+
+    def process(self):
+        import matplotlib.pyplot as plt # lazy import
+
+        for record in self.input:
+            self.records.append(record)
+
+        if self.kind == "scatter":
+            graph_scatter(self)
+        elif self.kind == "hist":
+            graph_hist(self)
+        elif self.kind == "cumulative":
+            graph_cumulative(self)
+        elif self.kind == "bar":
+            graph_bar_line(self, 'bar')
+        elif self.kind == "line":
+            graph_bar_line(self, 'line')
+        else:
+            raise TokenError(f"Unsupported graph type: {self.kind}")
+        
+        if not self.pause:
+            plt.show()
+        else:
+            plt.show(block=False)   
+            plt.pause(int(self.pause))           
+            plt.close()            

pjk/sinks/graph_bar_line.py

@@ -0,0 +1,229 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2024 Mike Schultz
+
+import re
+from datetime import date, datetime
+from collections import defaultdict
+
+def graph_bar_line(obj, type):
+    import matplotlib.pyplot as plt # lazy imports
+    import matplotlib.dates as mdates
+    import numpy as np
+    import pandas as pd
+    """
+    Plot grouped data as a bar or line chart. Automatically detects time-series X
+    unless explicitly overridden via `obj.x_is_time = True/False`.
+
+    Required fields on `obj`:
+      - obj.records: iterable of dicts
+      - obj.x_field: str
+      - obj.y_field: str
+      - obj.args_dict: optional dict of {matplotlib.pyplot function name -> value}
+      - obj.x_is_time: optional bool to force/disable time-series mode
+
+    `type` should be 'bar' or 'line'.
+    """
+    if not obj.x_field or not obj.y_field:
+        print("Both x_field and y_field must be specified.")
+        return
+
+    # ---------- collect raw rows ----------
+    rows = []  # (x, y, set_name)
+    count = 0
+    for r in obj.records:
+        if obj.x_field in r and obj.y_field in r:
+            x = r[obj.x_field]
+            y = r[obj.y_field]
+            set_name = r.get("set_name", "__default__")
+            try:
+                rows.append((x, float(y), set_name))
+                count += 1
+            except Exception:
+                pass
+
+    if not rows:
+        print(f"No valid '{obj.x_field}' and '{obj.y_field}' records found.")
+        return
+
+    plt.figure()
+
+    # ---------- robust time-series detection (without false positives) ----------
+    df = pd.DataFrame(rows, columns=["x", "y", "set"])
+
+    # Optional explicit override from caller
+    x_is_time_override = getattr(obj, "x_is_time", None)
+
+    def looks_like_datetime_str(s: str) -> bool:
+        # quick heuristics for common date/time strings (ISO, with separators, or HH:MM)
+        return bool(
+            re.search(r"\d{4}[-/]\d{1,2}[-/]\d{1,2}", s) or  # YYYY-MM-DD or YYYY/MM/DD
+            re.search(r"\d{1,2}[-/]\d{1,2}[-/]\d{2,4}", s) or  # MM-DD-YYYY, etc.
+            re.search(r"\d{1,2}:\d{2}", s) or                  # HH:MM present
+            ("T" in s) or ("Z" in s)                           # ISO 8601 hints
+        )
+
+    def detect_time_series(xs: pd.Series) -> bool:
+        # Already datetime dtype?
+        if pd.api.types.is_datetime64_any_dtype(xs):
+            return True
+
+        # Native datetime-like objects
+        if (xs.map(lambda v: isinstance(v, (datetime, date, pd.Timestamp, np.datetime64))).mean() >= 0.9):
+            return True
+
+        # String-looking dates
+        str_mask = xs.map(lambda v: isinstance(v, str))
+        if str_mask.mean() >= 0.9:
+            looks_mask = xs[str_mask].map(looks_like_datetime_str)
+            if looks_mask.mean() >= 0.9:
+                parsed = pd.to_datetime(xs[str_mask][looks_mask], errors="coerce", utc=False)
+                if parsed.notna().mean() >= 0.9:
+                    return True
+
+        # Numeric epoch seconds/millis (avoid generic integer → ns trap)
+        num = pd.to_numeric(xs, errors="coerce")
+        if num.notna().mean() >= 0.9:
+            # use quantiles to ignore a few bad points
+            q05, q95 = num.quantile(0.05), num.quantile(0.95)
+            # seconds since epoch ~ [1e9, 2e9] for 2001–2033
+            seconds_like = 1e9 <= q05 <= 2.2e9 and 1e9 <= q95 <= 2.2e9
+            # millis since epoch ~ [1e12, 2e12]
+            millis_like = 1e12 <= q05 <= 2.2e12 and 1e12 <= q95 <= 2.2e12
+            if seconds_like or millis_like:
+                return True
+
+        return False
+
+    if isinstance(x_is_time_override, bool):
+        is_time_series = x_is_time_override
+    else:
+        is_time_series = detect_time_series(df["x"])
+
+    if is_time_series:
+        # ===== time-series path (real datetime index) =====
+        # Parse with care: handle epoch seconds/millis and strings
+        x_series = df["x"]
+
+        # Try numeric epochs first
+        numeric = pd.to_numeric(x_series, errors="coerce")
+        parsed = None
+        if numeric.notna().mean() >= 0.9:
+            q05, q95 = numeric.quantile(0.05), numeric.quantile(0.95)
+            if 1e12 <= q05 <= 2.2e12 and 1e12 <= q95 <= 2.2e12:
+                parsed = pd.to_datetime(numeric, unit="ms", errors="coerce", utc=False)
+            elif 1e9 <= q05 <= 2.2e9 and 1e9 <= q95 <= 2.2e9:
+                parsed = pd.to_datetime(numeric, unit="s", errors="coerce", utc=False)
+
+        # Fallback to generic parser for strings/datetimes
+        if parsed is None:
+            parsed = pd.to_datetime(x_series, errors="coerce", utc=False)
+
+        df["ts"] = parsed
+        df = df.dropna(subset=["ts"])
+        if df.empty:
+            print("No parsable datetime values in x_field.")
+            return
+
+        # aggregate duplicates at same timestamp per set
+        df = df.groupby(["ts", "set"], as_index=False)["y"].sum()
+        sets = sorted(df["set"].unique())
+
+        ax = plt.gca()
+        for sname in sets:
+            s = df[df["set"] == sname].set_index("ts")["y"].sort_index()
+            label = None if sname == "__default__" else sname
+            # line plot for time series (clean)
+            ax.plot(s.index, s.values, label=label)
+
+        # ---- sparse, readable ticks (dynamic) ----
+        ts_min, ts_max = df["ts"].min(), df["ts"].max()
+        span_hours = max((ts_max - ts_min).total_seconds() / 3600.0, 1)
+
+        if span_hours <= 72:
+            major = mdates.HourLocator(interval=6)        # every 6 hours
+            fmt   = mdates.DateFormatter("%m-%d %H:%M")
+        elif span_hours <= 14 * 24:
+            major = mdates.HourLocator(interval=12)       # every 12 hours
+            fmt   = mdates.DateFormatter("%m-%d %H:%M")
+        elif span_hours <= 90 * 24:
+            major = mdates.DayLocator(interval=1)         # daily
+            fmt   = mdates.DateFormatter("%Y-%m-%d")
+        else:
+            major = mdates.WeekdayLocator(byweekday=mdates.MO, interval=1)  # weekly
+            fmt   = mdates.DateFormatter("%Y-%m-%d")
+
+        ax.xaxis.set_major_locator(major)
+        ax.xaxis.set_major_formatter(fmt)
+        # no minor tick labels; rotate to avoid overlap
+        plt.gcf().autofmt_xdate()
+
+        plt.xlabel(obj.x_field)
+        plt.ylabel(obj.y_field)
+        if any(s != "__default__" for s in sets):
+            plt.legend(title="data set")
+        plt.title(f"{obj.y_field} over time")
+        plt.text(1.0, 0.95, f"{count} data points", transform=ax.transAxes,
+                 ha='right', va='top', fontsize=10, color='gray')
+
+    else:
+        # ===== categorical / numeric path (original behavior, with tick thinning) =====
+        data = defaultdict(float)
+        all_x = []
+        all_sets = set()
+
+        for x, y, set_name in rows:
+            data[(x, set_name)] += y
+            all_sets.add(set_name)
+            all_x.append(x)
+
+        if not data:
+            print(f"No valid '{obj.x_field}' and '{obj.y_field}' records found.")
+            return
+
+        # preserve order but unique
+        seen = set()
+        x_vals = [x for x in all_x if not (x in seen or seen.add(x))]
+        set_names = sorted(all_sets)
+        x_indices = np.arange(len(x_vals))
+        width = 0.8 / len(set_names) if len(set_names) > 1 else 0.6
+
+        for i, set_name in enumerate(set_names):
+            heights = [data.get((x, set_name), 0) for x in x_vals]
+            label = None if set_name == "__default__" else set_name
+            offset = (i - (len(set_names) - 1) / 2) * width
+
+            if type == 'bar':
+                plt.bar(x_indices + offset, heights, width=width, label=label, edgecolor='black')
+            else:
+                plt.plot(x_indices, heights, marker='o', label=label)
+
+        # ---- thin xticks to at most 12 labels ----
+        max_ticks = 12
+        if len(x_vals) > max_ticks:
+            step = int(np.ceil(len(x_vals) / max_ticks))
+            tick_idx = x_indices[::step]
+            tick_lbl = [x_vals[i] for i in tick_idx]
+        else:
+            tick_idx = x_indices
+            tick_lbl = x_vals
+
+        plt.xticks(tick_idx, tick_lbl, rotation=45)
+        plt.xlabel(obj.x_field)
+        plt.ylabel(obj.y_field)
+        if len(set_names) > 1 or "__default__" not in set_names:
+            plt.legend(title="data set")
+        plt.title(f"{obj.y_field} by {obj.x_field}")
+        plt.text(1.0, 0.95, f"{count} data points", transform=plt.gca().transAxes,
+                 ha='right', va='top', fontsize=10, color='gray')
+
+    # ---------- optional plt args ----------
+    for name, val in getattr(obj, "args_dict", {}).items():
+        fn = getattr(plt, name, None)
+        if callable(fn):
+            try:
+                fn(val)
+            except Exception:
+                pass
+
+    plt.grid(True, linestyle='--', alpha=0.6)
+    plt.tight_layout()

pjk/sinks/graph_cumulative.py

@@ -0,0 +1,55 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2024 Mike Schultz
+
+def graph_cumulative(obj):
+    import matplotlib.pyplot as plt # lazy import
+    
+    # Filter and sort records by x
+    records = [
+        r for r in obj.records
+        if obj.x_field in r and obj.y_field in r
+    ]
+    try:
+        sorted_records = sorted(records, key=lambda r: r[obj.x_field])
+    except TypeError:
+        print(f"Unable to sort records by '{obj.x_field}' — incompatible types.")
+        return
+
+    x_vals = []
+    y_vals = []
+    total = 0
+    count = 0
+    for r in sorted_records:
+        try:
+            x = r[obj.x_field]
+            y = r[obj.y_field]
+            total += y
+            x_vals.append(x)
+            y_vals.append(total)
+            count += 1
+        except Exception:
+            pass  # silently skip bad records
+
+    if not x_vals:
+        print(f"No valid '{obj.x_field}' and '{obj.y_field}' data for cumulative plot.")
+        return
+
+    plt.figure()
+    plt.plot(x_vals, y_vals, marker='o', linestyle='-', label='Cumulative')
+
+    # Apply user-specified styling functions (e.g. title, xlabel, etc.)
+    #for name, val in obj.args_dict.items():
+    #    fn = getattr(plt, name, None)
+    #    if fn and callable(fn):
+    #        fn(val)
+
+    plt.xlabel(obj.x_field)
+    plt.ylabel(f"cumulative({obj.y_field})")
+    plt.text(1.0, 1.0, f"Cumulative {obj.y_field} over {obj.x_field}", transform=plt.gca().transAxes,
+            ha='right', va='top', fontsize=10, color='gray')
+    plt.text(1.0, 0.95, f"{count} data points", transform=plt.gca().transAxes,
+            ha='right', va='top', fontsize=10, color='gray')
+    plt.grid(True, linestyle='--', alpha=0.6)
+    plt.tight_layout()
+
+

pjk/sinks/graph_hist.py

@@ -0,0 +1,72 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2024 Mike Schultz
+
+from collections import defaultdict
+
+def aggregate_ys(obj):
+    agg = defaultdict(float)
+    count = 0
+    for r in obj.records:
+        if obj.x_field in r and obj.y_field in r:
+            try:
+                agg[r[obj.x_field]] += r[obj.y_field]
+                count += 1
+            except Exception:
+                pass  # skip malformed record
+
+    return count, agg
+
+def graph_hist(obj):
+    import matplotlib.pyplot as plt # lazy imports
+    import numpy as np
+
+    count, agg = aggregate_ys(obj)
+    if agg:
+        x_vals = sorted(agg)
+        y_vals = [agg[x] for x in x_vals]
+
+        plt.figure()
+        plt.bar(x_vals, y_vals, edgecolor='black')
+
+        ylabel = f"sum({obj.y_field})"
+    else:
+        # Count mode: standard histogram
+        x_vals = [r[obj.x_field] for r in obj.records if obj.x_field in r]
+        count = len(x_vals)
+
+        if not x_vals:
+            print(f"No valid '{obj.x_field}' data for histogram.")
+            return
+
+        bin_width = 1
+        min_val = min(x_vals)
+        max_val = max(x_vals)
+        bins = np.arange(min_val, max_val + bin_width, bin_width)
+
+        plt.figure()
+        plt.hist(
+            x_vals,
+            bins=bins,
+            edgecolor='black',
+            rwidth=0.8,
+            align='mid'
+        )
+
+        ylabel = "count"
+
+    # need to fix as usage.params and list them
+    #for name, val in obj.args_dict.items():
+    #    fn = getattr(plt, name, None)
+    #    if fn and callable(fn):
+    #       fn(val)
+
+    plt.xlabel(obj.x_field)
+    plt.ylabel(ylabel)
+    plt.text(1.0, 1.0, f"Histogram of {obj.x_field}", transform=plt.gca().transAxes,
+            ha='right', va='top', fontsize=10, color='gray')
+    plt.text(1.0, 0.95, f"{count} data points", transform=plt.gca().transAxes,
+            ha='right', va='top', fontsize=10, color='gray')
+    plt.grid(True, linestyle='--', alpha=0.6)
+    plt.tight_layout()
+    #plt.show()
+

pjk/sinks/graph_scatter.py

@@ -0,0 +1,29 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2024 Mike Schultz
+
+def graph_scatter(obj):
+    import matplotlib.pyplot as plt # lazy imports
+    import numpy as np
+
+    valid_records = [r for r in obj.records if obj.x_field in r and obj.y_field in r]
+    x_vals = [r[obj.x_field] for r in valid_records]
+    y_vals = [r[obj.y_field] for r in valid_records]
+
+    if not x_vals or not y_vals:
+        print(f"No valid '{obj.x_field}' and '{obj.y_field}' data for scatter plot.")
+        return
+
+    correlation = np.corrcoef(x_vals, y_vals)[0, 1]
+    slope, intercept = np.polyfit(x_vals, y_vals, 1)
+    regression_line = [slope * x + intercept for x in x_vals]
+
+    plt.figure()
+    plt.scatter(x_vals, y_vals, label="Data")
+    plt.plot(x_vals, regression_line, color="red", label=f"{obj.y_field} = {slope:.2f}*{obj.x_field} + {intercept:.2f}")
+    plt.xlabel(obj.x_field)
+    plt.ylabel(obj.y_field)
+    plt.title(f"Scatter Plot\nCorrelation: {correlation:.3f}")
+    plt.legend()
+    plt.grid(True)
+    plt.tight_layout()
+

pjk/sinks/json_sink.py

@@ -0,0 +1,23 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2024 Mike Schultz
+
+import os
+import gzip
+import json
+from pjk.base import Sink, Source, ParsedToken, Usage
+
+class JsonSink(Sink):
+    is_format = True
+
+    def __init__(self, ptok: ParsedToken, usage: Usage):
+        super().__init__(ptok, usage)
+        self.path_no_ext = ptok.pre_colon # NOTE: ptok built by framework, doesn't use usage
+        self.gz = ptok.get_arg(0) == 'True'# NOTE: ptok built by framework, doesn't use usage
+
+    def process(self) -> None:
+        path = self.path_no_ext + ('.json.gz' if self.gz else '.json')
+        open_func = gzip.open if self.gz else open
+
+        with open_func(path, 'wt', encoding='utf-8') as f:
+            for record in self.input:
+                f.write(json.dumps(record) + '\n')

pjk/sinks/s3_sink.py

@@ -0,0 +1,100 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2025 Mike Schultz
+
+from typing import Optional, Type
+from pjk.base import Source, Sink, ParsedToken, Usage
+from pjk.log import logger
+
+
+class S3Sink(Sink):
+    """
+    Write records to S3 in the given <format>, partitioned into:
+      s3:{bucket}/{prefix}/file-0000
+      s3:{bucket}/{prefix}/file-0001
+    Args (via Usage):
+      - path: 'bucket/path/to/files' (bucket required, prefix optional)
+    """
+
+    _FILENAME_BASE: str = "file"
+    _FILENAME_DIGITS: int = 4
+    _SCHEME: str = "s3:"
+
+    @classmethod
+    def usage(cls):
+        usage = Usage(
+            name="<format>",
+            desc="Write records to S3 in the given <format> (e.g., csv)",
+            component_class=cls,
+        )
+        usage.def_arg(name="path", usage="bucket/path/to/files")
+        return usage
+
+    def __init__(
+        self,
+        ptok: ParsedToken,
+        usage: Usage,
+        sink_class: Type[Sink],
+        is_gz: bool,
+        fileno: int = 0,
+    ):
+        super().__init__(ptok, usage)
+
+        raw_path: Optional[str] = usage.get_arg("path")
+        if not raw_path:
+            raise ValueError("S3Sink requires 'path' argument like 'bucket/path/to/files'")
+
+        # Normalize: allow 's3:bucket/...' or '/bucket/...', strip extras
+        path = raw_path.strip()
+        if path.startswith(self._SCHEME):
+            path = path[len(self._SCHEME) :]
+        path = path.lstrip("/")
+
+        # Ensure a trailing slash so we can append filenames cleanly
+        self.base_path: str = path if path.endswith("/") else path + "/"
+
+        self.ptok = ptok
+        self.usage = usage
+        self.sink_class = sink_class
+        self.is_gz = is_gz
+        self.fileno = fileno
+        self.num_files = 1  # next file index for deep_copy clones
+
+    def _build_object_key(self, index: int) -> str:
+        file_name = f"{self._FILENAME_BASE}-{index:0{self._FILENAME_DIGITS}d}"
+        return f"{self.base_path}{file_name}"
+
+    def _build_parsed_token_for_index(self, index: int) -> ParsedToken:
+        key = self._build_object_key(index)
+        token_str = f"{self._SCHEME}{key}:{self.is_gz}"
+        return ParsedToken(token_str)
+
+    def process(self):
+        file_ptok = self._build_parsed_token_for_index(self.fileno)
+
+        file_usage = self.sink_class.usage()
+        file_usage.bind(file_ptok)
+
+        file_sink = self.sink_class(file_ptok, file_usage)
+        file_sink.add_source(self.input)
+
+        logger.debug(
+            f"in process sinking to: s3:{self.base_path} (object index {self.fileno:0{self._FILENAME_DIGITS}d})"
+        )
+        file_sink.process()
+
+    def deep_copy(self):
+        source_clone: Optional[Source] = self.input.deep_copy()
+        if not source_clone:
+            return None
+
+        clone = S3Sink(
+            ptok=self.ptok,
+            usage=self.usage,
+            sink_class=self.sink_class,
+            is_gz=self.is_gz,
+            fileno=self.num_files,
+        )
+        clone.add_source(source_clone)
+
+        self.num_files += 1
+        return clone