jerry-thomas 0.3.0__py3-none-any.whl → 1.0.1__py3-none-any.whl

datapipeline/services/runs.py

@@ -0,0 +1,208 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, asdict
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Tuple
+
+import json
+import shutil
+
+
+@dataclass(frozen=True)
+class RunPaths:
+    """Resolved filesystem paths for a single run rooted at a serve directory.
+
+    The serve directory is typically the user-configured `directory` for the
+    filesystem transport (e.g. `data/processed/...`).
+
+    Layout:
+
+        serve_root/
+          runs/
+            <run_id>/
+              dataset/        # main output for this run
+              run.json        # metadata for this run
+          latest/             # symlink or copy pointing at the current live run
+          current_run.json    # pointer to the run currently marked as "latest"
+    """
+
+    serve_root: Path
+    runs_root: Path
+    run_id: str
+    run_root: Path
+    dataset_dir: Path
+    metadata_path: Path
+
+
+@dataclass
+class RunMetadata:
+    """Metadata describing a single run."""
+
+    run_id: str
+    started_at: str
+    finished_at: str | None = None
+    status: str | None = None  # e.g. "running", "success", "failed"
+    notes: str | None = None
+    stage: int | None = None
+
+
+def _now_utc_iso() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+def make_run_id() -> str:
+    """Create a filesystem-safe, sortable run identifier."""
+    return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H-%M-%SZ")
+
+
+def get_serve_root(directory: str | Path) -> Path:
+    """Resolve the user-configured serve directory to an absolute path."""
+    return Path(directory).expanduser().resolve()
+
+
+def get_run_paths(serve_root: Path, run_id: str | None = None) -> RunPaths:
+    """Build RunPaths for a run rooted at the given serve directory."""
+    if run_id is None:
+        run_id = make_run_id()
+
+    runs_root = serve_root / "runs"
+    run_root = runs_root / run_id
+    dataset_dir = run_root / "dataset"
+    metadata_path = run_root / "run.json"
+
+    return RunPaths(
+        serve_root=serve_root,
+        runs_root=runs_root,
+        run_id=run_id,
+        run_root=run_root,
+        dataset_dir=dataset_dir,
+        metadata_path=metadata_path,
+    )
+
+
+def _write_run_metadata(meta: RunMetadata, path: Path) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open("w", encoding="utf-8") as f:
+        json.dump(asdict(meta), f, indent=2, sort_keys=True)
+
+
+def _load_run_metadata(path: Path) -> RunMetadata:
+    with path.open("r", encoding="utf-8") as f:
+        data: dict[str, Any] = json.load(f)
+    return RunMetadata(**data)
+
+
+def start_run_for_directory(
+    directory: str | Path,
+    run_id: str | None = None,
+    *,
+    stage: int | None = None,
+) -> Tuple[RunPaths, RunMetadata]:
+    """Initialise a new run rooted at the given directory.
+
+    This will create the run's dataset directory and an initial metadata file
+    with status set to "running".
+    """
+    serve_root = get_serve_root(directory)
+    paths = get_run_paths(serve_root, run_id)
+
+    # Ensure the run directories exist
+    paths.dataset_dir.mkdir(parents=True, exist_ok=True)
+
+    meta = RunMetadata(
+        run_id=paths.run_id,
+        started_at=_now_utc_iso(),
+        finished_at=None,
+        status="running",
+        notes=None,
+        stage=stage,
+    )
+    _write_run_metadata(meta, paths.metadata_path)
+    return paths, meta
+
+
+def finish_run(paths: RunPaths, status: str, notes: str | None = None) -> RunMetadata:
+    """Mark an existing run as finished with the given status."""
+    if paths.metadata_path.exists():
+        meta = _load_run_metadata(paths.metadata_path)
+    else:
+        # Fallback: create a minimal metadata record if none exists yet
+        meta = RunMetadata(
+            run_id=paths.run_id,
+            started_at=_now_utc_iso(),
+        )
+
+    meta.finished_at = _now_utc_iso()
+    meta.status = status
+    if notes is not None:
+        meta.notes = notes
+
+    _write_run_metadata(meta, paths.metadata_path)
+    return meta
+
+
+def finish_run_success(paths: RunPaths, notes: str | None = None) -> RunMetadata:
+    """Convenience wrapper to mark a run as successful."""
+    return finish_run(paths, status="success", notes=notes)
+
+
+def finish_run_failed(paths: RunPaths, notes: str | None = None) -> RunMetadata:
+    """Convenience wrapper to mark a run as failed."""
+    return finish_run(paths, status="failed", notes=notes)
+
+
+def set_latest_run(paths: RunPaths) -> None:
+    """Mark the given run as the latest/live run for its serve directory.
+
+    This updates two things under the serve root:
+
+      * `latest/` – a symlink (or copied directory as a fallback) pointing to
+        this run's root directory, so consumers can read from
+        `<directory>/latest/dataset`.
+
+      * `current_run.json` – a small pointer file recording which run is
+        currently live and when this pointer was updated.
+    """
+    serve_root = paths.serve_root
+    latest_root = serve_root / "latest"
+
+    # Ensure serve_root exists so that the layout is predictable
+    serve_root.mkdir(parents=True, exist_ok=True)
+
+    # Remove any existing "latest" pointer
+    if latest_root.is_symlink() or latest_root.is_file():
+        latest_root.unlink()
+    elif latest_root.is_dir():
+        shutil.rmtree(latest_root)
+
+    # Prefer a symlink for efficiency; fall back to copying if symlinks fail
+    try:
+        latest_root.symlink_to(paths.run_root, target_is_directory=True)
+    except OSError:
+        shutil.copytree(paths.run_root, latest_root)
+
+    # Write/update current_run.json with a simple pointer
+    current_meta_path = serve_root / "current_run.json"
+    current_data: dict[str, Any] = {
+        "run_id": paths.run_id,
+        "run_root": str(paths.run_root),
+        "dataset_dir": str(paths.dataset_dir),
+        "updated_at": _now_utc_iso(),
+    }
+    with current_meta_path.open("w", encoding="utf-8") as f:
+        json.dump(current_data, f, indent=2, sort_keys=True)
+
+
+__all__ = [
+    "RunPaths",
+    "RunMetadata",
+    "make_run_id",
+    "get_serve_root",
+    "get_run_paths",
+    "start_run_for_directory",
+    "finish_run",
+    "finish_run_success",
+    "finish_run_failed",
+    "set_latest_run",
+]

datapipeline/services/scaffold/domain.py

@@ -9,6 +9,7 @@ from ..paths import pkg_root, resolve_base_pkg_dir
 def create_domain(*, domain: str, root: Optional[Path]) -> None:
     root_dir, name, _ = pkg_root(root)
     base = resolve_base_pkg_dir(root_dir, name)
+    package_name = base.name
     pkg_dir = base / "domains" / domain
     pkg_dir.mkdir(parents=True, exist_ok=True)
     (pkg_dir / "__init__.py").touch(exist_ok=True)
@@ -16,10 +17,10 @@ def create_domain(*, domain: str, root: Optional[Path]) -> None:
     def write_missing(path: Path, tpl: str, **ctx):
         if not path.exists():
             path.write_text(render(tpl, **ctx))
-            print(f"[new] Created: {path}")
+            print(f"[new] {path}")
 
     cls_ = "".join(w.capitalize() for w in domain.split("_"))
     parent = "TemporalRecord"
     write_missing(pkg_dir / "model.py", "record.py.j2",
-                  PACKAGE_NAME=name, DOMAIN=domain, CLASS_NAME=f"{cls_}Record",
+                  PACKAGE_NAME=package_name, DOMAIN=domain, CLASS_NAME=f"{cls_}Record",
                   PARENT_CLASS=parent, time_aware=True)

datapipeline/services/scaffold/filter.py

@@ -9,6 +9,7 @@ from ..paths import pkg_root, resolve_base_pkg_dir
 def create_filter(*, name: str, root: Optional[Path]) -> None:
     root_dir, pkg_name, _ = pkg_root(root)
     base = resolve_base_pkg_dir(root_dir, pkg_name)
+    package_name = base.name
     filters_dir = base / FILTERS_GROUP
     filters_dir.mkdir(parents=True, exist_ok=True)
     (filters_dir / "__init__.py").touch(exist_ok=True)
@@ -18,7 +19,7 @@ def create_filter(*, name: str, root: Optional[Path]) -> None:
     path = filters_dir / f"{module_name}.py"
     if not path.exists():
         path.write_text(render("filter.py.j2", FUNCTION_NAME=name))
-        print(f"[new] Created: {path}")
+        print(f"[new] {path}")
 
     # Register entry point under datapipeline.filters
     toml_path = root_dir / "pyproject.toml"
@@ -26,6 +27,6 @@ def create_filter(*, name: str, root: Optional[Path]) -> None:
         toml_path.read_text(),
         FILTERS_GROUP,
         name,
-        f"{pkg_name}.filters.{module_name}:{name}",
+        f"{package_name}.filters.{module_name}:{name}",
     )
     toml_path.write_text(toml)

datapipeline/services/scaffold/mappers.py

@@ -16,8 +16,11 @@ def _slug(s: str) -> str:
 def attach_source_to_domain(*, domain: str, provider: str, dataset: str, root: Optional[Path]) -> None:
     root_dir, name, pyproject = pkg_root(root)
     base = resolve_base_pkg_dir(root_dir, name)
+    package_name = base.name
     mappers_root = base / MAPPERS_GROUP
-    prov = _slug(provider); ds = _slug(dataset); dom = _slug(domain)
+    _ = _slug(provider)
+    ds = _slug(dataset)
+    dom = _slug(domain)
 
     # Option B layout: mappers/{provider}/{dataset}/to_{domain}.py
     pkg_dir = mappers_root / provider / dataset
@@ -32,7 +35,7 @@ def attach_source_to_domain(*, domain: str, provider: str, dataset: str, root: O
         function_name = "map"
         path.write_text(render(
             "mapper.py.j2",
-            PACKAGE_NAME=name,
+            PACKAGE_NAME=package_name,
             ORIGIN=provider,
             DATASET=dataset,
             TARGET_DOMAIN=dom,
@@ -42,11 +45,11 @@ def attach_source_to_domain(*, domain: str, provider: str, dataset: str, root: O
             OriginDTO=f"{camel(provider)}{camel(dataset)}DTO",
             time_aware=True,
         ))
-        print(f"[new] Created: {path}")
+        print(f"[new] {path}")
 
-    # Register the mapper EP as domain.provider (fallback to domain.provider.dataset on collision handled elsewhere)
-    ep_key = f"{dom}.{prov}"
-    ep_target = f"{name}.mappers.{provider}.{dataset}.{module_name}:map"
+    # Register the mapper EP as domain.dataset
+    ep_key = f"{dom}.{ds}"
+    ep_target = f"{package_name}.mappers.{provider}.{dataset}.{module_name}:map"
     toml = (root_dir / "pyproject.toml").read_text()
     toml = inject_ep(toml, MAPPERS_GROUP, ep_key, ep_target)
     (root_dir / "pyproject.toml").write_text(toml)

datapipeline/services/scaffold/plugin.py

@@ -1,7 +1,15 @@
 from importlib.resources import as_file, files
 from pathlib import Path
+import logging
+import os
 
-from ..constants import COMPOSED_LOADER_EP
+import yaml
+
+from datapipeline.utils.load import load_yaml
+
+from ..constants import DEFAULT_IO_LOADER_EP
+
+logger = logging.getLogger(__name__)
 
 _RESERVED_PACKAGE_NAMES = {"datapipeline"}
 
@@ -9,15 +17,13 @@ _RESERVED_PACKAGE_NAMES = {"datapipeline"}
 def _normalized_package_name(dist_name: str) -> str:
     package_name = dist_name.replace("-", "_")
     if package_name in _RESERVED_PACKAGE_NAMES:
-        print(
-            "[error] `datapipeline` is reserved for the core package. "
-            "Choose a different plugin name."
+        logger.error(
+            "`datapipeline` is reserved for the core package. Choose a different plugin name."
         )
         raise SystemExit(1)
     if not package_name.isidentifier():
-        print(
-            "[error] Plugin names must be valid Python identifiers once hyphens are replaced "
-            "with underscores."
+        logger.error(
+            "Plugin names must be valid Python identifiers once hyphens are replaced with underscores."
         )
         raise SystemExit(1)
     return package_name
@@ -26,7 +32,7 @@ def _normalized_package_name(dist_name: str) -> str:
 def scaffold_plugin(name: str, outdir: Path) -> None:
     target = (outdir / name).absolute()
     if target.exists():
-        print(f"[error] `{target}` already exists")
+        logger.error("`%s` already exists", target)
         raise SystemExit(1)
     import shutil
 
@@ -39,11 +45,49 @@ def scaffold_plugin(name: str, outdir: Path) -> None:
     replacements = {
         "{{PACKAGE_NAME}}": package_name,
         "{{DIST_NAME}}": name,
-        "{{COMPOSED_LOADER_EP}}": COMPOSED_LOADER_EP,
+        "{{DEFAULT_IO_LOADER_EP}}": DEFAULT_IO_LOADER_EP,
     }
     for p in (target / "pyproject.toml", target / "README.md"):
         text = p.read_text()
         for placeholder, value in replacements.items():
             text = text.replace(placeholder, value)
         p.write_text(text)
-    print(f"[new] Created plugin skeleton at {target}")
+
+    # Move jerry.yaml up to the workspace root (current working directory) so
+    # users can run the CLI from the workspace without cd'ing into the plugin.
+    # We adjust plugin_root and dataset paths to point at the plugin directory
+    # relative to the workspace. Do not overwrite an existing workspace
+    # jerry.yaml.
+    plugin_jerry = target / "jerry.yaml"
+    workspace_root = Path.cwd().resolve()
+    workspace_jerry = workspace_root / "jerry.yaml"
+    if plugin_jerry.exists() and not workspace_jerry.exists():
+        try:
+            plugin_root_rel = target.relative_to(workspace_root)
+        except ValueError:
+            # Fall back to a relative path between arbitrary directories; this
+            # may include ".." segments.
+            try:
+                plugin_root_rel = Path(os.path.relpath(target, workspace_root))
+            except Exception:
+                plugin_root_rel = target
+
+        data = load_yaml(plugin_jerry)
+        data["plugin_root"] = plugin_root_rel.as_posix()
+        datasets = data.get("datasets") or {}
+        updated_datasets = {}
+        for alias, path in datasets.items():
+            p = Path(path)
+            if p.is_absolute():
+                updated_datasets[alias] = p.as_posix()
+            else:
+                updated_datasets[alias] = (plugin_root_rel / p).as_posix()
+        data["datasets"] = updated_datasets
+
+        workspace_jerry.write_text(
+            yaml.safe_dump(data, sort_keys=False), encoding="utf-8"
+        )
+        plugin_jerry.unlink()
+        logger.info("workspace jerry.yaml created at %s", workspace_jerry)
+
+    logger.info("plugin skeleton created at %s", target)

datapipeline/services/scaffold/source.py

@@ -4,12 +4,13 @@ from typing import Optional
 
 from datapipeline.services.scaffold.templates import camel, render
 
-from ..constants import COMPOSED_LOADER_EP
+from ..constants import DEFAULT_IO_LOADER_EP
 from ..entrypoints import inject_ep
 from ..paths import pkg_root, resolve_base_pkg_dir
 from datapipeline.services.project_paths import (
     sources_dir as resolve_sources_dir,
     ensure_project_scaffold,
+    resolve_project_yaml_path,
 )
 
 
@@ -19,14 +20,14 @@ def _class_prefix(provider: str, dataset: str) -> str:
 
 
 def _source_alias(provider: str, dataset: str) -> str:
-    return f"{provider}_{dataset}"
+    return f"{provider}.{dataset}"
 
 
 def _write_if_missing(path: Path, text: str) -> None:
     """Write file only if it does not exist; echo a friendly message."""
     if not path.exists():
         path.write_text(text)
-        print(f"[new] Created: {path}")
+        print(f"[new] {path}")
 
 
 def _render_loader_stub(transport: str, loader_class: str,
@@ -55,86 +56,120 @@ def _update_ep(toml_text: str, provider: str, dataset: str, pkg_name: str,
     return toml_text, ep_key
 
 
-def _loader_ep_and_args(transport: str, fmt: Optional[str], ep_key: str) -> tuple[str, dict]:
+def _loader_ep_and_args(transport: str, fmt: Optional[str], ep_key: Optional[str]) -> tuple[str, dict]:
     """Return (loader EP name, default args) for the YAML snippet."""
     if transport == "fs":
         args = {
             "transport": "fs",
-            "format": fmt or "<FORMAT (csv|json|json-lines)>",
+            "format": fmt or "<FORMAT (csv|json|json-lines|pickle)>",
             "path": "<PATH OR GLOB>",
             "glob": False,
             "encoding": "utf-8",
         }
         if fmt == "csv":
             args["delimiter"] = ","
-        return COMPOSED_LOADER_EP, args
+        return DEFAULT_IO_LOADER_EP, args
     if transport == "synthetic":
+        if ep_key is None:
+            raise ValueError("synthetic transport requires scaffolding a loader entrypoint")
         return ep_key, {"start": "<ISO8601>", "end": "<ISO8601>", "frequency": "1h"}
-    if transport == "url":
+    if transport == "http":
         args = {
-            "transport": "url",
+            "transport": "http",
             "format": fmt or "<FORMAT (json|json-lines|csv)>",
             "url": "<https://api.example.com/data.json>",
             "headers": {},
+            "params": {},
             "encoding": "utf-8",
         }
         if fmt == "csv":
             args["delimiter"] = ","
-        return COMPOSED_LOADER_EP, args
+        return DEFAULT_IO_LOADER_EP, args
+    if ep_key is None:
+        raise ValueError(f"unsupported transport '{transport}' for identity scaffold")
     return ep_key, {}
 
 
-def create_source(*, provider: str, dataset: str, transport: str,
-                  format: Optional[str], root: Optional[Path]) -> None:
+def create_source(
+    *,
+    provider: str,
+    dataset: str,
+    transport: str,
+    format: Optional[str],
+    root: Optional[Path],
+    identity: bool = False,
+) -> None:
     root_dir, name, _ = pkg_root(root)
     base = resolve_base_pkg_dir(root_dir, name)
-    src_pkg_dir = base / "sources" / provider / dataset
-    src_pkg_dir.mkdir(parents=True, exist_ok=True)
-    (src_pkg_dir / "__init__.py").touch(exist_ok=True)
-
-    class_prefix = _class_prefix(provider, dataset)
-    dto_class = f"{class_prefix}DTO"
-    parser_class = f"{class_prefix}Parser"
-    loader_class = f"{class_prefix}DataLoader"
-
-    # DTO
-    dto_path = src_pkg_dir / "dto.py"
-    _write_if_missing(dto_path, render(
-        "dto.py.j2",
-        PACKAGE_NAME=name, ORIGIN=provider, DOMAIN=dataset,
-        CLASS_NAME=dto_class, time_aware=True
-    ))
-
-    # Parser
-    parser_path = src_pkg_dir / "parser.py"
-    _write_if_missing(parser_path, render(
-        "parser.py.j2",
-        PACKAGE_NAME=name, ORIGIN=provider, DOMAIN=dataset,
-        CLASS_NAME=parser_class, DTO_CLASS=dto_class, time_aware=True
-    ))
-
-    # Optional loader stub: synthetic (url uses composed loader by default)
-    if transport in {"synthetic"}:
-        loader_path = src_pkg_dir / "loader.py"
-        stub = _render_loader_stub(transport, loader_class, fmt=format)
-        if stub is not None:
-            _write_if_missing(loader_path, stub)
-
-    toml_path = root_dir / "pyproject.toml"
-    toml_text, ep_key = _update_ep(
-        toml_path.read_text(),
-        provider, dataset, name,
-        transport, parser_class, loader_class
-    )
-    toml_path.write_text(toml_text)
+    package_name = base.name
 
     alias = _source_alias(provider, dataset)
+    parser_ep: str
+    parser_args: dict
+    ep_key: Optional[str] = None
+
+    if identity:
+        if transport == "synthetic":
+            raise ValueError(
+                "identity parser scaffold is not supported for synthetic sources; "
+                "generate the standard parser instead."
+            )
+        parser_ep = "identity"
+        parser_args = {}
+    else:
+        src_pkg_dir = base / "sources" / provider / dataset
+        src_pkg_dir.mkdir(parents=True, exist_ok=True)
+        (src_pkg_dir / "__init__.py").touch(exist_ok=True)
+
+        class_prefix = _class_prefix(provider, dataset)
+        dto_class = f"{class_prefix}DTO"
+        parser_class = f"{class_prefix}Parser"
+        loader_class = f"{class_prefix}DataLoader"
+
+        # DTO
+        dto_path = src_pkg_dir / "dto.py"
+        _write_if_missing(dto_path, render(
+            "dto.py.j2",
+            PACKAGE_NAME=package_name, ORIGIN=provider, DOMAIN=dataset,
+            CLASS_NAME=dto_class, time_aware=True
+        ))
+
+        # Parser
+        parser_path = src_pkg_dir / "parser.py"
+        _write_if_missing(parser_path, render(
+            "parser.py.j2",
+            PACKAGE_NAME=package_name, ORIGIN=provider, DOMAIN=dataset,
+            CLASS_NAME=parser_class, DTO_CLASS=dto_class, time_aware=True
+        ))
+
+        # Optional loader stub: synthetic (http uses core IO loader by default)
+        if transport in {"synthetic"}:
+            loader_path = src_pkg_dir / "loader.py"
+            stub = _render_loader_stub(transport, loader_class, fmt=format)
+            if stub is not None:
+                _write_if_missing(loader_path, stub)
+
+        toml_path = root_dir / "pyproject.toml"
+        toml_text, ep_key = _update_ep(
+            toml_path.read_text(),
+            provider,
+            dataset,
+            package_name,
+            transport,
+            parser_class,
+            loader_class,
+        )
+        toml_path.write_text(toml_text)
+
+        parser_ep = ep_key
+        parser_args = {}
+
     loader_ep, loader_args = _loader_ep_and_args(transport, format, ep_key)
 
     # Resolve sources directory from a single dataset-scoped project config.
     # If not present or invalid, let the exception bubble up to prompt the user
     # to provide a valid project path.
-    proj_yaml = root_dir / "config" / "datasets" / "default" / "project.yaml"
+    proj_yaml = resolve_project_yaml_path(root_dir)
     # Best-effort: create a minimal project scaffold if missing
     ensure_project_scaffold(proj_yaml)
     sources_dir = resolve_sources_dir(proj_yaml).resolve()
@@ -143,11 +178,13 @@ def create_source(*, provider: str, dataset: str, transport: str,
     if not src_cfg_path.exists():
         src_cfg_path.write_text(render(
             "source.yaml.j2",
-            source_id=alias,
-            parser_ep=ep_key,
-            parser_args={},
+            id=alias,
+            parser_ep=parser_ep,
+            parser_args=parser_args,
             loader_ep=loader_ep,
             loader_args=loader_args,
-            composed_loader_ep=COMPOSED_LOADER_EP,
+            default_io_loader_ep=DEFAULT_IO_LOADER_EP,
         ))
-        print(f"[new] Created: {src_cfg_path.resolve()}")
+        print(f"[new] {src_cfg_path.resolve()}")
+    elif identity:
+        print(f"[info] Source YAML already exists; skipped identity scaffold at {src_cfg_path.resolve()}")

datapipeline/sources/{composed_loader.py → data_loader.py}

@@ -1,21 +1,21 @@
 from __future__ import annotations
 
 from typing import Iterator, Any, Optional
-from .models.loader import RawDataLoader
-from .transports import TextSource, UrlSource
+from .models.loader import BaseDataLoader
+from .transports import Transport, HttpTransport
 from .decoders import Decoder
 
 
-class ComposedRawLoader(RawDataLoader):
-    """Compose a transport TextSource with a row Decoder."""
+class DataLoader(BaseDataLoader):
+    """Compose a Transport with a row Decoder."""
 
-    def __init__(self, source: TextSource, decoder: Decoder, *, allow_network_count: bool = False):
-        self.source = source
+    def __init__(self, transport: Transport, decoder: Decoder, *, allow_network_count: bool = False):
+        self.transport = transport
         self.decoder = decoder
         self._allow_net_count = bool(allow_network_count)
 
     def load(self) -> Iterator[Any]:
-        for stream in self.source.streams():
+        for stream in self.transport.streams():
             for row in self.decoder.decode(stream):
                 yield row
 
@@ -23,11 +23,11 @@ class ComposedRawLoader(RawDataLoader):
         # Delegate counting to the decoder using the transport streams.
         # Avoid counting over network unless explicitly enabled.
         try:
-            if isinstance(self.source, UrlSource) and not self._allow_net_count:
+            if isinstance(self.transport, HttpTransport) and not self._allow_net_count:
                 return None
             total = 0
             any_stream = False
-            for stream in self.source.streams():
+            for stream in self.transport.streams():
                 any_stream = True
                 c = self.decoder.count(stream)
                 if c is None: