ohlcutils 0.2.0__tar.gz → 0.2.2__tar.gz

{ohlcutils-0.2.0 → ohlcutils-0.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ohlcutils
-Version: 0.2.0
+Version: 0.2.2
 Summary: A lightweight Python library for python utilities for holidays, dates, options and other tools
 Author-email: Pankaj Sharma <sharma.pankaj.kumar@gmail.com>
 License-Expression: MIT

ohlcutils-0.2.2/ohlcutils/archive.py

@@ -0,0 +1,99 @@
+"""Extract one archived persecond session day from a yearly tar into a temp dir."""
+
+from __future__ import annotations
+
+import os
+import shutil
+import subprocess
+import tempfile
+import uuid
+from pathlib import Path
+
+
+def persecond_tar_path(archive_root: Path, kind: str, year: str) -> Path:
+    return archive_root / f"persecond-{kind}-{year}.tar.gz"
+
+
+def live_persecond_day_dir(data_root: Path, kind: str, day: str) -> Path:
+    return data_root / "persecond" / kind / day
+
+
+class PersecondDayResolver:
+    """Resolve a persecond RDS path, extracting one day from archive if needed."""
+
+    def __init__(
+        self,
+        data_root: Path,
+        archive_root: Path | None = None,
+        extract_root: Path | None = None,
+    ):
+        self.data_root = Path(data_root)
+        self.archive_root = Path(archive_root) if archive_root is not None else self.data_root / "archive"
+        self._extract_root = Path(extract_root) if extract_root is not None else None
+        self._owned_tmps: list[Path] = []
+        self._extract_parent_dir: Path | None = None
+        self._day_dirs: dict[tuple[str, str], Path] = {}
+
+    def resolve(self, path: str) -> str:
+        source = Path(path)
+        if source.exists():
+            return path
+        try:
+            day = source.parent.name
+            kind = source.parent.parent.name
+            periodicity = source.parent.parent.parent.name
+        except Exception:
+            return path
+        if periodicity != "persecond":
+            return path
+        day_dir = self.ensure_day(kind, day)
+        if day_dir is None:
+            return path
+        return str(day_dir / source.name)
+
+    def ensure_day(self, kind: str, day: str) -> Path | None:
+        key = (kind, day)
+        if key in self._day_dirs:
+            return self._day_dirs[key]
+        live = live_persecond_day_dir(self.data_root, kind, day)
+        if live.is_dir():
+            self._day_dirs[key] = live
+            return live
+        year = day[:4]
+        tar_path = persecond_tar_path(self.archive_root, kind, year)
+        if not tar_path.is_file():
+            return None
+        dest_parent = self._shared_extract_parent()
+        dest_parent.mkdir(parents=True, exist_ok=True)
+        try:
+            subprocess.run(
+                ["tar", "-xzf", str(tar_path), "-C", str(dest_parent), day],
+                check=True,
+                capture_output=True,
+                text=True,
+            )
+        except (subprocess.CalledProcessError, FileNotFoundError):
+            return None
+        extracted = dest_parent / day
+        if not extracted.is_dir():
+            return None
+        self._day_dirs[key] = extracted
+        return extracted
+
+    def _shared_extract_parent(self) -> Path:
+        if self._extract_parent_dir is not None:
+            return self._extract_parent_dir
+        if self._extract_root is not None:
+            parent = self._extract_root / str(uuid.uuid4())
+        else:
+            parent = Path(tempfile.gettempdir()) / "rds-untar" / str(os.getpid()) / str(uuid.uuid4())
+        self._extract_parent_dir = parent
+        self._owned_tmps.append(parent)
+        return parent
+
+    def cleanup(self) -> None:
+        for tmp in self._owned_tmps:
+            shutil.rmtree(tmp, ignore_errors=True)
+        self._owned_tmps.clear()
+        self._extract_parent_dir = None
+        self._day_dirs.clear()

{ohlcutils-0.2.0 → ohlcutils-0.2.2}/ohlcutils/data.py

@@ -4,6 +4,7 @@ import datetime as dt
 import os
 import sys
 import warnings
+from pathlib import Path
 from typing import Literal
 
 import numpy as np
@@ -28,6 +29,7 @@ from chameli.interactions import readRDS
 
 from ._arg_validators import _process_kwargs, _valid_load_symbol_kwargs
 from .config import get_config, get_market_close_time, get_market_open_time
+from .archive import PersecondDayResolver
 from .enums import Periodicity
 from .indicators import create_index_if_missing
 
@@ -96,17 +98,29 @@ def get_linked_symbols(short_symbol: str, complete=False) -> pd.DataFrame:
     """
     out = pd.DataFrame()
     end_time = dt.date.today().strftime("%Y-%m-%d")
+    # Several rows can share a symbol (NSE lists e.g. NOVARTIS and NOVARTIND -> EVONILE); take the latest
+    # row (by date, then file order) and stop on repeats so the chain can never loop forever.
     if complete:
+        seen = {short_symbol}
         while True:
             temp = symbolchange.loc[(symbolchange["oldsymbol"] == short_symbol) & (symbolchange["newsymbol"] != "")]
-            if len(temp) == 1:
-                short_symbol = temp["newsymbol"].item()
             if len(temp) == 0:
                 break
+            temp = temp.sort_values("effectivedate", kind="stable").tail(1)
+            if temp["newsymbol"].item() in seen:
+                break
+            short_symbol = temp["newsymbol"].item()
+            seen.add(short_symbol)
 
+    seen = {short_symbol}
     while True:
         temp = symbolchange.loc[(symbolchange["newsymbol"] == short_symbol) & (symbolchange["oldsymbol"] != "")]
+        if len(temp) > 1:
+            temp = temp.sort_values("effectivedate", kind="stable").tail(1)
+        if len(temp) == 1 and temp.iloc[0]["oldsymbol"] in seen:
+            temp = temp.iloc[0:0]
         if len(temp) == 1:
+            seen.add(temp.iloc[0]["oldsymbol"])
             new_row = {
                 "starttime": temp.iloc[0]["effectivedate"],
                 "endtime": end_time,
@@ -187,9 +201,7 @@ def _split_adjust_market_data(md: pd.DataFrame, src: Periodicity, tz: str) -> pd
         symbol_vector = md.symbol.iloc[0].split("_", -1)
         split_info = get_split_info(symbol_vector[0])
         if not split_info.empty:
-            split_info = split_info.groupby("date", sort=False).apply(
-                lambda x: x.select_dtypes(include=[np.number]).prod()
-            )
+            split_info = split_info.groupby("date", sort=False).prod(numeric_only=True)
             split_info["splitadjust"] = split_info["newshares"] / split_info["oldshares"]
             split_info["splitadjust"] = split_info["splitadjust"].cumprod()
             tz_locale = pytz.timezone(tz)
@@ -393,29 +405,36 @@ def load_symbol(symbol: str, **kwargs) -> pd.DataFrame:
                 symbol_temp_str = "_".join(symbol_temp)
                 paths.extend([_get_data_path(symbol_temp_str, params["src"], params["src"])])
 
-    # Read rds from path
-    if len(paths) == 0:
-        return pd.DataFrame()
-    elif len(paths) == 1:
-        try:
-            out = readRDS(paths[0])
-        except Exception as e:
-            get_ohlcutils_logger().log_error("Failed to read RDS file", e, {"file_path": paths[0], "symbol": symbol})
-            return pd.DataFrame()
-    else:
-        list_out = []
-        for p in paths:
+    resolver = None
+    try:
+        if params["src"] == Periodicity.PERSECOND:
+            resolver = PersecondDayResolver(data_root=Path(datapath))
+            paths = [resolver.resolve(p) for p in paths]
+        if len(paths) == 0:
+            out = pd.DataFrame()
+        elif len(paths) == 1:
             try:
-                out = readRDS(p)
-                list_out.append(out)
+                out = readRDS(paths[0])
             except Exception as e:
-                get_ohlcutils_logger().log_error("Failed to read RDS file", e, {"file_path": p, "symbol": symbol})
-                warnings.warn(f"Failed to read RDS file {p}: {e}", RuntimeWarning)
-        filtered_list_out = [df.dropna(axis=1, how="all") for df in list_out if is_good_df(df)]
-        if filtered_list_out:
-            out = pd.concat(filtered_list_out, ignore_index=True)
+                get_ohlcutils_logger().log_error("Failed to read RDS file", e, {"file_path": paths[0], "symbol": symbol})
+                out = pd.DataFrame()
         else:
-            out = pd.DataFrame()
+            list_out = []
+            for p in paths:
+                try:
+                    out = readRDS(p)
+                    list_out.append(out)
+                except Exception as e:
+                    get_ohlcutils_logger().log_error("Failed to read RDS file", e, {"file_path": p, "symbol": symbol})
+                    warnings.warn(f"Failed to read RDS file {p}: {e}", RuntimeWarning)
+            filtered_list_out = [df.dropna(axis=1, how="all") for df in list_out if is_good_df(df)]
+            if filtered_list_out:
+                out = pd.concat(filtered_list_out, ignore_index=True)
+            else:
+                out = pd.DataFrame()
+    finally:
+        if resolver is not None:
+            resolver.cleanup()
 
     if not isinstance(out, pd.DataFrame):
         return pd.DataFrame()

{ohlcutils-0.2.0 → ohlcutils-0.2.2}/ohlcutils.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ohlcutils
-Version: 0.2.0
+Version: 0.2.2
 Summary: A lightweight Python library for python utilities for holidays, dates, options and other tools
 Author-email: Pankaj Sharma <sharma.pankaj.kumar@gmail.com>
 License-Expression: MIT

{ohlcutils-0.2.0 → ohlcutils-0.2.2}/ohlcutils.egg-info/SOURCES.txt

@@ -2,6 +2,7 @@ README.md
 pyproject.toml
 ohlcutils/__init__.py
 ohlcutils/_arg_validators.py
+ohlcutils/archive.py
 ohlcutils/charting.py
 ohlcutils/config.py
 ohlcutils/data.py
@@ -13,4 +14,5 @@ ohlcutils.egg-info/SOURCES.txt
 ohlcutils.egg-info/dependency_links.txt
 ohlcutils.egg-info/requires.txt
 ohlcutils.egg-info/top_level.txt
-ohlcutils/config/config_sample.yaml
+ohlcutils/config/config_sample.yaml
+tests/test_archive.py

{ohlcutils-0.2.0 → ohlcutils-0.2.2}/pyproject.toml

@@ -27,7 +27,7 @@ include-package-data = true
 
 [project]
 name = "ohlcutils"
-version = "0.2.0"
+version = "0.2.2"
 description = "A lightweight Python library for python utilities for holidays, dates, options and other tools"
 readme = "README.md"
 license = "MIT"

ohlcutils-0.2.2/tests/test_archive.py

@@ -0,0 +1,90 @@
+import hashlib
+import subprocess
+from pathlib import Path
+
+from ohlcutils.archive import PersecondDayResolver, persecond_tar_path
+
+
+def _make_day(root: Path, kind: str, day: str, names: list[str]) -> Path:
+    day_dir = root / "persecond" / kind / day
+    day_dir.mkdir(parents=True)
+    for name in names:
+        (day_dir / name).write_text("payload-" + name, encoding="utf-8")
+    return day_dir
+
+
+def _tar_year(data_root: Path, archive_root: Path, kind: str, year: str, days: list[str]) -> Path:
+    archive_root.mkdir(parents=True, exist_ok=True)
+    dest = persecond_tar_path(archive_root, kind, year)
+    parent = data_root / "persecond" / kind
+    subprocess.run(["tar", "-czf", str(dest), "-C", str(parent), *days], check=True)
+    return dest
+
+
+def test_live_day_needs_no_tar(tmp_path: Path):
+    day = "2018-01-02"
+    _make_day(tmp_path, "stk", day, ["AAA_STK____2018-01-02.rds"])
+    resolver = PersecondDayResolver(tmp_path, archive_root=tmp_path / "archive")
+    try:
+        path = tmp_path / "persecond" / "stk" / day / "AAA_STK____2018-01-02.rds"
+        assert resolver.resolve(str(path)) == str(path)
+        assert resolver._owned_tmps == []
+    finally:
+        resolver.cleanup()
+
+
+def test_extract_one_day_then_cleanup(tmp_path: Path):
+    day = "2018-01-02"
+    other = "2018-01-03"
+    _make_day(tmp_path, "stk", day, ["AAA_STK____2018-01-02.rds"])
+    _make_day(tmp_path, "stk", other, ["BBB_STK____2018-01-03.rds"])
+    archive_root = tmp_path / "archive"
+    _tar_year(tmp_path, archive_root, "stk", "2018", [day, other])
+    live_day = tmp_path / "persecond" / "stk" / day
+    other_dir = tmp_path / "persecond" / "stk" / other
+    import shutil
+
+    shutil.rmtree(live_day)
+    shutil.rmtree(other_dir)
+
+    missing = tmp_path / "persecond" / "stk" / day / "AAA_STK____2018-01-02.rds"
+    assert not missing.exists()
+    extract_root = tmp_path / "extracts"
+    resolver = PersecondDayResolver(tmp_path, archive_root=archive_root, extract_root=extract_root)
+    try:
+        resolved = Path(resolver.resolve(str(missing)))
+        assert resolved.exists()
+        assert resolved.read_text(encoding="utf-8") == "payload-AAA_STK____2018-01-02.rds"
+        assert resolved.parent.name == day
+        other_extracted = extract_root.rglob("BBB_STK____2018-01-03.rds")
+        assert list(other_extracted) == []
+        owned = list(resolver._owned_tmps)
+        assert owned
+    finally:
+        resolver.cleanup()
+    for owned in owned:
+        assert not owned.exists()
+    assert not missing.exists()
+
+
+def test_missing_tar_returns_original_path(tmp_path: Path):
+    path = tmp_path / "persecond" / "stk" / "2018-01-02" / "AAA_STK____2018-01-02.rds"
+    resolver = PersecondDayResolver(tmp_path, archive_root=tmp_path / "archive")
+    try:
+        assert resolver.resolve(str(path)) == str(path)
+    finally:
+        resolver.cleanup()
+
+
+def test_tar_member_checksum_stable(tmp_path: Path):
+    day = "2014-04-28"
+    _make_day(tmp_path, "ind", day, ["NIFTY_IND____2014-04-28.rds"])
+    archive_root = tmp_path / "archive"
+    dest = _tar_year(tmp_path, archive_root, "ind", "2014", [day])
+    listing = subprocess.check_output(["tar", "-tzf", str(dest)], text=True)
+    digest = hashlib.sha256("".join(sorted(listing.splitlines(True))).encode()).hexdigest()
+    listing2 = subprocess.check_output(["tar", "-tzf", str(dest)], text=True)
+    digest2 = hashlib.sha256("".join(sorted(listing2.splitlines(True))).encode()).hexdigest()
+    assert digest == digest2
+    top = sorted({line.split("/")[0] for line in listing.splitlines() if line.strip()})
+    assert top == [day]