horizon-data-core 0.0.1__tar.gz

horizon_data_core-0.0.1/.coveragerc

@@ -0,0 +1,11 @@
+[html]
+directory = .coverage_reports/html
+
+[report]
+fail_under = 70
+exclude_lines =
+    pragma: no cover
+    if TYPE_CHECKING:
+
+[xml]
+output = .coverage_reports/coverage.xml

horizon_data_core-0.0.1/.gitignore

@@ -0,0 +1,18 @@
+*.tsbuildinfo
+.coverage
+.coverage_reports
+.mypy_cache
+.pytest_cache
+.ruff_cache
+.turbo
+.venv
+__pycache__
+node_modules
+tmp*
+minio_data
+*.duckdb
+*.duckdb.wal
+dagster_home/*
+!dagster_home/dagster.yaml
+pretrained_models
+htmlcov

horizon_data_core-0.0.1/.npmpackagejsonlintrc.json

@@ -0,0 +1,11 @@
+{
+  "extends": "@spear-ai/npm-package-json-lint-config/spear-application",
+  "rules": {
+    "prefer-caret-version-dependencies": [
+      "error",
+      {
+        "exceptions": ["@spear-ai/horizon-database"]
+      }
+    ]
+  }
+}

horizon_data_core-0.0.1/CHANGELOG.md

@@ -0,0 +1,210 @@
+# @spear-ai/horizon-data-core
+
+## 3.11.2
+
+### Patch Changes
+
+- 64177c0: Synchronize shared developer tooling versions across packages via a catalog.
+- Updated dependencies [64177c0]
+- Updated dependencies [bcb856f]
+  - @spear-ai/horizon-database@3.11.2
+
+## 3.11.1
+
+### Patch Changes
+
+- d77906c: Fixed the release action after the pnpm migration.
+- Updated dependencies [e9e4e5a]
+- Updated dependencies [488ed7c]
+- Updated dependencies [d77906c]
+  - @spear-ai/horizon-database@3.11.1
+
+## 3.11.0
+
+### Minor Changes
+
+- bef40ed: Migrated from Yarn 4 to PNPM 10 for workspace and package management.
+- cae99c1: Added a human-friendly `name` field to the data_stream table for display in the UI, and updated the SDK to generate idempotent data stream IDs based on sensor and platform names.
+
+### Patch Changes
+
+- 798ad81: Added elevation, update_rate, and normalizer fields to BearingTimeRecordSpecification across horizon-data-core, horizon-database, and horizon-app packages
+- 8d93639: Completed a dependency management and update pass.
+- dfc3dd6: Add pitch, roll, and speed_over_ground fields to metadata_row table in horizon-database
+- b2f2d36: Added elevation, update_rate, and normalizer fields to BeamgramSpecification across horizon-data-core, horizon-database, and horizon-app packages
+- Updated dependencies [5de93a0]
+- Updated dependencies [087f0c7]
+- Updated dependencies [f044c34]
+- Updated dependencies [798ad81]
+- Updated dependencies [7bf8707]
+- Updated dependencies [7de66a6]
+- Updated dependencies [8c32a5d]
+- Updated dependencies [8d93639]
+- Updated dependencies [087f0c7]
+- Updated dependencies [7440a27]
+- Updated dependencies [bef40ed]
+- Updated dependencies [dfc3dd6]
+- Updated dependencies [b2f2d36]
+- Updated dependencies [cae99c1]
+- Updated dependencies [9262ea7]
+- Updated dependencies [8641344]
+  - @spear-ai/horizon-database@3.11.0
+
+## 3.10.0
+
+### Minor Changes
+
+- 8ff4746: Make several fixes to how the SDK handles the life cycle of a connection to iceberg catalog
+
+  Changed the default SDK behavior to only write data to postgres instead of iceberg
+
+  BREAKING CHANGES:
+  - Changes how SDK is initialized. Now must provide catalog parameters, not a catalog
+  - Change from "create" and "create_or_update" verbs to "insert" and "upsert" verbs in the SDK
+
+### Patch Changes
+
+- 7affe45: Update shared dependencies and tooling
+- Updated dependencies [c1f5996]
+- Updated dependencies [7affe45]
+- Updated dependencies [5462d23]
+- Updated dependencies [8ff4746]
+  - @spear-ai/horizon-database@3.10.0
+
+## 3.9.0
+
+### Patch Changes
+
+- Updated dependencies [d9ecabb]
+  - @spear-ai/horizon-database@3.9.0
+
+## 3.8.0
+
+### Patch Changes
+
+- Updated dependencies [2502b1f]
+- Updated dependencies [0aaed7e]
+  - @spear-ai/horizon-database@3.8.0
+
+## 3.7.1
+
+### Patch Changes
+
+- f97fca5: Linked Horizon package versions.
+- Updated dependencies [197dd9f]
+- Updated dependencies [f97fca5]
+  - @spear-ai/horizon-database@3.7.1
+
+## 0.3.9
+
+### Patch Changes
+
+- Updated dependencies [39d3fab]
+  - @spear-ai/horizon-database@3.1.4
+
+## 0.3.8
+
+### Patch Changes
+
+- Updated dependencies [186b115]
+  - @spear-ai/horizon-database@3.1.3
+
+## 0.3.7
+
+### Patch Changes
+
+- Updated dependencies [2595d3c]
+  - @spear-ai/horizon-database@3.1.2
+
+## 0.3.6
+
+### Patch Changes
+
+- Updated dependencies [ef40c73]
+  - @spear-ai/horizon-database@3.1.1
+
+## 0.3.5
+
+### Patch Changes
+
+- 7d111f8: Updated dependencies.
+- Updated dependencies [1c37e74]
+- Updated dependencies [1c37e74]
+- Updated dependencies [7d111f8]
+- Updated dependencies [f11ab43]
+- Updated dependencies [26fb39f]
+- Updated dependencies [26fb39f]
+- Updated dependencies [26fb39f]
+  - @spear-ai/horizon-database@3.1.0
+
+## 0.3.4
+
+### Patch Changes
+
+- Updated dependencies [0e34c3f]
+  - @spear-ai/horizon-database@3.0.1
+
+## 0.3.3
+
+### Patch Changes
+
+- 0c63eab: Switch out psycopg2 for psycopg2-binary to avoid installation errors
+- 873d1c2: Adjusted postgres connector to handle special sslmode case for neon connection.
+- Updated dependencies [873d1c2]
+- Updated dependencies [873d1c2]
+- Updated dependencies [b60aeba]
+  - @spear-ai/horizon-database@3.0.0
+
+## 0.3.2
+
+### Patch Changes
+
+- 099931f: Fake patch to induce release.
+- Updated dependencies [099931f]
+  - @spear-ai/horizon-database@2.1.2
+
+## 0.3.1
+
+### Patch Changes
+
+- 1152a0e: Fake patch to allow release.
+- Updated dependencies [1152a0e]
+  - @spear-ai/horizon-database@2.1.1
+
+## 0.3.0
+
+### Minor Changes
+
+- 5e32b51: Adjusted pipelines generating visual artifacts to produce tiles rather than JSON.
+- 5398cf6: Bump python version from 3.11 to 3.12
+
+### Patch Changes
+
+- dd7f8bb: Updated linting dependencies across the packages.
+- 5e6c2c2: Bumped dependency versions.
+- 0639322: Updated and synchronized dependencies across packages.
+- Updated dependencies [5e32b51]
+- Updated dependencies [dd7f8bb]
+- Updated dependencies [72f5207]
+- Updated dependencies [1fb74d0]
+- Updated dependencies [51b4231]
+- Updated dependencies [6878a4e]
+- Updated dependencies [5e6c2c2]
+- Updated dependencies [9aa39ae]
+- Updated dependencies [535cd6d]
+- Updated dependencies [0639322]
+- Updated dependencies [249e936]
+  - @spear-ai/horizon-database@2.1.0
+
+## 0.2.0
+
+### Minor Changes
+
+- 9862f58: Migrate all data pipelines out into separate repo level packages
+
+### Patch Changes
+
+- Updated dependencies [b1442f6]
+- Updated dependencies [b1442f6]
+- Updated dependencies [bd1628a]
+  - @spear-ai/horizon-database@2.0.0

horizon_data_core-0.0.1/PKG-INFO

@@ -0,0 +1,23 @@
+Metadata-Version: 2.4
+Name: horizon-data-core
+Version: 0.0.1
+Summary: Horizon’s core data SDK.
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.12
+Requires-Python: <3.13,>=3.12
+Requires-Dist: dagster-aws>=0.22.1
+Requires-Dist: dagster-iceberg>=0.3.3
+Requires-Dist: dagster-pandera>=0.22.1
+Requires-Dist: dagster>=1.6.1
+Requires-Dist: geojson>=3
+Requires-Dist: pandas>=2
+Requires-Dist: pandera[mypy]>=0.18
+Requires-Dist: polars>=1.33.1
+Requires-Dist: psycopg2-binary>=2.9.10
+Requires-Dist: pyarrow>=21.0.0
+Requires-Dist: pydantic>=2.5
+Requires-Dist: pyiceberg[pyarrow,s3fs]>=0.10
+Requires-Dist: pytest>=8.3.4
+Requires-Dist: s3path>=0.6
+Requires-Dist: sqlalchemy[mypy]>=2.0.36
+Requires-Dist: turfpy>=0.0.7

horizon_data_core-0.0.1/SDK_README.md

@@ -0,0 +1,259 @@
+# Horizon Data Core SDK
+
+The Horizon Data Core SDK provides a simple interface for working with both PostgreSQL and Iceberg tables in the Horizon system.
+
+## Features
+
+- **Pydantic BaseModels**: Type-safe data models for all entities
+- **PostgreSQL Operations**: Full CRUD operations for PostgreSQL tables
+- **Iceberg Operations**: Write and read operations for Iceberg tables
+- **Automatic Conversion**: Seamless conversion between Pydantic models and SQLAlchemy ORM models
+
+## Quick Start
+
+### 1. Initialize the SDK
+
+```python
+from horizon_data_core.api import initialize_sdk
+from horizon_data_core.client import PostgresClient
+from pyiceberg.catalog import load_catalog
+from uuid import uuid4
+
+# Set up PostgreSQL client
+postgres_client = PostgresClient(
+    organization_id=uuid4(),
+    user="postgres",
+    passwd="password",
+    host="localhost",
+    port=5432,
+    database_name="horizon"
+)
+
+# Set up Iceberg catalog
+iceberg_catalog = load_catalog("rest", uri="http://localhost:8181")
+
+# Initialize the SDK with organization_id
+organization_id = uuid4()  # This should come from your user context
+initialize_sdk(postgres_client, iceberg_catalog, organization_id)
+```
+
+### 2. Working with PostgreSQL Tables
+
+```python
+from horizon_data_core.api import create_entity, read_entity, update_entity, delete_entity, list_entities
+from horizon_data_core.base_types import Entity
+from uuid import uuid4
+from datetime import datetime
+
+# Create a new entity
+entity = Entity(
+    id=uuid4(),
+    name="My Entity",
+    kind_id=uuid4(),
+    # organization_id will be automatically set by the SDK
+)
+created_entity = create_entity(entity)
+
+# Read the entity
+retrieved_entity = read_entity(created_entity.id)
+
+# Update the entity
+retrieved_entity.name = "Updated Entity Name"
+updated_entity = update_entity(retrieved_entity)
+
+# List entities with filters (organization_id is automatically applied)
+entities = list_entities()
+
+# Delete the entity
+delete_entity(updated_entity.id)
+```
+
+### 3. Working with Iceberg Tables
+
+```python
+from horizon_data_core.api import create_data_row, create_metadata_row, list_data_rows, list_metadata_rows
+from horizon_data_core.base_types import DataRow, MetadataRow
+from datetime import datetime
+
+# Create a data row
+data_row = DataRow(
+    data_stream_id="stream-123",
+    datetime=datetime.now(),
+    vector=[1.0, 2.0, 3.0, 4.0, 5.0],
+    data_type="sensor_data",
+    vector_start_bound=0.0,
+    vector_end_bound=10.0
+)
+create_data_row(data_row)
+
+# Create a metadata row
+metadata_row = MetadataRow(
+    data_stream_id="stream-123",
+    datetime=datetime.now(),
+    latitude=40.7128,
+    longitude=-74.0060,
+    altitude=10.5,
+    speed=25.0,
+    heading=90.0
+)
+create_metadata_row(metadata_row)
+
+# List data rows with filters
+data_rows = list_data_rows(data_stream_id="stream-123")
+metadata_rows = list_metadata_rows(data_stream_id="stream-123")
+```
+
+## Available Models
+
+### PostgreSQL Models
+
+- `EntityKind`: A descriptive class of which an `entity` is a physical instantiation of.
+- `Entity`: Core entity instances
+- `DataStream`: Data streams associated with entities
+- `Mission`: Mission definitions
+- `MissionEntity`: Mission-entity relationships
+- `Ontology`: Ontology definitions
+- `OntologyClass`: Classes within ontologies
+- `BeamgramSpecification`: The set of parameters used to specify how a beamgram is constructed
+- `BearingTimeRecordSpecification`: The set of parameters used to specify how a bearing time record is constructed
+- `DataRow`: Time-series data with vector information
+- `MetadataRow`: Location and movement metadata
+
+### Iceberg Models
+
+- `DataRow`: Time-series data with vector information
+- `MetadataRow`: Location and movement metadata
+
+## API Reference
+
+### PostgreSQL Operations
+
+For each PostgreSQL model, the following operations are available:
+
+- `create_[model](model_instance)`: Create a new record
+- `read_[model](id)`: Read a record by ID
+- `read_[model](model_instance)`: Read a record by matching non-id fields
+- `update_[model](model_instance)`: Update an existing record
+- `delete_[model](id)`: Delete a record by ID
+- `list_[models](**filters)`: List records with optional filters
+
+### Iceberg Operations
+
+For Iceberg models, the following operations are available:
+
+- `create_[model](model_instance)`: Create a new record
+- `list_[models](**filters)`: List records with optional filters
+
+Note: Update and delete operations for Iceberg tables require table-specific implementation due to the nature of Iceberg's data model.
+
+## Error Handling
+
+The SDK includes proper error handling for:
+
+- Invalid model data
+- Database connection issues
+- Missing records
+- Iceberg catalog connectivity
+
+## Examples
+
+See below for complete working examples of SDK operations.
+
+```python
+"""Example usage of the Horizon Data Core SDK."""
+
+from datetime import datetime
+from uuid import uuid4
+
+from .api import initialize_sdk
+from .base_types import DataRow, DataStream, Entity, MetadataRow, Mission
+from .client import PostgresClient
+from .helpers import name_to_uuid
+
+
+def example_usage() -> None:
+    """Example of how to use the Horizon Data Core SDK."""
+    # Initialize the SDK
+    postgres_client = PostgresClient(
+        user="postgres",
+        passwd="password",
+        host="localhost",
+        port=5432,
+        database_name="horizon",
+    )
+
+    # Initialize the SDK with organization_id
+    organization_id = uuid4()  # This should come from your user context
+    sdk = initialize_sdk(postgres_client, {}, organization_id)
+
+    # Create a new entity
+    entity = Entity(
+        id=uuid4(),
+        name="Example Entity",
+        kind_id=uuid4(),
+        free_text="This is an example entity",
+        # organization_id will be automatically set by the SDK
+    )
+    created_entity = sdk.create_entity(entity)
+    print(f"Created entity: {created_entity}")
+
+    # Read the entity back
+    assert created_entity.id is not None
+    retrieved_entity = sdk.read_entity(created_entity.id)
+    print(f"Retrieved entity: {retrieved_entity}")
+
+    # List entities with filters (organization_id is automatically applied)
+    entities = sdk.list_entities()
+    print(f"Found {len(entities)} entities")
+
+    # Create a data stream
+    data_stream = DataStream(
+        id=uuid4(),
+        entity_id=created_entity.id,
+        # organization_id will be automatically set by the SDK
+    )
+    created_data_stream = sdk.create_data_stream(data_stream)
+    print(f"Created data stream: {created_data_stream}")
+    assert created_data_stream.id is not None
+    # Create a mission
+    mission = Mission(
+        id=uuid4(),
+        name="Example Mission",
+        start_datetime=datetime.now(),
+        # organization_id will be automatically set by the SDK
+    )
+    created_mission = sdk.create_mission(mission)
+    print(f"Created mission: {created_mission}")
+
+    # Create a data row in Postgres
+    data_row = DataRow(
+        data_stream_id=created_data_stream.id,
+        datetime=datetime.now(),
+        vector=[1.0, 2.0, 3.0, 4.0, 5.0],
+        data_type="example_data",
+        track_id=name_to_uuid(
+            "example_track"
+        ),  # This should be a valid specification/track_id that has been created previously
+        vector_start_bound=0.0,
+        vector_end_bound=10.0,
+    )
+    created_data_row = sdk.create_data_row(data_row)
+    print(f"Created data row: {created_data_row}")
+
+    # Create a metadata row in Iceberg table
+    metadata_row = MetadataRow(
+        data_stream_id=created_data_stream.id,
+        datetime=datetime.now(),
+        latitude=40.7128,
+        longitude=-74.0060,
+        altitude=10.5,
+        speed=25.0,
+        heading=90.0,
+    )
+    created_metadata_row = sdk.create_metadata_row(metadata_row)
+    print(f"Created metadata row: {created_metadata_row}")
+
+
+if __name__ == "__main__":
+    example_usage()
+```

horizon_data_core-0.0.1/package.json

@@ -0,0 +1,47 @@
+{
+  "name": "@spear-ai/horizon-data-core",
+  "version": "0.0.1",
+  "description": "Horizon’s core data SDK.",
+  "author": {
+    "name": "Spear AI",
+    "email": "org@spear.ai",
+    "url": "https://spear.ai"
+  },
+  "type": "module",
+  "dependencies": {
+    "@spear-ai/horizon-database": "workspace:^"
+  },
+  "devDependencies": {
+    "@dotenv-run/cli": "catalog:shared-dev",
+    "prettier": "catalog:shared-dev",
+    "turbo": "catalog:shared-dev"
+  },
+  "license": "UNLICENSED",
+  "packageManager": "pnpm@10.26.2",
+  "private": true,
+  "repository": {
+    "type": "git",
+    "directory": "packages/horizon-data-core",
+    "url": "https://github.com/spear-ai/horizon.git"
+  },
+  "scripts": {
+    "build": "uv build",
+    "env:dev": "dotenv-run -e development -r ../..",
+    "env:production": "dotenv-run -e production -r ../..",
+    "env:test": "dotenv-run -e test -r ../..",
+    "mypy:check": "uv run mypy .",
+    "postinstall": "uv sync",
+    "prettier:check": "prettier --check .",
+    "prettier:fix": "prettier --write .",
+    "publish-package": "pnpm run publish-package:jfrog && pnpm run publish-package:pypi",
+    "publish-package:jfrog": "uv publish --check-url \"${JFROG_URL}/artifactory/api/pypi/spear-ai-pypi/simple\" --publish-url \"${JFROG_URL}/artifactory/api/pypi/spear-ai-pypi\" --username \"${JFROG_USERNAME}\" --password \"${JFROG_ACCESS_TOKEN}\"",
+    "publish-package:pypi": "uv publish --check-url https://pypi.org/simple --token \"${PYPI_TOKEN}\"",
+    "pyprojectfmt:check": "uv run pyproject-fmt --check pyproject.toml --keep-full-version",
+    "pyprojectfmt:fix": "uv run pyproject-fmt pyproject.toml --keep-full-version",
+    "pytest:test": "pnpm run env:test -- uv run pytest --tb native",
+    "ruff-format:check": "uv run ruff format --check .",
+    "ruff-format:fix": "uv run ruff format .",
+    "ruff-lint:check": "uv run ruff check .",
+    "ruff-lint:fix": "uv run ruff check --fix ."
+  }
+}

horizon_data_core-0.0.1/pyproject.toml

@@ -0,0 +1,56 @@
+[build-system]
+build-backend = "hatchling.build"
+requires = [ "hatch-nodejs-version==0.4.0", "hatchling==1.28.0" ]
+
+[project]
+name = "horizon-data-core"
+description = "Horizon’s core data SDK."
+readme = "README.md"
+requires-python = "<3.13,>=3.12"
+classifiers = [
+  "Programming Language :: Python :: 3 :: Only",
+  "Programming Language :: Python :: 3.12",
+]
+dynamic = [ "version" ]
+dependencies = [
+  "dagster>=1.6.1",
+  "dagster-aws>=0.22.1",
+  "dagster-iceberg>=0.3.3",
+  "dagster-pandera>=0.22.1",
+  "geojson>=3",
+  "pandas>=2",
+  "pandera[mypy]>=0.18",
+  "polars>=1.33.1",
+  "psycopg2-binary>=2.9.10",
+  "pyarrow>=21.0.0",
+  "pydantic>=2.5",
+  "pyiceberg[s3fs,pyarrow]>=0.10",
+  "pytest>=8.3.4",
+  "s3path>=0.6",
+  "sqlalchemy[mypy]>=2.0.36",
+  "turfpy>=0.0.7",
+]
+
+[dependency-groups]
+dev = [
+  "boto3-stubs[s3]>=1.34.29",
+  "mypy>=1.8",
+  "pandas-stubs>=2",
+  "pyarrow-stubs>=20.0.0.20250928",
+  "pyproject-fmt>=2.6",
+  "pytest>=7",
+  "pytest-cov>=4",
+  "ruff>=0.6.4",
+  "types-pillow>=10",
+]
+
+[tool.hatch.version]
+source = "nodejs"
+
+[tool.pytest.ini_options]
+addopts = "--cov=src --cov-report html --cov-fail-under=90"
+testpaths = [ "." ]
+
+[tool.pyright]
+venvPath = "."
+venv = ".venv"

horizon_data_core-0.0.1/src/horizon_data_core/__init__.py

@@ -0,0 +1,5 @@
+"""Horizon Data Core."""
+
+from .api import get_horizon_sdk
+
+__all__ = ["get_horizon_sdk"]