sgn-skig 0.1.0__tar.gz

sgn_skig-0.1.0/.gitignore

@@ -0,0 +1,107 @@
+# Auto-generated version file
+src/sgneskig/_version.py
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# IDEs
+.idea/
+.vscode/
+*.swp
+*.swo
+*~
+
+# macOS
+.DS_Store
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# Secrets / credentials
+*.pem
+*.key
+credentials.json
+secrets.yaml
+.env.local

sgn_skig-0.1.0/.gitlab-ci.yml

@@ -0,0 +1,23 @@
+include:
+  - component: git.ligo.org/computing/gitlab/components/python/sdist@1
+  - component: git.ligo.org/computing/gitlab/components/python/wheel@1
+  - component: git.ligo.org/computing/gitlab/components/python/code-quality@1
+    inputs:
+      requirements: ".[lint]"
+  - component: git.ligo.org/computing/gitlab/components/python/dependency-scanning@1
+  - component: git.ligo.org/computing/gitlab/components/python/test@1
+    inputs:
+      install_extra: test
+      python_versions:
+        - "3.11"
+        - "3.12"
+        - "3.13"
+
+check_makefile:
+  image: python:3.11
+  stage: test
+  script:
+    - apt-get update
+    - apt-get install make
+    - pip3 install ".[dev]"
+    - make all

sgn_skig-0.1.0/Makefile

@@ -0,0 +1,30 @@
+.PHONY: help all install test lint format clean
+
+all: format lint test
+
+help:
+	@echo "Available targets:"
+	@echo "  all        Run format, lint, and test (in order)"
+	@echo "  install    Install package in development mode"
+	@echo "  test       Run tests with coverage"
+	@echo "  lint       Run ruff linter"
+	@echo "  format     Format code with ruff"
+	@echo "  clean      Remove build artifacts"
+
+install:
+	pip install -e ".[dev]"
+
+test:
+	pytest
+
+lint:
+	ruff check src/ tests/
+
+format:
+	ruff format src/ tests/
+	ruff check --fix src/ tests/
+
+clean:
+	rm -rf build/ dist/ *.egg-info src/*.egg-info
+	find . -type d -name __pycache__ -exec rm -rf {} +
+	find . -type f -name "*.pyc" -delete

sgn_skig-0.1.0/PKG-INFO

@@ -0,0 +1,20 @@
+Metadata-Version: 2.4
+Name: sgn-skig
+Version: 0.1.0
+Summary: SGN infrastructure for metrics, Kafka, InfluxDB, and Grafana integration
+Author-email: Chad Hanna <crh184@psu.edu>
+License: MPL-2.0
+Requires-Python: >=3.11
+Requires-Dist: sgn>=0.5
+Requires-Dist: confluent-kafka>=2.0
+Requires-Dist: ligo-scald
+Requires-Dist: pyyaml
+Requires-Dist: requests
+Provides-Extra: test
+Requires-Dist: pytest; extra == "test"
+Requires-Dist: pytest-cov; extra == "test"
+Requires-Dist: pytest-asyncio; extra == "test"
+Provides-Extra: lint
+Requires-Dist: ruff; extra == "lint"
+Provides-Extra: dev
+Requires-Dist: sgn-skig[lint,test]; extra == "dev"

sgn_skig-0.1.0/README.md

@@ -0,0 +1,231 @@
+# sgn-skig
+
+SGN infrastructure for **S**cald, **K**afka, **I**nfluxDB, and **G**rafana.
+
+Provides reusable metrics collection, pipeline infrastructure, and generic elements for [SGN](https://git.ligo.org/cbc/sgn) pipelines.
+
+## Installation
+
+```bash
+pip install sgn-skig
+```
+
+Or for development:
+
+```bash
+pip install -e ".[dev]"
+```
+
+## Features
+
+### MetricsPipeline
+
+A drop-in replacement for `sgn.Pipeline` that adds automatic metrics infrastructure:
+
+```python
+from sgneskig import MetricsPipeline
+
+pipeline = MetricsPipeline(
+    name="my-pipeline",
+    influxdb_host="localhost",
+    influxdb_port=8086,
+    influxdb_db="my_metrics",
+)
+
+# Insert elements - metrics are automatically configured
+pipeline.insert(source, transform, sink)
+pipeline.connect(source, transform)
+pipeline.connect(transform, sink)
+
+# Export Grafana dashboard from discovered metrics
+pipeline.export_grafana_dashboard("dashboard.json")
+```
+
+### MetricsCollectorMixin
+
+Add metrics collection to any SGN element:
+
+```python
+from dataclasses import dataclass
+from sgn.base import TransformElement
+from sgneskig.metrics import MetricsCollectorMixin, metrics
+
+@dataclass
+class MyTransform(TransformElement, MetricsCollectorMixin):
+    # Declarative metrics schema - auto-discovered by MetricsPipeline
+    metrics_schema = metrics([
+        ("events_processed", "counter", ["pipeline"], "Events processed"),
+        ("process_time", "timing", [], "Processing duration"),
+    ])
+
+    # Enable automatic cycle time tracking
+    track_elapsed_time: bool = True
+    elapsed_metric_name: str = "my_transform_elapsed"
+
+    def __post_init__(self):
+        super().__post_init__()
+        self._init_metrics()
+
+    def internal(self):
+        with self.time_operation("process_time"):
+            # ... do work ...
+            pass
+        self.increment_counter("events_processed", tags={"pipeline": "SGNL"})
+```
+
+### Kafka Integration
+
+**KafkaSource** - Read from Kafka topics with automatic batching:
+
+```python
+from sgneskig.sources import KafkaSource
+
+source = KafkaSource(
+    name="kafka_source",
+    bootstrap_servers="localhost:9092",
+    topics=["events"],
+    group_id="my-consumer",
+    auto_offset_reset="latest",
+)
+```
+
+**KafkaSink** - Write to Kafka topics:
+
+```python
+from sgneskig.sinks import KafkaSink
+
+sink = KafkaSink(
+    name="kafka_sink",
+    bootstrap_servers="localhost:9092",
+    topic="output-events",
+)
+```
+
+### Generic Transforms
+
+**DelayBuffer** - Buffer events for a configurable delay:
+
+```python
+from sgneskig.transforms import DelayBuffer
+
+buffer = DelayBuffer(
+    name="delay_buffer",
+    delay_seconds=30.0,  # Hold events for 30s
+)
+```
+
+**EventLatency** - Measure end-to-end latency:
+
+```python
+from sgneskig.transforms import EventLatency
+
+latency = EventLatency(
+    name="latency_tracker",
+    timestamp_field="gpstime",  # Field containing event timestamp
+)
+```
+
+**RoundRobinDistributor** - Distribute work across multiple outputs:
+
+```python
+from sgneskig.transforms import RoundRobinDistributor
+
+distributor = RoundRobinDistributor(
+    name="distributor",
+    num_outputs=4,  # Creates output pads: out_0, out_1, out_2, out_3
+)
+```
+
+### Grafana Integration
+
+Generate dashboards and datasources from your pipeline's metrics:
+
+```python
+# After inserting elements into the pipeline
+pipeline.export_grafana_dashboard("grafana/dashboards/my-pipeline.json")
+pipeline.export_grafana_datasource("grafana/provisioning/datasources/influxdb.yaml")
+
+# Or get the exporter for more control
+exporter = pipeline.get_grafana_exporter(dashboard_title="My Pipeline Metrics")
+print(exporter.datasource_curl_command())
+```
+
+## CLI Tools
+
+### sgn-grafana-datasource
+
+Generate Grafana datasource provisioning YAML:
+
+```bash
+sgn-grafana-datasource --host localhost --port 8086 --db my_metrics > datasource.yaml
+```
+
+### sgn-grafana-import
+
+Import a dashboard JSON file into Grafana:
+
+```bash
+sgn-grafana-import dashboard.json --grafana-url http://localhost:3000 --auth admin:admin
+```
+
+## Architecture
+
+```
+┌─────────────────────────────────────────────────────────────────┐
+│                        MetricsPipeline                          │
+│  ┌──────────────────────────────────────────────────────────┐  │
+│  │                     MetricsWriter                         │  │
+│  │  (shared writer for direct InfluxDB writes via ligo.scald)│  │
+│  └──────────────────────────────────────────────────────────┘  │
+│           ▲                    ▲                    ▲          │
+│           │                    │                    │          │
+│  ┌────────┴───────┐   ┌───────┴────────┐   ┌───────┴───────┐  │
+│  │  KafkaSource   │──▶│  MyTransform   │──▶│   KafkaSink   │  │
+│  │  (metrics via  │   │  (metrics via  │   │  (metrics via │  │
+│  │   mixin)       │   │   mixin)       │   │   mixin)      │  │
+│  └────────────────┘   └────────────────┘   └───────────────┘  │
+└─────────────────────────────────────────────────────────────────┘
+                                │
+                                ▼
+                         ┌────────────┐
+                         │  InfluxDB  │
+                         │ (1s, 10s,  │
+                         │  100s, ... │
+                         │ retention) │
+                         └────────────┘
+                                │
+                                ▼
+                         ┌────────────┐
+                         │  Grafana   │
+                         └────────────┘
+```
+
+## Metrics Storage
+
+sgn-skig uses [ligo.scald](https://lscsoft.docs.ligo.org/scald/) for multi-resolution time series storage. Data is automatically aggregated at multiple resolutions:
+
+- `1s` - 1 second buckets
+- `10s` - 10 second buckets
+- `100s` - ~1.5 minute buckets
+- `1000s` - ~16 minute buckets
+- `10000s` - ~2.7 hour buckets
+- `100000s` - ~1 day buckets
+
+Retention policies are created automatically on first write.
+
+## Development
+
+```bash
+# Run tests
+make test
+
+# Run linter
+make lint
+
+# Run all checks
+make
+```
+
+## License
+
+MPL-2.0

sgn_skig-0.1.0/pyproject.toml

@@ -0,0 +1,49 @@
+[build-system]
+requires = [
+    "setuptools>=42",
+    "setuptools_scm[toml]>=3.4.3",
+    "wheel",
+]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "sgn-skig"
+dynamic = ["version"]
+description = "SGN infrastructure for metrics, Kafka, InfluxDB, and Grafana integration"
+authors = [{ name = "Chad Hanna", email = "crh184@psu.edu" }]
+license = { text = "MPL-2.0" }
+requires-python = ">= 3.11"
+dependencies = [
+    "sgn >= 0.5",
+    "confluent-kafka >= 2.0",
+    "ligo-scald",
+    "pyyaml",
+    "requests",
+]
+
+[project.optional-dependencies]
+test = ["pytest", "pytest-cov", "pytest-asyncio"]
+lint = ["ruff"]
+dev = ["sgn-skig[test,lint]"]
+
+[project.scripts]
+sgn-grafana-datasource = "sgneskig.bin.grafana_datasource:main"
+sgn-grafana-import = "sgneskig.bin.grafana_import:main"
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.setuptools_scm]
+write_to = "src/sgneskig/_version.py"
+
+[tool.ruff]
+line-length = 88
+target-version = "py311"
+
+[tool.ruff.lint]
+select = ["E", "F", "W", "I", "UP", "S", "B", "C4"]
+ignore = ["S101", "S310", "S311", "S603", "S608"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+addopts = "--cov=sgneskig --cov-report=term-missing"

sgn_skig-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

sgn_skig-0.1.0/src/sgn_skig.egg-info/PKG-INFO

@@ -0,0 +1,20 @@
+Metadata-Version: 2.4
+Name: sgn-skig
+Version: 0.1.0
+Summary: SGN infrastructure for metrics, Kafka, InfluxDB, and Grafana integration
+Author-email: Chad Hanna <crh184@psu.edu>
+License: MPL-2.0
+Requires-Python: >=3.11
+Requires-Dist: sgn>=0.5
+Requires-Dist: confluent-kafka>=2.0
+Requires-Dist: ligo-scald
+Requires-Dist: pyyaml
+Requires-Dist: requests
+Provides-Extra: test
+Requires-Dist: pytest; extra == "test"
+Requires-Dist: pytest-cov; extra == "test"
+Requires-Dist: pytest-asyncio; extra == "test"
+Provides-Extra: lint
+Requires-Dist: ruff; extra == "lint"
+Provides-Extra: dev
+Requires-Dist: sgn-skig[lint,test]; extra == "dev"

sgn_skig-0.1.0/src/sgn_skig.egg-info/SOURCES.txt

@@ -0,0 +1,40 @@
+.gitignore
+.gitlab-ci.yml
+Makefile
+README.md
+pyproject.toml
+src/sgn_skig.egg-info/PKG-INFO
+src/sgn_skig.egg-info/SOURCES.txt
+src/sgn_skig.egg-info/dependency_links.txt
+src/sgn_skig.egg-info/entry_points.txt
+src/sgn_skig.egg-info/requires.txt
+src/sgn_skig.egg-info/top_level.txt
+src/sgneskig/__init__.py
+src/sgneskig/_version.py
+src/sgneskig/pipeline.py
+src/sgneskig/bin/__init__.py
+src/sgneskig/bin/grafana_datasource.py
+src/sgneskig/bin/grafana_import.py
+src/sgneskig/metrics/__init__.py
+src/sgneskig/metrics/collector.py
+src/sgneskig/metrics/grafana.py
+src/sgneskig/metrics/writer.py
+src/sgneskig/sinks/__init__.py
+src/sgneskig/sinks/kafka_sink.py
+src/sgneskig/sinks/scald_metrics_sink.py
+src/sgneskig/sources/__init__.py
+src/sgneskig/sources/kafka_source.py
+src/sgneskig/transforms/__init__.py
+src/sgneskig/transforms/delay_buffer.py
+src/sgneskig/transforms/event_latency.py
+src/sgneskig/transforms/round_robin_distributor.py
+tests/__init__.py
+tests/test_bin_scripts.py
+tests/test_collector.py
+tests/test_grafana.py
+tests/test_imports.py
+tests/test_pipeline.py
+tests/test_sinks.py
+tests/test_sources.py
+tests/test_transforms.py
+tests/test_writer.py

sgn_skig-0.1.0/src/sgn_skig.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

sgn_skig-0.1.0/src/sgn_skig.egg-info/entry_points.txt

@@ -0,0 +1,3 @@
+[console_scripts]
+sgn-grafana-datasource = sgneskig.bin.grafana_datasource:main
+sgn-grafana-import = sgneskig.bin.grafana_import:main

sgn_skig-0.1.0/src/sgn_skig.egg-info/requires.txt

@@ -0,0 +1,16 @@
+sgn>=0.5
+confluent-kafka>=2.0
+ligo-scald
+pyyaml
+requests
+
+[dev]
+sgn-skig[lint,test]
+
+[lint]
+ruff
+
+[test]
+pytest
+pytest-cov
+pytest-asyncio

sgn_skig-0.1.0/src/sgn_skig.egg-info/top_level.txt

@@ -0,0 +1 @@
+sgneskig

sgn_skig-0.1.0/src/sgneskig/__init__.py

@@ -0,0 +1,14 @@
+"""SGN-SKIG: SGN infrastructure for Scald, Kafka, InfluxDB, and Grafana.
+
+Provides reusable metrics collection, pipeline infrastructure, and
+generic transforms/sinks/sources for SGN pipelines.
+"""
+
+from sgneskig.pipeline import MetricsPipeline
+
+try:
+    from sgneskig._version import __version__
+except ModuleNotFoundError:
+    __version__ = "0.0.0"
+
+__all__ = ["MetricsPipeline", "__version__"]

sgn_skig-0.1.0/src/sgneskig/_version.py

@@ -0,0 +1,34 @@
+# file generated by setuptools-scm
+# don't change, don't track in version control
+
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
+
+TYPE_CHECKING = False
+if TYPE_CHECKING:
+    from typing import Tuple
+    from typing import Union
+
+    VERSION_TUPLE = Tuple[Union[int, str], ...]
+    COMMIT_ID = Union[str, None]
+else:
+    VERSION_TUPLE = object
+    COMMIT_ID = object
+
+version: str
+__version__: str
+__version_tuple__: VERSION_TUPLE
+version_tuple: VERSION_TUPLE
+commit_id: COMMIT_ID
+__commit_id__: COMMIT_ID
+
+__version__ = version = '0.1.0'
+__version_tuple__ = version_tuple = (0, 1, 0)
+
+__commit_id__ = commit_id = 'gc8f50f74a'

sgn_skig-0.1.0/src/sgneskig/bin/__init__.py

@@ -0,0 +1 @@
+"""Command-line tools for sgn-skig."""