epoch-source-connections 0.1.0a4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (24) hide show
  1. epoch_source_connections-0.1.0a4/.gitignore +196 -0
  2. epoch_source_connections-0.1.0a4/PKG-INFO +24 -0
  3. epoch_source_connections-0.1.0a4/README.md +203 -0
  4. epoch_source_connections-0.1.0a4/notebooks/postgres_airbyte_review.ipynb +627 -0
  5. epoch_source_connections-0.1.0a4/openapi.json +3852 -0
  6. epoch_source_connections-0.1.0a4/pyproject.toml +42 -0
  7. epoch_source_connections-0.1.0a4/src/epoch_sources/__init__.py +3 -0
  8. epoch_source_connections-0.1.0a4/src/epoch_sources/airbyte.py +429 -0
  9. epoch_source_connections-0.1.0a4/src/epoch_sources/auth.py +82 -0
  10. epoch_source_connections-0.1.0a4/src/epoch_sources/client.py +158 -0
  11. epoch_source_connections-0.1.0a4/src/epoch_sources/landing.py +254 -0
  12. epoch_source_connections-0.1.0a4/src/epoch_sources/models.py +201 -0
  13. epoch_source_connections-0.1.0a4/src/epoch_sources/native.py +73 -0
  14. epoch_source_connections-0.1.0a4/src/epoch_sources/py.typed +0 -0
  15. epoch_source_connections-0.1.0a4/src/epoch_sources/server.py +265 -0
  16. epoch_source_connections-0.1.0a4/src/epoch_sources/service.py +611 -0
  17. epoch_source_connections-0.1.0a4/src/epoch_sources/settings.py +73 -0
  18. epoch_source_connections-0.1.0a4/src/epoch_sources/storage.py +207 -0
  19. epoch_source_connections-0.1.0a4/tests/conftest.py +3 -0
  20. epoch_source_connections-0.1.0a4/tests/test_airbyte.py +113 -0
  21. epoch_source_connections-0.1.0a4/tests/test_auth.py +144 -0
  22. epoch_source_connections-0.1.0a4/tests/test_landing.py +171 -0
  23. epoch_source_connections-0.1.0a4/tests/test_settings.py +42 -0
  24. epoch_source_connections-0.1.0a4/tests/test_sources.py +428 -0
@@ -0,0 +1,196 @@
1
+ # ============================================================================
2
+ # Build Directories
3
+ # ============================================================================
4
+ build/
5
+ build-*/
6
+ cmake-build-*/
7
+ out/
8
+ bin/
9
+ lib/
10
+ # infra/{lib,bin} are CDK SOURCE, not build output — re-include past the broad lib//bin/ rules
11
+ !infra/lib/
12
+ !infra/lib/**
13
+ !infra/bin/
14
+ !infra/bin/**
15
+ # market_data/ build artifacts (test runner has its own .gitignore for job_data/market_data/)
16
+ **/job_data/**
17
+ # ============================================================================
18
+ # CMake Generated
19
+ # ============================================================================
20
+ CMakeFiles/
21
+ CMakeCache.txt
22
+ cmake_install.cmake
23
+ Makefile
24
+ *.cmake.in
25
+ compile_commands.json
26
+
27
+ # ============================================================================
28
+ # IDE & Editors
29
+ # ============================================================================
30
+ .idea/
31
+ .vscode/
32
+ *.swp
33
+ *.swo
34
+ *~
35
+ *.png
36
+ .project
37
+ .cproject
38
+ .settings/
39
+ *.code-workspace
40
+ .epoch_storage
41
+
42
+ # ============================================================================
43
+ # OS Generated
44
+ # ============================================================================
45
+ .DS_Store
46
+ .DS_Store?
47
+ ._*
48
+ .Spotlight-V100
49
+ .Trashes
50
+ ehthumbs.db
51
+ Thumbs.db
52
+
53
+ # ============================================================================
54
+ # Dependencies
55
+ # ============================================================================
56
+ _deps/
57
+ vcpkg_installed/
58
+ conan/
59
+ node_modules/
60
+
61
+ # ============================================================================
62
+ # Python (if any scripts)
63
+ # ============================================================================
64
+ __pycache__/
65
+ *.py[cod]
66
+ *$py.class
67
+ .Python
68
+ *.egg-info/
69
+ .eggs/
70
+ venv/
71
+ .venv/
72
+
73
+ # ============================================================================
74
+ # Testing & Coverage
75
+ # ============================================================================
76
+ Testing/
77
+ build-coverage/
78
+ coverage-report
79
+ *.gcno
80
+ *.gcda
81
+ *.gcov
82
+
83
+ # ============================================================================
84
+ # Documentation
85
+ # ============================================================================
86
+ docs/_build/
87
+ docs/html/
88
+ doxygen/
89
+
90
+ # ============================================================================
91
+ # Secrets & Environment
92
+ # ============================================================================
93
+ .env
94
+ .env.local
95
+ .env.*.local
96
+ *.pem
97
+ *.key
98
+ secrets/
99
+
100
+ # ============================================================================
101
+ # Docker
102
+ # ============================================================================
103
+ .docker/
104
+
105
+ # ============================================================================
106
+ # Logs
107
+ # ============================================================================
108
+ *.log
109
+ logs/
110
+
111
+ # ============================================================================
112
+ # Temporary Files
113
+ # ============================================================================
114
+ tmp/
115
+ temp/
116
+ *.tmp
117
+ *.temp
118
+ *.bak
119
+ *.backup
120
+
121
+ # ============================================================================
122
+ # Package specific ignores
123
+ # ============================================================================
124
+ # Data files (too large for git)
125
+ *.parquet
126
+ *.arrow
127
+ *.csv
128
+ !**/test/**/*.csv
129
+
130
+ # Large binary files
131
+ *.o
132
+ *.a
133
+ *.so
134
+ *.dylib
135
+ *.dll
136
+
137
+ # Reference implementations
138
+ references/
139
+
140
+ # Generated/build files
141
+ .cache/
142
+ .clangd
143
+
144
+ # Generated StratifyX audit state. The scripts and source definitions are
145
+ # tracked; these files contain environment-specific IDs or expanded configs.
146
+ /services/stratifyx/docs/plans/data/event_trail_audit.json
147
+ /services/stratifyx/docs/plans/data/shape_defs.json
148
+ /services/stratifyx/docs/plans/data/shapes/
149
+
150
+ # Fern: OpenAPI spec staged from the build dir by scripts/generate_sdk.sh
151
+ fern/openapi/
152
+
153
+ test_output*
154
+ sprints/*
155
+
156
+ #.epoch_storage/
157
+ coverage_report/
158
+ coverage_html/
159
+
160
+ # Vendored vendor SDKs (cloned for porting reference; not part of the build).
161
+ packages/epoch-datasdk/ref/
162
+
163
+ # Generated grammar/EBNF artifacts (source of truth lives in
164
+ # packages/epoch-script/src/core/grammar/ and packages/epoch-script/grammars/).
165
+ grammars/lezer-epochscript/src/epochscript.grammar
166
+ grammars/tree-sitter-epochscript/grammar.js
167
+ /epochscript.ebnf
168
+
169
+ # Claude Code agent working dir — worktrees (embedded git repos), memory,
170
+ # projects, local settings, scratch. None of it belongs in the repo, and a bare
171
+ # `git add -A` would otherwise stage the agent worktrees as broken gitlinks.
172
+ .claude/
173
+ **/.claude/scheduled_tasks.lock
174
+
175
+ # Playwright MCP scratch snapshots
176
+ .playwright-mcp/
177
+
178
+ # ClangBuildAnalyzer capture blob (459M, regenerable via run_timetrace.sh)
179
+ docs/build_perf/measurements/*.capture
180
+
181
+ # Stray clang -ftime-trace artifact (accidental "-o -" output)
182
+ /-.json
183
+
184
+ # Local EpochAI working dir (not part of the backend repo)
185
+ /EpochAI/
186
+
187
+ # Isolated SDK packaging and verification output
188
+ .sdk-dist*/
189
+ .sdk-stage-*/
190
+ sdk/.venv/
191
+ sdk/node_modules
192
+ sdk/.venv312/
193
+ .sdk-installed/
194
+
195
+ # Generated local reports and acceptance artifacts
196
+ /output/
@@ -0,0 +1,24 @@
1
+ Metadata-Version: 2.4
2
+ Name: epoch-source-connections
3
+ Version: 0.1.0a4
4
+ Summary: Typed Epoch source connections over Airbyte and the existing dataset APIs
5
+ Requires-Python: >=3.12
6
+ Requires-Dist: httpx<1,>=0.28
7
+ Requires-Dist: pydantic<3,>=2.11
8
+ Requires-Dist: stratifyx<4,>=3.3.0rc44
9
+ Provides-Extra: dev
10
+ Requires-Dist: boto3-stubs[s3]<2,>=1.43.121; extra == 'dev'
11
+ Requires-Dist: moto[s3]<6,>=5.2.5; extra == 'dev'
12
+ Requires-Dist: pyarrow-stubs==20.0.0.20260819; extra == 'dev'
13
+ Requires-Dist: pyright<2,>=1.1; extra == 'dev'
14
+ Requires-Dist: pytest-asyncio<2,>=1; extra == 'dev'
15
+ Requires-Dist: pytest<10,>=8; extra == 'dev'
16
+ Requires-Dist: ruff<1,>=0.17; extra == 'dev'
17
+ Provides-Extra: server
18
+ Requires-Dist: airbyte-api==1.0.1; extra == 'server'
19
+ Requires-Dist: boto3<2,>=1.43.121; extra == 'server'
20
+ Requires-Dist: fastapi<1,>=0.162; extra == 'server'
21
+ Requires-Dist: pyarrow<27,>=26; extra == 'server'
22
+ Requires-Dist: pydantic-settings<3,>=2.16; extra == 'server'
23
+ Requires-Dist: pyjwt[crypto]<3,>=2.15; extra == 'server'
24
+ Requires-Dist: uvicorn<1,>=0.49; extra == 'server'
@@ -0,0 +1,203 @@
1
+ # Epoch source connections
2
+
3
+ A Python package for PostgreSQL replication through Airbyte into S3. The optional FastAPI
4
+ router mounts in the existing Epoch Python gateway. Airbyte owns database credentials;
5
+ Epoch stores owner-scoped connection metadata and immutable snapshot lineage in private S3.
6
+ The installed native SDK owns file upload, dataset preparation, review and study references.
7
+
8
+ ## Where the demo data comes from and goes
9
+
10
+ The retained demo uses synthetic rows in the local Docker PostgreSQL database
11
+ `epoch_source_demo`, in `research.prices` and `research.precision_probe`. The test
12
+ setup created and seeded that database. Creating an Airbyte **source** registers
13
+ its address and credentials; it does not create a PostgreSQL database.
14
+
15
+ ```text
16
+ Local test PostgreSQL
17
+ → local self-hosted Airbyte reads the selected tables
18
+ → Airbyte's S3 destination writes Parquet into an AWS landing bucket
19
+ → Epoch captures a completed output into its own file storage
20
+ → Epoch prepares a draft → explicit review saves a dataset → research uses it
21
+ ```
22
+
23
+ Airbyte runs on this machine and its operator UI is at `http://localhost:18090`.
24
+ The separate Airbyte Cloud account and its MCP connection are not part of this
25
+ running data path. Embedding Airbyte's setup UI is a later product integration
26
+ decision described below; the local replication engine already runs.
27
+
28
+ There are two meanings of **destination** to distinguish. The test setup
29
+ provisioned the actual S3 bucket in AWS. The Epoch Python service then created
30
+ an Airbyte destination *configuration* pointing to that pre-existing bucket,
31
+ region and connection-specific folder. The configuration lives in Airbyte;
32
+ the Parquet files live in AWS S3. The destination does not choose an arbitrary
33
+ customer bucket: the operator supplies it through server configuration.
34
+
35
+ The notebook/API test and browser test created separate saved connections to
36
+ the same PostgreSQL database. Each has its own Airbyte source, destination and
37
+ connection configuration. They share the landing bucket and write to separate
38
+ folders:
39
+
40
+ ```text
41
+ s3://<landing-bucket>/epoch-sources/<owner-hash>/<connection-hash>/
42
+ <schema>/<table>/<airbyte-sync-id>/<part>.parquet
43
+ ```
44
+
45
+ Epoch's connection records and sync history use a separate metadata bucket.
46
+ Captured input files and accepted datasets use Epoch's existing private file
47
+ storage. An accepted dataset remains available after Airbyte replaces its
48
+ landing output during a later refresh.
49
+
50
+ The review notebook calls the Epoch API at `http://127.0.0.1:19091`; the Python
51
+ source service calls Airbyte behind that API. Its SQL cell is an independent
52
+ read-only comparison with PostgreSQL, not the data transfer into Epoch. The
53
+ notebook reopens existing resources and reuses the saved preparation/review
54
+ identities. `sync()` checks an existing job; `start()` submits a new Airbyte job.
55
+ `prepare()` captures completed output and prepares a draft; it does not start
56
+ another sync. `review()` explicitly saves or discards that draft.
57
+
58
+ ## Product and provider ownership
59
+
60
+ The owner's 28 October 2026 clarification is to use Airbyte-managed connector
61
+ setup and management views. The PostgreSQL form and controls in the local
62
+ integration prototype are not an agreed product design.
63
+
64
+ Airbyte should own connector configuration, authorization and connector
65
+ management. Epoch should associate the authorized connection with the correct
66
+ user/workspace, expose relevant status and available outputs, and provide its
67
+ existing dataset preparation, review and research experience. Preserve the
68
+ working capture and dataset integration while revising that boundary.
69
+
70
+ Before further product UI implementation, brainstorm the complete journey and
71
+ design implications, inspect the existing Figma patterns, and establish a
72
+ reviewable design shared with document/file inputs. Verify the management
73
+ handoff supported by the selected Airbyte deployment, including access scope and
74
+ the return to Epoch. An Airbyte admin console is not automatically an embeddable,
75
+ customer-scoped management surface. Do not silently substitute connector-specific
76
+ Epoch forms if the desired managed view is unavailable.
77
+
78
+ ### Proposed journey for design review
79
+
80
+ The owner selected **embedded Airbyte inside Epoch** for customer setup and
81
+ credential repair. Do not redirect customers to their own Airbyte admin
82
+ workspace as the product flow. The [four-state Figma proposal](https://www.figma.com/design/xkedRdVvBHJ7Z8ACcB77Iv/Epoch---Design-System---Prototype?node-id=8317-24959)
83
+ extends the canonical Your data screen; it remains a design proposal rather
84
+ than an accepted implementation.
85
+
86
+ Keep the existing Data Explorer navigation and its **Your data** context. Saved
87
+ datasets remain the research-ready outputs. Add a contextual entry for saved
88
+ connections and an **Add data** action; do not introduce another primary
89
+ navigation destination for each input technology. A file is a retained input,
90
+ a connection is a reusable route to a source, and a dataset is a saved output
91
+ of preparation. Document extraction will use that same input-to-draft-to-dataset
92
+ relationship when its later milestone is implemented.
93
+
94
+ 1. **Connect:** launch an Airbyte-managed setup flow from Your data. Airbyte
95
+ collects credentials and validates configuration. Cancellation returns to the
96
+ same Epoch context without implying that a connection or dataset was saved.
97
+ 2. **Inspect a connection:** show its name, selected tables, last observed sync
98
+ state, last successful completion and available captures. Configuration and
99
+ credential repair return to the managed provider flow. A manual sync and
100
+ preparation from an existing capture are distinct actions.
101
+ 3. **Prepare and review:** open a completed table capture in the shared dataset
102
+ review experience. Show a bounded preview, physical types, conversion limits,
103
+ origin and capture time. Study use requires explicit time mapping and
104
+ availability metadata. Saving a draft produces a dataset; connecting alone
105
+ does not. Failed or incomplete syncs cannot produce an accepted snapshot.
106
+ 4. **Reuse:** reopen the accepted dataset in Your data and use its exact saved
107
+ reference in research. Show that it is a captured version. A later sync does
108
+ not modify an already accepted dataset or a prior study's input.
109
+
110
+ The proposed states must also cover cancelled setup, expired managed sessions,
111
+ credential repair, sync failure with a previous successful capture, unsupported
112
+ conversion and unordered time indexes. Retain usable saved data while reporting
113
+ current connection problems. Deletion copy must distinguish disconnecting a
114
+ source from deleting its retained captures and datasets.
115
+
116
+ Airbyte's [authentication module documentation](https://docs.airbyte.com/ai-agents/platform/authenticate/build-auth/authentication-module)
117
+ and [official widget repository](https://github.com/airbytehq/airbyte-embedded-widget)
118
+ describe a provider-owned connector setup UI. The current documentation uses the
119
+ Agent Engine service and the widget has commercial terms. The signed-in Cloud
120
+ replication trial has not established entitlement or PostgreSQL/S3 compatibility
121
+ for that module. Treat those as integration prerequisites, not proof that the
122
+ ordinary Cloud admin console can be embedded.
123
+
124
+ The [current Embedded offering](https://airbyte.com/embedded-integrations)
125
+ explicitly covers customer setup inside an application's UI, workspace-per-customer
126
+ isolation and delivery to the operator's S3. Its price is a custom quote based on
127
+ customer count, sync volume and deployment model. The [general pricing page](https://airbyte.com/pricing)
128
+ states that Core is free to self-host. Continue the local development proof with
129
+ Core; obtain Embedded evaluation access and commercial terms before committing
130
+ to a production plan. A regular Cloud subscription or MCP login is not evidence
131
+ of an enabled Embedded entitlement.
132
+ The deployment evaluation should cover fully self-managed Embedded, including
133
+ whether any Airbyte-managed control plane is required. Airbyte's [Embedded
134
+ announcement](https://airbyte.com/blog/announcing-airbyte-embedded) describes
135
+ deployment in the operator's cloud; the [widget license](https://github.com/airbytehq/airbyte-embedded-widget/blob/main/LICENSE.txt)
136
+ still requires commercial Embedded terms. Hosting choice and widget licensing
137
+ are separate decisions.
138
+
139
+ Before implementing the handoff, verify account enablement, supported PostgreSQL
140
+ replication templates, scoped setup/update behavior and the exact completion
141
+ callback. Resolve the provider customer/workspace from the authenticated Epoch
142
+ owner on the server. A browser callback is a notification to verify, not proof
143
+ of ownership. Never issue a shared operator token to a browser or expose the
144
+ prototype's shared Airbyte workspace as a customer management view.
145
+
146
+ Install `.[server,dev]` for development, or the base package for its typed asynchronous
147
+ client and shared Pydantic models. Enable the gateway with `GATEWAY_SOURCES_ENABLED=true`.
148
+ Set `GATEWAY_UPSTREAM_URL` and `EPOCH_SOURCES_NATIVE_URL` to the same native service.
149
+ Configure the `EPOCH_SOURCES_` settings in `epoch_sources/settings.py`; use private environment
150
+ files for credentials. Cognito issuer and app ID are mandatory, including on localhost.
151
+
152
+ New S3 resources use **`us-east-1`**. The source service defaults
153
+ `EPOCH_SOURCES_S3_REGION` to that region; select the intended AWS account/profile
154
+ explicitly when provisioning its buckets. The retained review demo was created
155
+ in the legacy account in `us-west-2`. At the owner's request, its existing buckets,
156
+ Airbyte destination configurations and review data stay there. That fixture is
157
+ not the configuration to copy for new work. The source service configures a
158
+ destination in an existing bucket; it does not provision or migrate S3 storage.
159
+
160
+ The first supported path uses PostgreSQL direct connections, TLS `require` or `disable`,
161
+ explicit schema/table selection, manual full-refresh overwrite and S3 Parquet output.
162
+ Schema/table names currently support ASCII letters, digits and underscores to preserve
163
+ one-to-one S3 path identities. This is not a multi-table transaction snapshot or CDC adapter.
164
+
165
+ Source registration and table discovery are separate from starting a sync. Reopen saved
166
+ connections without querying PostgreSQL. Start each refresh explicitly and retain its request
167
+ identity. Unconfirmed job submissions are never replayed or matched to a guessed latest job.
168
+ The API exposes last-observed status and completion time. Failed/incomplete/cancelled output
169
+ cannot be prepared. The pinned self-managed Airbyte 2.4 API supplies attempt history; this
170
+ version rejects jobs with multiple attempts instead of merging potentially duplicated batches.
171
+
172
+ Snapshot capture verifies the successful job, complete selected-stream row counts, object ETags,
173
+ generation metadata, record sync IDs and Airbyte-reported field changes. It rejects unverified
174
+ generations, changed inventories, duplicate record IDs and reported nulled/truncated fields.
175
+ It preserves the Parquet physical types and embeds lineage in the captured file. PostgreSQL
176
+ to Parquet conversion is performed by Airbyte, and is covered separately by the real-source
177
+ acceptance checks. Missing output for empty tables does not manufacture a schema.
178
+
179
+ The real source check confirmed PostgreSQL dates, finite double precision values, signed
180
+ bigints beyond JavaScript's exact-number range, booleans, microsecond UTC timestamps and nulls.
181
+ It also found that this S3 Parquet destination rounds NUMERIC(30,10) into binary floating
182
+ point without an Airbyte field-change warning. Exact PostgreSQL NUMERIC/DECIMAL is therefore
183
+ unsupported by this slice. Connection and snapshot responses expose this limitation before
184
+ review; the negative precision-probe draft was rejected. Preserving Parquet's physical values
185
+ does not certify a lossless conversion of every PostgreSQL type. No decimal substitution,
186
+ rounding policy or changed source modeling assumption is introduced.
187
+
188
+ Imports retain existing native limits: at most 50 MiB captured input, 1 million rows and
189
+ 256 MiB expanded data. The first capture is copied into Epoch's existing private file store.
190
+ Later refreshes may replace Airbyte landing files; an already captured file and accepted
191
+ dataset remain reusable. Preparation and review are separate, explicit operations. Study
192
+ sources require the native reviewed time mapping and availability metadata.
193
+
194
+ Deleting a connection removes its exactly named Airbyte resources and landing versions.
195
+ Accepted datasets, original captured files and snapshot lineage survive, with existing
196
+ dependency-aware native deletion APIs. Sync history is not pruned automatically.
197
+
198
+ `pytest tests` covers contracts, authentication, owner isolation, write replay boundaries and
199
+ failure injection. These tests use Moto/HTTP fixtures; they do not replace the separate
200
+ PostgreSQL/Airbyte/AWS S3/native service/browser/SDK/MCP acceptance run. The review demo retains
201
+ its notebook and demo accounts until the user's review is complete; failed setup attempts
202
+ and unrelated temporary resources are cleaned separately. Never save credentials in notebooks,
203
+ their outputs or public validation reports.