epoch-source-connections 0.1.0a4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- epoch_source_connections-0.1.0a4/.gitignore +196 -0
- epoch_source_connections-0.1.0a4/PKG-INFO +24 -0
- epoch_source_connections-0.1.0a4/README.md +203 -0
- epoch_source_connections-0.1.0a4/notebooks/postgres_airbyte_review.ipynb +627 -0
- epoch_source_connections-0.1.0a4/openapi.json +3852 -0
- epoch_source_connections-0.1.0a4/pyproject.toml +42 -0
- epoch_source_connections-0.1.0a4/src/epoch_sources/__init__.py +3 -0
- epoch_source_connections-0.1.0a4/src/epoch_sources/airbyte.py +429 -0
- epoch_source_connections-0.1.0a4/src/epoch_sources/auth.py +82 -0
- epoch_source_connections-0.1.0a4/src/epoch_sources/client.py +158 -0
- epoch_source_connections-0.1.0a4/src/epoch_sources/landing.py +254 -0
- epoch_source_connections-0.1.0a4/src/epoch_sources/models.py +201 -0
- epoch_source_connections-0.1.0a4/src/epoch_sources/native.py +73 -0
- epoch_source_connections-0.1.0a4/src/epoch_sources/py.typed +0 -0
- epoch_source_connections-0.1.0a4/src/epoch_sources/server.py +265 -0
- epoch_source_connections-0.1.0a4/src/epoch_sources/service.py +611 -0
- epoch_source_connections-0.1.0a4/src/epoch_sources/settings.py +73 -0
- epoch_source_connections-0.1.0a4/src/epoch_sources/storage.py +207 -0
- epoch_source_connections-0.1.0a4/tests/conftest.py +3 -0
- epoch_source_connections-0.1.0a4/tests/test_airbyte.py +113 -0
- epoch_source_connections-0.1.0a4/tests/test_auth.py +144 -0
- epoch_source_connections-0.1.0a4/tests/test_landing.py +171 -0
- epoch_source_connections-0.1.0a4/tests/test_settings.py +42 -0
- epoch_source_connections-0.1.0a4/tests/test_sources.py +428 -0
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
# ============================================================================
|
|
2
|
+
# Build Directories
|
|
3
|
+
# ============================================================================
|
|
4
|
+
build/
|
|
5
|
+
build-*/
|
|
6
|
+
cmake-build-*/
|
|
7
|
+
out/
|
|
8
|
+
bin/
|
|
9
|
+
lib/
|
|
10
|
+
# infra/{lib,bin} are CDK SOURCE, not build output — re-include past the broad lib//bin/ rules
|
|
11
|
+
!infra/lib/
|
|
12
|
+
!infra/lib/**
|
|
13
|
+
!infra/bin/
|
|
14
|
+
!infra/bin/**
|
|
15
|
+
# market_data/ build artifacts (test runner has its own .gitignore for job_data/market_data/)
|
|
16
|
+
**/job_data/**
|
|
17
|
+
# ============================================================================
|
|
18
|
+
# CMake Generated
|
|
19
|
+
# ============================================================================
|
|
20
|
+
CMakeFiles/
|
|
21
|
+
CMakeCache.txt
|
|
22
|
+
cmake_install.cmake
|
|
23
|
+
Makefile
|
|
24
|
+
*.cmake.in
|
|
25
|
+
compile_commands.json
|
|
26
|
+
|
|
27
|
+
# ============================================================================
|
|
28
|
+
# IDE & Editors
|
|
29
|
+
# ============================================================================
|
|
30
|
+
.idea/
|
|
31
|
+
.vscode/
|
|
32
|
+
*.swp
|
|
33
|
+
*.swo
|
|
34
|
+
*~
|
|
35
|
+
*.png
|
|
36
|
+
.project
|
|
37
|
+
.cproject
|
|
38
|
+
.settings/
|
|
39
|
+
*.code-workspace
|
|
40
|
+
.epoch_storage
|
|
41
|
+
|
|
42
|
+
# ============================================================================
|
|
43
|
+
# OS Generated
|
|
44
|
+
# ============================================================================
|
|
45
|
+
.DS_Store
|
|
46
|
+
.DS_Store?
|
|
47
|
+
._*
|
|
48
|
+
.Spotlight-V100
|
|
49
|
+
.Trashes
|
|
50
|
+
ehthumbs.db
|
|
51
|
+
Thumbs.db
|
|
52
|
+
|
|
53
|
+
# ============================================================================
|
|
54
|
+
# Dependencies
|
|
55
|
+
# ============================================================================
|
|
56
|
+
_deps/
|
|
57
|
+
vcpkg_installed/
|
|
58
|
+
conan/
|
|
59
|
+
node_modules/
|
|
60
|
+
|
|
61
|
+
# ============================================================================
|
|
62
|
+
# Python (if any scripts)
|
|
63
|
+
# ============================================================================
|
|
64
|
+
__pycache__/
|
|
65
|
+
*.py[cod]
|
|
66
|
+
*$py.class
|
|
67
|
+
.Python
|
|
68
|
+
*.egg-info/
|
|
69
|
+
.eggs/
|
|
70
|
+
venv/
|
|
71
|
+
.venv/
|
|
72
|
+
|
|
73
|
+
# ============================================================================
|
|
74
|
+
# Testing & Coverage
|
|
75
|
+
# ============================================================================
|
|
76
|
+
Testing/
|
|
77
|
+
build-coverage/
|
|
78
|
+
coverage-report
|
|
79
|
+
*.gcno
|
|
80
|
+
*.gcda
|
|
81
|
+
*.gcov
|
|
82
|
+
|
|
83
|
+
# ============================================================================
|
|
84
|
+
# Documentation
|
|
85
|
+
# ============================================================================
|
|
86
|
+
docs/_build/
|
|
87
|
+
docs/html/
|
|
88
|
+
doxygen/
|
|
89
|
+
|
|
90
|
+
# ============================================================================
|
|
91
|
+
# Secrets & Environment
|
|
92
|
+
# ============================================================================
|
|
93
|
+
.env
|
|
94
|
+
.env.local
|
|
95
|
+
.env.*.local
|
|
96
|
+
*.pem
|
|
97
|
+
*.key
|
|
98
|
+
secrets/
|
|
99
|
+
|
|
100
|
+
# ============================================================================
|
|
101
|
+
# Docker
|
|
102
|
+
# ============================================================================
|
|
103
|
+
.docker/
|
|
104
|
+
|
|
105
|
+
# ============================================================================
|
|
106
|
+
# Logs
|
|
107
|
+
# ============================================================================
|
|
108
|
+
*.log
|
|
109
|
+
logs/
|
|
110
|
+
|
|
111
|
+
# ============================================================================
|
|
112
|
+
# Temporary Files
|
|
113
|
+
# ============================================================================
|
|
114
|
+
tmp/
|
|
115
|
+
temp/
|
|
116
|
+
*.tmp
|
|
117
|
+
*.temp
|
|
118
|
+
*.bak
|
|
119
|
+
*.backup
|
|
120
|
+
|
|
121
|
+
# ============================================================================
|
|
122
|
+
# Package specific ignores
|
|
123
|
+
# ============================================================================
|
|
124
|
+
# Data files (too large for git)
|
|
125
|
+
*.parquet
|
|
126
|
+
*.arrow
|
|
127
|
+
*.csv
|
|
128
|
+
!**/test/**/*.csv
|
|
129
|
+
|
|
130
|
+
# Large binary files
|
|
131
|
+
*.o
|
|
132
|
+
*.a
|
|
133
|
+
*.so
|
|
134
|
+
*.dylib
|
|
135
|
+
*.dll
|
|
136
|
+
|
|
137
|
+
# Reference implementations
|
|
138
|
+
references/
|
|
139
|
+
|
|
140
|
+
# Generated/build files
|
|
141
|
+
.cache/
|
|
142
|
+
.clangd
|
|
143
|
+
|
|
144
|
+
# Generated StratifyX audit state. The scripts and source definitions are
|
|
145
|
+
# tracked; these files contain environment-specific IDs or expanded configs.
|
|
146
|
+
/services/stratifyx/docs/plans/data/event_trail_audit.json
|
|
147
|
+
/services/stratifyx/docs/plans/data/shape_defs.json
|
|
148
|
+
/services/stratifyx/docs/plans/data/shapes/
|
|
149
|
+
|
|
150
|
+
# Fern: OpenAPI spec staged from the build dir by scripts/generate_sdk.sh
|
|
151
|
+
fern/openapi/
|
|
152
|
+
|
|
153
|
+
test_output*
|
|
154
|
+
sprints/*
|
|
155
|
+
|
|
156
|
+
#.epoch_storage/
|
|
157
|
+
coverage_report/
|
|
158
|
+
coverage_html/
|
|
159
|
+
|
|
160
|
+
# Vendored vendor SDKs (cloned for porting reference; not part of the build).
|
|
161
|
+
packages/epoch-datasdk/ref/
|
|
162
|
+
|
|
163
|
+
# Generated grammar/EBNF artifacts (source of truth lives in
|
|
164
|
+
# packages/epoch-script/src/core/grammar/ and packages/epoch-script/grammars/).
|
|
165
|
+
grammars/lezer-epochscript/src/epochscript.grammar
|
|
166
|
+
grammars/tree-sitter-epochscript/grammar.js
|
|
167
|
+
/epochscript.ebnf
|
|
168
|
+
|
|
169
|
+
# Claude Code agent working dir — worktrees (embedded git repos), memory,
|
|
170
|
+
# projects, local settings, scratch. None of it belongs in the repo, and a bare
|
|
171
|
+
# `git add -A` would otherwise stage the agent worktrees as broken gitlinks.
|
|
172
|
+
.claude/
|
|
173
|
+
**/.claude/scheduled_tasks.lock
|
|
174
|
+
|
|
175
|
+
# Playwright MCP scratch snapshots
|
|
176
|
+
.playwright-mcp/
|
|
177
|
+
|
|
178
|
+
# ClangBuildAnalyzer capture blob (459M, regenerable via run_timetrace.sh)
|
|
179
|
+
docs/build_perf/measurements/*.capture
|
|
180
|
+
|
|
181
|
+
# Stray clang -ftime-trace artifact (accidental "-o -" output)
|
|
182
|
+
/-.json
|
|
183
|
+
|
|
184
|
+
# Local EpochAI working dir (not part of the backend repo)
|
|
185
|
+
/EpochAI/
|
|
186
|
+
|
|
187
|
+
# Isolated SDK packaging and verification output
|
|
188
|
+
.sdk-dist*/
|
|
189
|
+
.sdk-stage-*/
|
|
190
|
+
sdk/.venv/
|
|
191
|
+
sdk/node_modules
|
|
192
|
+
sdk/.venv312/
|
|
193
|
+
.sdk-installed/
|
|
194
|
+
|
|
195
|
+
# Generated local reports and acceptance artifacts
|
|
196
|
+
/output/
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: epoch-source-connections
|
|
3
|
+
Version: 0.1.0a4
|
|
4
|
+
Summary: Typed Epoch source connections over Airbyte and the existing dataset APIs
|
|
5
|
+
Requires-Python: >=3.12
|
|
6
|
+
Requires-Dist: httpx<1,>=0.28
|
|
7
|
+
Requires-Dist: pydantic<3,>=2.11
|
|
8
|
+
Requires-Dist: stratifyx<4,>=3.3.0rc44
|
|
9
|
+
Provides-Extra: dev
|
|
10
|
+
Requires-Dist: boto3-stubs[s3]<2,>=1.43.121; extra == 'dev'
|
|
11
|
+
Requires-Dist: moto[s3]<6,>=5.2.5; extra == 'dev'
|
|
12
|
+
Requires-Dist: pyarrow-stubs==20.0.0.20260819; extra == 'dev'
|
|
13
|
+
Requires-Dist: pyright<2,>=1.1; extra == 'dev'
|
|
14
|
+
Requires-Dist: pytest-asyncio<2,>=1; extra == 'dev'
|
|
15
|
+
Requires-Dist: pytest<10,>=8; extra == 'dev'
|
|
16
|
+
Requires-Dist: ruff<1,>=0.17; extra == 'dev'
|
|
17
|
+
Provides-Extra: server
|
|
18
|
+
Requires-Dist: airbyte-api==1.0.1; extra == 'server'
|
|
19
|
+
Requires-Dist: boto3<2,>=1.43.121; extra == 'server'
|
|
20
|
+
Requires-Dist: fastapi<1,>=0.162; extra == 'server'
|
|
21
|
+
Requires-Dist: pyarrow<27,>=26; extra == 'server'
|
|
22
|
+
Requires-Dist: pydantic-settings<3,>=2.16; extra == 'server'
|
|
23
|
+
Requires-Dist: pyjwt[crypto]<3,>=2.15; extra == 'server'
|
|
24
|
+
Requires-Dist: uvicorn<1,>=0.49; extra == 'server'
|
|
@@ -0,0 +1,203 @@
|
|
|
1
|
+
# Epoch source connections
|
|
2
|
+
|
|
3
|
+
A Python package for PostgreSQL replication through Airbyte into S3. The optional FastAPI
|
|
4
|
+
router mounts in the existing Epoch Python gateway. Airbyte owns database credentials;
|
|
5
|
+
Epoch stores owner-scoped connection metadata and immutable snapshot lineage in private S3.
|
|
6
|
+
The installed native SDK owns file upload, dataset preparation, review and study references.
|
|
7
|
+
|
|
8
|
+
## Where the demo data comes from and goes
|
|
9
|
+
|
|
10
|
+
The retained demo uses synthetic rows in the local Docker PostgreSQL database
|
|
11
|
+
`epoch_source_demo`, in `research.prices` and `research.precision_probe`. The test
|
|
12
|
+
setup created and seeded that database. Creating an Airbyte **source** registers
|
|
13
|
+
its address and credentials; it does not create a PostgreSQL database.
|
|
14
|
+
|
|
15
|
+
```text
|
|
16
|
+
Local test PostgreSQL
|
|
17
|
+
→ local self-hosted Airbyte reads the selected tables
|
|
18
|
+
→ Airbyte's S3 destination writes Parquet into an AWS landing bucket
|
|
19
|
+
→ Epoch captures a completed output into its own file storage
|
|
20
|
+
→ Epoch prepares a draft → explicit review saves a dataset → research uses it
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
Airbyte runs on this machine and its operator UI is at `http://localhost:18090`.
|
|
24
|
+
The separate Airbyte Cloud account and its MCP connection are not part of this
|
|
25
|
+
running data path. Embedding Airbyte's setup UI is a later product integration
|
|
26
|
+
decision described below; the local replication engine already runs.
|
|
27
|
+
|
|
28
|
+
There are two meanings of **destination** to distinguish. The test setup
|
|
29
|
+
provisioned the actual S3 bucket in AWS. The Epoch Python service then created
|
|
30
|
+
an Airbyte destination *configuration* pointing to that pre-existing bucket,
|
|
31
|
+
region and connection-specific folder. The configuration lives in Airbyte;
|
|
32
|
+
the Parquet files live in AWS S3. The destination does not choose an arbitrary
|
|
33
|
+
customer bucket: the operator supplies it through server configuration.
|
|
34
|
+
|
|
35
|
+
The notebook/API test and browser test created separate saved connections to
|
|
36
|
+
the same PostgreSQL database. Each has its own Airbyte source, destination and
|
|
37
|
+
connection configuration. They share the landing bucket and write to separate
|
|
38
|
+
folders:
|
|
39
|
+
|
|
40
|
+
```text
|
|
41
|
+
s3://<landing-bucket>/epoch-sources/<owner-hash>/<connection-hash>/
|
|
42
|
+
<schema>/<table>/<airbyte-sync-id>/<part>.parquet
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
Epoch's connection records and sync history use a separate metadata bucket.
|
|
46
|
+
Captured input files and accepted datasets use Epoch's existing private file
|
|
47
|
+
storage. An accepted dataset remains available after Airbyte replaces its
|
|
48
|
+
landing output during a later refresh.
|
|
49
|
+
|
|
50
|
+
The review notebook calls the Epoch API at `http://127.0.0.1:19091`; the Python
|
|
51
|
+
source service calls Airbyte behind that API. Its SQL cell is an independent
|
|
52
|
+
read-only comparison with PostgreSQL, not the data transfer into Epoch. The
|
|
53
|
+
notebook reopens existing resources and reuses the saved preparation/review
|
|
54
|
+
identities. `sync()` checks an existing job; `start()` submits a new Airbyte job.
|
|
55
|
+
`prepare()` captures completed output and prepares a draft; it does not start
|
|
56
|
+
another sync. `review()` explicitly saves or discards that draft.
|
|
57
|
+
|
|
58
|
+
## Product and provider ownership
|
|
59
|
+
|
|
60
|
+
The owner's 28 October 2026 clarification is to use Airbyte-managed connector
|
|
61
|
+
setup and management views. The PostgreSQL form and controls in the local
|
|
62
|
+
integration prototype are not an agreed product design.
|
|
63
|
+
|
|
64
|
+
Airbyte should own connector configuration, authorization and connector
|
|
65
|
+
management. Epoch should associate the authorized connection with the correct
|
|
66
|
+
user/workspace, expose relevant status and available outputs, and provide its
|
|
67
|
+
existing dataset preparation, review and research experience. Preserve the
|
|
68
|
+
working capture and dataset integration while revising that boundary.
|
|
69
|
+
|
|
70
|
+
Before further product UI implementation, brainstorm the complete journey and
|
|
71
|
+
design implications, inspect the existing Figma patterns, and establish a
|
|
72
|
+
reviewable design shared with document/file inputs. Verify the management
|
|
73
|
+
handoff supported by the selected Airbyte deployment, including access scope and
|
|
74
|
+
the return to Epoch. An Airbyte admin console is not automatically an embeddable,
|
|
75
|
+
customer-scoped management surface. Do not silently substitute connector-specific
|
|
76
|
+
Epoch forms if the desired managed view is unavailable.
|
|
77
|
+
|
|
78
|
+
### Proposed journey for design review
|
|
79
|
+
|
|
80
|
+
The owner selected **embedded Airbyte inside Epoch** for customer setup and
|
|
81
|
+
credential repair. Do not redirect customers to their own Airbyte admin
|
|
82
|
+
workspace as the product flow. The [four-state Figma proposal](https://www.figma.com/design/xkedRdVvBHJ7Z8ACcB77Iv/Epoch---Design-System---Prototype?node-id=8317-24959)
|
|
83
|
+
extends the canonical Your data screen; it remains a design proposal rather
|
|
84
|
+
than an accepted implementation.
|
|
85
|
+
|
|
86
|
+
Keep the existing Data Explorer navigation and its **Your data** context. Saved
|
|
87
|
+
datasets remain the research-ready outputs. Add a contextual entry for saved
|
|
88
|
+
connections and an **Add data** action; do not introduce another primary
|
|
89
|
+
navigation destination for each input technology. A file is a retained input,
|
|
90
|
+
a connection is a reusable route to a source, and a dataset is a saved output
|
|
91
|
+
of preparation. Document extraction will use that same input-to-draft-to-dataset
|
|
92
|
+
relationship when its later milestone is implemented.
|
|
93
|
+
|
|
94
|
+
1. **Connect:** launch an Airbyte-managed setup flow from Your data. Airbyte
|
|
95
|
+
collects credentials and validates configuration. Cancellation returns to the
|
|
96
|
+
same Epoch context without implying that a connection or dataset was saved.
|
|
97
|
+
2. **Inspect a connection:** show its name, selected tables, last observed sync
|
|
98
|
+
state, last successful completion and available captures. Configuration and
|
|
99
|
+
credential repair return to the managed provider flow. A manual sync and
|
|
100
|
+
preparation from an existing capture are distinct actions.
|
|
101
|
+
3. **Prepare and review:** open a completed table capture in the shared dataset
|
|
102
|
+
review experience. Show a bounded preview, physical types, conversion limits,
|
|
103
|
+
origin and capture time. Study use requires explicit time mapping and
|
|
104
|
+
availability metadata. Saving a draft produces a dataset; connecting alone
|
|
105
|
+
does not. Failed or incomplete syncs cannot produce an accepted snapshot.
|
|
106
|
+
4. **Reuse:** reopen the accepted dataset in Your data and use its exact saved
|
|
107
|
+
reference in research. Show that it is a captured version. A later sync does
|
|
108
|
+
not modify an already accepted dataset or a prior study's input.
|
|
109
|
+
|
|
110
|
+
The proposed states must also cover cancelled setup, expired managed sessions,
|
|
111
|
+
credential repair, sync failure with a previous successful capture, unsupported
|
|
112
|
+
conversion and unordered time indexes. Retain usable saved data while reporting
|
|
113
|
+
current connection problems. Deletion copy must distinguish disconnecting a
|
|
114
|
+
source from deleting its retained captures and datasets.
|
|
115
|
+
|
|
116
|
+
Airbyte's [authentication module documentation](https://docs.airbyte.com/ai-agents/platform/authenticate/build-auth/authentication-module)
|
|
117
|
+
and [official widget repository](https://github.com/airbytehq/airbyte-embedded-widget)
|
|
118
|
+
describe a provider-owned connector setup UI. The current documentation uses the
|
|
119
|
+
Agent Engine service and the widget has commercial terms. The signed-in Cloud
|
|
120
|
+
replication trial has not established entitlement or PostgreSQL/S3 compatibility
|
|
121
|
+
for that module. Treat those as integration prerequisites, not proof that the
|
|
122
|
+
ordinary Cloud admin console can be embedded.
|
|
123
|
+
|
|
124
|
+
The [current Embedded offering](https://airbyte.com/embedded-integrations)
|
|
125
|
+
explicitly covers customer setup inside an application's UI, workspace-per-customer
|
|
126
|
+
isolation and delivery to the operator's S3. Its price is a custom quote based on
|
|
127
|
+
customer count, sync volume and deployment model. The [general pricing page](https://airbyte.com/pricing)
|
|
128
|
+
states that Core is free to self-host. Continue the local development proof with
|
|
129
|
+
Core; obtain Embedded evaluation access and commercial terms before committing
|
|
130
|
+
to a production plan. A regular Cloud subscription or MCP login is not evidence
|
|
131
|
+
of an enabled Embedded entitlement.
|
|
132
|
+
The deployment evaluation should cover fully self-managed Embedded, including
|
|
133
|
+
whether any Airbyte-managed control plane is required. Airbyte's [Embedded
|
|
134
|
+
announcement](https://airbyte.com/blog/announcing-airbyte-embedded) describes
|
|
135
|
+
deployment in the operator's cloud; the [widget license](https://github.com/airbytehq/airbyte-embedded-widget/blob/main/LICENSE.txt)
|
|
136
|
+
still requires commercial Embedded terms. Hosting choice and widget licensing
|
|
137
|
+
are separate decisions.
|
|
138
|
+
|
|
139
|
+
Before implementing the handoff, verify account enablement, supported PostgreSQL
|
|
140
|
+
replication templates, scoped setup/update behavior and the exact completion
|
|
141
|
+
callback. Resolve the provider customer/workspace from the authenticated Epoch
|
|
142
|
+
owner on the server. A browser callback is a notification to verify, not proof
|
|
143
|
+
of ownership. Never issue a shared operator token to a browser or expose the
|
|
144
|
+
prototype's shared Airbyte workspace as a customer management view.
|
|
145
|
+
|
|
146
|
+
Install `.[server,dev]` for development, or the base package for its typed asynchronous
|
|
147
|
+
client and shared Pydantic models. Enable the gateway with `GATEWAY_SOURCES_ENABLED=true`.
|
|
148
|
+
Set `GATEWAY_UPSTREAM_URL` and `EPOCH_SOURCES_NATIVE_URL` to the same native service.
|
|
149
|
+
Configure the `EPOCH_SOURCES_` settings in `epoch_sources/settings.py`; use private environment
|
|
150
|
+
files for credentials. Cognito issuer and app ID are mandatory, including on localhost.
|
|
151
|
+
|
|
152
|
+
New S3 resources use **`us-east-1`**. The source service defaults
|
|
153
|
+
`EPOCH_SOURCES_S3_REGION` to that region; select the intended AWS account/profile
|
|
154
|
+
explicitly when provisioning its buckets. The retained review demo was created
|
|
155
|
+
in the legacy account in `us-west-2`. At the owner's request, its existing buckets,
|
|
156
|
+
Airbyte destination configurations and review data stay there. That fixture is
|
|
157
|
+
not the configuration to copy for new work. The source service configures a
|
|
158
|
+
destination in an existing bucket; it does not provision or migrate S3 storage.
|
|
159
|
+
|
|
160
|
+
The first supported path uses PostgreSQL direct connections, TLS `require` or `disable`,
|
|
161
|
+
explicit schema/table selection, manual full-refresh overwrite and S3 Parquet output.
|
|
162
|
+
Schema/table names currently support ASCII letters, digits and underscores to preserve
|
|
163
|
+
one-to-one S3 path identities. This is not a multi-table transaction snapshot or CDC adapter.
|
|
164
|
+
|
|
165
|
+
Source registration and table discovery are separate from starting a sync. Reopen saved
|
|
166
|
+
connections without querying PostgreSQL. Start each refresh explicitly and retain its request
|
|
167
|
+
identity. Unconfirmed job submissions are never replayed or matched to a guessed latest job.
|
|
168
|
+
The API exposes last-observed status and completion time. Failed/incomplete/cancelled output
|
|
169
|
+
cannot be prepared. The pinned self-managed Airbyte 2.4 API supplies attempt history; this
|
|
170
|
+
version rejects jobs with multiple attempts instead of merging potentially duplicated batches.
|
|
171
|
+
|
|
172
|
+
Snapshot capture verifies the successful job, complete selected-stream row counts, object ETags,
|
|
173
|
+
generation metadata, record sync IDs and Airbyte-reported field changes. It rejects unverified
|
|
174
|
+
generations, changed inventories, duplicate record IDs and reported nulled/truncated fields.
|
|
175
|
+
It preserves the Parquet physical types and embeds lineage in the captured file. PostgreSQL
|
|
176
|
+
to Parquet conversion is performed by Airbyte, and is covered separately by the real-source
|
|
177
|
+
acceptance checks. Missing output for empty tables does not manufacture a schema.
|
|
178
|
+
|
|
179
|
+
The real source check confirmed PostgreSQL dates, finite double precision values, signed
|
|
180
|
+
bigints beyond JavaScript's exact-number range, booleans, microsecond UTC timestamps and nulls.
|
|
181
|
+
It also found that this S3 Parquet destination rounds NUMERIC(30,10) into binary floating
|
|
182
|
+
point without an Airbyte field-change warning. Exact PostgreSQL NUMERIC/DECIMAL is therefore
|
|
183
|
+
unsupported by this slice. Connection and snapshot responses expose this limitation before
|
|
184
|
+
review; the negative precision-probe draft was rejected. Preserving Parquet's physical values
|
|
185
|
+
does not certify a lossless conversion of every PostgreSQL type. No decimal substitution,
|
|
186
|
+
rounding policy or changed source modeling assumption is introduced.
|
|
187
|
+
|
|
188
|
+
Imports retain existing native limits: at most 50 MiB captured input, 1 million rows and
|
|
189
|
+
256 MiB expanded data. The first capture is copied into Epoch's existing private file store.
|
|
190
|
+
Later refreshes may replace Airbyte landing files; an already captured file and accepted
|
|
191
|
+
dataset remain reusable. Preparation and review are separate, explicit operations. Study
|
|
192
|
+
sources require the native reviewed time mapping and availability metadata.
|
|
193
|
+
|
|
194
|
+
Deleting a connection removes its exactly named Airbyte resources and landing versions.
|
|
195
|
+
Accepted datasets, original captured files and snapshot lineage survive, with existing
|
|
196
|
+
dependency-aware native deletion APIs. Sync history is not pruned automatically.
|
|
197
|
+
|
|
198
|
+
`pytest tests` covers contracts, authentication, owner isolation, write replay boundaries and
|
|
199
|
+
failure injection. These tests use Moto/HTTP fixtures; they do not replace the separate
|
|
200
|
+
PostgreSQL/Airbyte/AWS S3/native service/browser/SDK/MCP acceptance run. The review demo retains
|
|
201
|
+
its notebook and demo accounts until the user's review is complete; failed setup attempts
|
|
202
|
+
and unrelated temporary resources are cleaned separately. Never save credentials in notebooks,
|
|
203
|
+
their outputs or public validation reports.
|