litelink 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- litelink-0.1.0/.gitignore +38 -0
- litelink-0.1.0/LICENSE +202 -0
- litelink-0.1.0/NOTICE +4 -0
- litelink-0.1.0/PKG-INFO +310 -0
- litelink-0.1.0/README.md +288 -0
- litelink-0.1.0/benchmarks/README.md +42 -0
- litelink-0.1.0/docs/SPEC.md +2748 -0
- litelink-0.1.0/examples/README.md +195 -0
- litelink-0.1.0/hatch_build.py +94 -0
- litelink-0.1.0/pyproject.toml +161 -0
- litelink-0.1.0/scripts/check_blank_lines.py +255 -0
- litelink-0.1.0/scripts/check_commit_msg.py +127 -0
- litelink-0.1.0/scripts/install_duckdb_extensions.py +147 -0
- litelink-0.1.0/scripts/release_notes.py +187 -0
- litelink-0.1.0/scripts/vendor_duckdb_extension.py +191 -0
- litelink-0.1.0/scripts/vendor_litestream.py +144 -0
- litelink-0.1.0/src/litelink/__init__.py +77 -0
- litelink-0.1.0/src/litelink/__main__.py +10 -0
- litelink-0.1.0/src/litelink/_archive.py +213 -0
- litelink-0.1.0/src/litelink/_assembly.py +460 -0
- litelink-0.1.0/src/litelink/_buffer.py +2474 -0
- litelink-0.1.0/src/litelink/_claim.py +218 -0
- litelink-0.1.0/src/litelink/_config.py +322 -0
- litelink-0.1.0/src/litelink/_fs.py +29 -0
- litelink-0.1.0/src/litelink/_layout.py +190 -0
- litelink-0.1.0/src/litelink/_maintenance.py +1495 -0
- litelink-0.1.0/src/litelink/_predicates.py +30 -0
- litelink-0.1.0/src/litelink/_preflight.py +255 -0
- litelink-0.1.0/src/litelink/_read.py +501 -0
- litelink-0.1.0/src/litelink/_replication.py +248 -0
- litelink-0.1.0/src/litelink/_s3.py +81 -0
- litelink-0.1.0/src/litelink/_table.py +1397 -0
- litelink-0.1.0/src/litelink/_types.py +231 -0
- litelink-0.1.0/src/litelink/log.py +3758 -0
- litelink-0.1.0/src/litelink/py.typed +0 -0
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
# Python-generated files
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[oc]
|
|
4
|
+
build/
|
|
5
|
+
dist/
|
|
6
|
+
wheels/
|
|
7
|
+
*.egg-info
|
|
8
|
+
|
|
9
|
+
# Virtual environments
|
|
10
|
+
.venv
|
|
11
|
+
|
|
12
|
+
# Tool caches
|
|
13
|
+
.pytest_cache/
|
|
14
|
+
.ruff_cache/
|
|
15
|
+
.mypy_cache/
|
|
16
|
+
|
|
17
|
+
# Local capture roots: a SQLite buffer, a file:// warehouse, and the two
|
|
18
|
+
# catalog databases (docs/SPEC.md §1). Never commit a database.
|
|
19
|
+
litelink-data/
|
|
20
|
+
litelink-ws/
|
|
21
|
+
*.db
|
|
22
|
+
*.db-wal
|
|
23
|
+
*.db-shm
|
|
24
|
+
|
|
25
|
+
# Fetched binaries: the litestream sidecar (`just litestream`). Not a project
|
|
26
|
+
# dependency — a separate process, pinned rather than vendored.
|
|
27
|
+
.bin/
|
|
28
|
+
|
|
29
|
+
# Working plans. Scaffolding for a change rather than documentation of the
|
|
30
|
+
# system — what survives a plan belongs in SPEC or RUNTIME, and what does not
|
|
31
|
+
# is answered by the commit messages.
|
|
32
|
+
docs/plans/
|
|
33
|
+
|
|
34
|
+
# Claude work artifacts
|
|
35
|
+
.claude/worktrees/
|
|
36
|
+
/.roborev/
|
|
37
|
+
.env
|
|
38
|
+
src/litelink/.bin/
|
litelink-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
|
|
2
|
+
Apache License
|
|
3
|
+
Version 2.0, January 2004
|
|
4
|
+
http://www.apache.org/licenses/
|
|
5
|
+
|
|
6
|
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
|
7
|
+
|
|
8
|
+
1. Definitions.
|
|
9
|
+
|
|
10
|
+
"License" shall mean the terms and conditions for use, reproduction,
|
|
11
|
+
and distribution as defined by Sections 1 through 9 of this document.
|
|
12
|
+
|
|
13
|
+
"Licensor" shall mean the copyright owner or entity authorized by
|
|
14
|
+
the copyright owner that is granting the License.
|
|
15
|
+
|
|
16
|
+
"Legal Entity" shall mean the union of the acting entity and all
|
|
17
|
+
other entities that control, are controlled by, or are under common
|
|
18
|
+
control with that entity. For the purposes of this definition,
|
|
19
|
+
"control" means (i) the power, direct or indirect, to cause the
|
|
20
|
+
direction or management of such entity, whether by contract or
|
|
21
|
+
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
|
22
|
+
outstanding shares, or (iii) beneficial ownership of such entity.
|
|
23
|
+
|
|
24
|
+
"You" (or "Your") shall mean an individual or Legal Entity
|
|
25
|
+
exercising permissions granted by this License.
|
|
26
|
+
|
|
27
|
+
"Source" form shall mean the preferred form for making modifications,
|
|
28
|
+
including but not limited to software source code, documentation
|
|
29
|
+
source, and configuration files.
|
|
30
|
+
|
|
31
|
+
"Object" form shall mean any form resulting from mechanical
|
|
32
|
+
transformation or translation of a Source form, including but
|
|
33
|
+
not limited to compiled object code, generated documentation,
|
|
34
|
+
and conversions to other media types.
|
|
35
|
+
|
|
36
|
+
"Work" shall mean the work of authorship, whether in Source or
|
|
37
|
+
Object form, made available under the License, as indicated by a
|
|
38
|
+
copyright notice that is included in or attached to the work
|
|
39
|
+
(an example is provided in the Appendix below).
|
|
40
|
+
|
|
41
|
+
"Derivative Works" shall mean any work, whether in Source or Object
|
|
42
|
+
form, that is based on (or derived from) the Work and for which the
|
|
43
|
+
editorial revisions, annotations, elaborations, or other modifications
|
|
44
|
+
represent, as a whole, an original work of authorship. For the purposes
|
|
45
|
+
of this License, Derivative Works shall not include works that remain
|
|
46
|
+
separable from, or merely link (or bind by name) to the interfaces of,
|
|
47
|
+
the Work and Derivative Works thereof.
|
|
48
|
+
|
|
49
|
+
"Contribution" shall mean any work of authorship, including
|
|
50
|
+
the original version of the Work and any modifications or additions
|
|
51
|
+
to that Work or Derivative Works thereof, that is intentionally
|
|
52
|
+
submitted to Licensor for inclusion in the Work by the copyright owner
|
|
53
|
+
or by an individual or Legal Entity authorized to submit on behalf of
|
|
54
|
+
the copyright owner. For the purposes of this definition, "submitted"
|
|
55
|
+
means any form of electronic, verbal, or written communication sent
|
|
56
|
+
to the Licensor or its representatives, including but not limited to
|
|
57
|
+
communication on electronic mailing lists, source code control systems,
|
|
58
|
+
and issue tracking systems that are managed by, or on behalf of, the
|
|
59
|
+
Licensor for the purpose of discussing and improving the Work, but
|
|
60
|
+
excluding communication that is conspicuously marked or otherwise
|
|
61
|
+
designated in writing by the copyright owner as "Not a Contribution."
|
|
62
|
+
|
|
63
|
+
"Contributor" shall mean Licensor and any individual or Legal Entity
|
|
64
|
+
on behalf of whom a Contribution has been received by Licensor and
|
|
65
|
+
subsequently incorporated within the Work.
|
|
66
|
+
|
|
67
|
+
2. Grant of Copyright License. Subject to the terms and conditions of
|
|
68
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
69
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
70
|
+
copyright license to reproduce, prepare Derivative Works of,
|
|
71
|
+
publicly display, publicly perform, sublicense, and distribute the
|
|
72
|
+
Work and such Derivative Works in Source or Object form.
|
|
73
|
+
|
|
74
|
+
3. Grant of Patent License. Subject to the terms and conditions of
|
|
75
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
76
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
77
|
+
(except as stated in this section) patent license to make, have made,
|
|
78
|
+
use, offer to sell, sell, import, and otherwise transfer the Work,
|
|
79
|
+
where such license applies only to those patent claims licensable
|
|
80
|
+
by such Contributor that are necessarily infringed by their
|
|
81
|
+
Contribution(s) alone or by combination of their Contribution(s)
|
|
82
|
+
with the Work to which such Contribution(s) was submitted. If You
|
|
83
|
+
institute patent litigation against any entity (including a
|
|
84
|
+
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
|
85
|
+
or a Contribution incorporated within the Work constitutes direct
|
|
86
|
+
or contributory patent infringement, then any patent licenses
|
|
87
|
+
granted to You under this License for that Work shall terminate
|
|
88
|
+
as of the date such litigation is filed.
|
|
89
|
+
|
|
90
|
+
4. Redistribution. You may reproduce and distribute copies of the
|
|
91
|
+
Work or Derivative Works thereof in any medium, with or without
|
|
92
|
+
modifications, and in Source or Object form, provided that You
|
|
93
|
+
meet the following conditions:
|
|
94
|
+
|
|
95
|
+
(a) You must give any other recipients of the Work or
|
|
96
|
+
Derivative Works a copy of this License; and
|
|
97
|
+
|
|
98
|
+
(b) You must cause any modified files to carry prominent notices
|
|
99
|
+
stating that You changed the files; and
|
|
100
|
+
|
|
101
|
+
(c) You must retain, in the Source form of any Derivative Works
|
|
102
|
+
that You distribute, all copyright, patent, trademark, and
|
|
103
|
+
attribution notices from the Source form of the Work,
|
|
104
|
+
excluding those notices that do not pertain to any part of
|
|
105
|
+
the Derivative Works; and
|
|
106
|
+
|
|
107
|
+
(d) If the Work includes a "NOTICE" text file as part of its
|
|
108
|
+
distribution, then any Derivative Works that You distribute must
|
|
109
|
+
include a readable copy of the attribution notices contained
|
|
110
|
+
within such NOTICE file, excluding those notices that do not
|
|
111
|
+
pertain to any part of the Derivative Works, in at least one
|
|
112
|
+
of the following places: within a NOTICE text file distributed
|
|
113
|
+
as part of the Derivative Works; within the Source form or
|
|
114
|
+
documentation, if provided along with the Derivative Works; or,
|
|
115
|
+
within a display generated by the Derivative Works, if and
|
|
116
|
+
wherever such third-party notices normally appear. The contents
|
|
117
|
+
of the NOTICE file are for informational purposes only and
|
|
118
|
+
do not modify the License. You may add Your own attribution
|
|
119
|
+
notices within Derivative Works that You distribute, alongside
|
|
120
|
+
or as an addendum to the NOTICE text from the Work, provided
|
|
121
|
+
that such additional attribution notices cannot be construed
|
|
122
|
+
as modifying the License.
|
|
123
|
+
|
|
124
|
+
You may add Your own copyright statement to Your modifications and
|
|
125
|
+
may provide additional or different license terms and conditions
|
|
126
|
+
for use, reproduction, or distribution of Your modifications, or
|
|
127
|
+
for any such Derivative Works as a whole, provided Your use,
|
|
128
|
+
reproduction, and distribution of the Work otherwise complies with
|
|
129
|
+
the conditions stated in this License.
|
|
130
|
+
|
|
131
|
+
5. Submission of Contributions. Unless You explicitly state otherwise,
|
|
132
|
+
any Contribution intentionally submitted for inclusion in the Work
|
|
133
|
+
by You to the Licensor shall be under the terms and conditions of
|
|
134
|
+
this License, without any additional terms or conditions.
|
|
135
|
+
Notwithstanding the above, nothing herein shall supersede or modify
|
|
136
|
+
the terms of any separate license agreement you may have executed
|
|
137
|
+
with Licensor regarding such Contributions.
|
|
138
|
+
|
|
139
|
+
6. Trademarks. This License does not grant permission to use the trade
|
|
140
|
+
names, trademarks, service marks, or product names of the Licensor,
|
|
141
|
+
except as required for reasonable and customary use in describing the
|
|
142
|
+
origin of the Work and reproducing the content of the NOTICE file.
|
|
143
|
+
|
|
144
|
+
7. Disclaimer of Warranty. Unless required by applicable law or
|
|
145
|
+
agreed to in writing, Licensor provides the Work (and each
|
|
146
|
+
Contributor provides its Contributions) on an "AS IS" BASIS,
|
|
147
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
148
|
+
implied, including, without limitation, any warranties or conditions
|
|
149
|
+
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
|
150
|
+
PARTICULAR PURPOSE. You are solely responsible for determining the
|
|
151
|
+
appropriateness of using or redistributing the Work and assume any
|
|
152
|
+
risks associated with Your exercise of permissions under this License.
|
|
153
|
+
|
|
154
|
+
8. Limitation of Liability. In no event and under no legal theory,
|
|
155
|
+
whether in tort (including negligence), contract, or otherwise,
|
|
156
|
+
unless required by applicable law (such as deliberate and grossly
|
|
157
|
+
negligent acts) or agreed to in writing, shall any Contributor be
|
|
158
|
+
liable to You for damages, including any direct, indirect, special,
|
|
159
|
+
incidental, or consequential damages of any character arising as a
|
|
160
|
+
result of this License or out of the use or inability to use the
|
|
161
|
+
Work (including but not limited to damages for loss of goodwill,
|
|
162
|
+
work stoppage, computer failure or malfunction, or any and all
|
|
163
|
+
other commercial damages or losses), even if such Contributor
|
|
164
|
+
has been advised of the possibility of such damages.
|
|
165
|
+
|
|
166
|
+
9. Accepting Warranty or Additional Liability. While redistributing
|
|
167
|
+
the Work or Derivative Works thereof, You may choose to offer,
|
|
168
|
+
and charge a fee for, acceptance of support, warranty, indemnity,
|
|
169
|
+
or other liability obligations and/or rights consistent with this
|
|
170
|
+
License. However, in accepting such obligations, You may act only
|
|
171
|
+
on Your own behalf and on Your sole responsibility, not on behalf
|
|
172
|
+
of any other Contributor, and only if You agree to indemnify,
|
|
173
|
+
defend, and hold each Contributor harmless for any liability
|
|
174
|
+
incurred by, or claims asserted against, such Contributor by reason
|
|
175
|
+
of your accepting any such warranty or additional liability.
|
|
176
|
+
|
|
177
|
+
END OF TERMS AND CONDITIONS
|
|
178
|
+
|
|
179
|
+
APPENDIX: How to apply the Apache License to your work.
|
|
180
|
+
|
|
181
|
+
To apply the Apache License to your work, attach the following
|
|
182
|
+
boilerplate notice, with the fields enclosed by brackets "[]"
|
|
183
|
+
replaced with your own identifying information. (Don't include
|
|
184
|
+
the brackets!) The text should be enclosed in the appropriate
|
|
185
|
+
comment syntax for the file format. We also recommend that a
|
|
186
|
+
file or class name and description of purpose be included on the
|
|
187
|
+
same "printed page" as the copyright notice for easier
|
|
188
|
+
identification within third-party archives.
|
|
189
|
+
|
|
190
|
+
Copyright [yyyy] [name of copyright owner]
|
|
191
|
+
|
|
192
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
|
193
|
+
you may not use this file except in compliance with the License.
|
|
194
|
+
You may obtain a copy of the License at
|
|
195
|
+
|
|
196
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
|
197
|
+
|
|
198
|
+
Unless required by applicable law or agreed to in writing, software
|
|
199
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
|
200
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
201
|
+
See the License for the specific language governing permissions and
|
|
202
|
+
limitations under the License.
|
litelink-0.1.0/NOTICE
ADDED
litelink-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,310 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: litelink
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Durable append-only capture into Iceberg tables. Embedded and local-first.
|
|
5
|
+
Project-URL: Source, https://github.com/nhobin219/litelink
|
|
6
|
+
Project-URL: Specification, https://github.com/nhobin219/litelink/blob/main/docs/SPEC.md
|
|
7
|
+
Author-email: Nico Bautista Hobin <nico@penca.io>
|
|
8
|
+
License-Expression: Apache-2.0
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
License-File: NOTICE
|
|
11
|
+
Keywords: append-only,capture,iceberg,parquet,sqlite
|
|
12
|
+
Classifier: Development Status :: 3 - Alpha
|
|
13
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
14
|
+
Classifier: Topic :: Database :: Database Engines/Servers
|
|
15
|
+
Classifier: Typing :: Typed
|
|
16
|
+
Requires-Python: <3.14,>=3.11
|
|
17
|
+
Requires-Dist: duckdb<1.5.6,>=1.5.5
|
|
18
|
+
Requires-Dist: pyarrow>=17.0.0
|
|
19
|
+
Requires-Dist: pyiceberg[pyarrow,sql-sqlite]>=0.11.1
|
|
20
|
+
Requires-Dist: xxhash>=3.6.0
|
|
21
|
+
Description-Content-Type: text/markdown
|
|
22
|
+
|
|
23
|
+
<p align="center">
|
|
24
|
+
<img src="docs/assets/litelink-logo.svg" alt="litelink" width="330">
|
|
25
|
+
</p>
|
|
26
|
+
|
|
27
|
+
[](https://github.com/nhobin219/litelink/actions/workflows/ci.yml)
|
|
28
|
+
[](LICENSE)
|
|
29
|
+
[](pyproject.toml)
|
|
30
|
+
[](https://iceberg.apache.org/)
|
|
31
|
+
|
|
32
|
+
# Durable append-only capture into Iceberg tables
|
|
33
|
+
|
|
34
|
+
**Embedded and local-first.**
|
|
35
|
+
|
|
36
|
+
## Introduction
|
|
37
|
+
|
|
38
|
+
litelink is a Python library for the thing every capture pipeline hand-rolls badly: getting a
|
|
39
|
+
stream of observations onto disk durably, into well-sized Parquet, and eventually into object
|
|
40
|
+
storage — without a daemon, a broker, or a catalog service. `append()` returns once the row is
|
|
41
|
+
durable, and a query a moment later sees it.
|
|
42
|
+
|
|
43
|
+
```
|
|
44
|
+
SQLite buffer durable on commit. unsealed rows only.
|
|
45
|
+
│ seal at target_seal_size
|
|
46
|
+
▼
|
|
47
|
+
local Iceberg table a rolling window. reads land here.
|
|
48
|
+
│ sync: upload data files, register into the archive
|
|
49
|
+
▼
|
|
50
|
+
remote Iceberg table full history, on S3.
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
Reads span all three tiers, and the catalog is a SQLite file rather than a service, so **no
|
|
54
|
+
read on the hot path touches the network.** Every other machine reads the archive instead,
|
|
55
|
+
with any Iceberg engine and nothing from litelink.
|
|
56
|
+
|
|
57
|
+
It exists because doing this by hand goes wrong the same way every time: one production
|
|
58
|
+
capture system had 125,884 objects, 62.5% of them under 16 KiB, Parquet files at 2 rows each,
|
|
59
|
+
a compaction routine nothing ever scheduled, and an in-memory buffer a `SIGKILL` emptied.
|
|
60
|
+
Durable capture, file sizing and tiering are each easy alone and nobody's job together.
|
|
61
|
+
|
|
62
|
+
**Status: early.** All three tiers work, and a log survives losing its machine. Read
|
|
63
|
+
[what it is not](#what-it-is-not) and [not implemented yet](#not-implemented-yet) first.
|
|
64
|
+
|
|
65
|
+
## Install
|
|
66
|
+
|
|
67
|
+
```bash
|
|
68
|
+
pip install litelink # or: uv add litelink
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
That is the whole of it. The wheel carries what the library shells out to — a
|
|
72
|
+
checksum-verified litestream, and the DuckDB `iceberg`, `avro` and `httpfs` extensions built
|
|
73
|
+
for the DuckDB it pins — so a machine with no egress, nothing on `PATH` and no DuckDB
|
|
74
|
+
extension cache still reads, writes and restores. Verified that way in CI, not assumed.
|
|
75
|
+
|
|
76
|
+
That costs about 124 MB per wheel, and it buys the failure mode you do not want: a missing
|
|
77
|
+
binary discovered during a failover, or an extension that autoinstalls fine on your laptop
|
|
78
|
+
and cannot on the box that matters.
|
|
79
|
+
|
|
80
|
+
```bash
|
|
81
|
+
python -m litelink # PASS/FAIL per requirement, non-zero exit if not ready
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
Platform wheels are published for Linux and macOS on x86-64 and arm64. Anywhere else, pip
|
|
85
|
+
builds from the sdist — which produces a working pure-Python wheel with no binaries — and you
|
|
86
|
+
supply litestream and the DuckDB extensions yourself; `python -m litelink` tells you which are
|
|
87
|
+
missing and how to get them.
|
|
88
|
+
|
|
89
|
+
## Quick start
|
|
90
|
+
|
|
91
|
+
```bash
|
|
92
|
+
git clone https://github.com/nhobin219/litelink && cd litelink
|
|
93
|
+
just bootstrap # uv sync + git hooks + DuckDB extensions
|
|
94
|
+
just demo-websocket # capture a live public feed, one process, ~30 seconds
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
For working ON litelink you need [`uv`](https://docs.astral.sh/uv/) and
|
|
98
|
+
[`just`](https://github.com/casey/just); for using it you need neither. Either way the demo
|
|
99
|
+
needs nothing else: no producer, no credentials, no maintainer, no container. That demo is
|
|
100
|
+
[`examples/websocket.py`](examples/websocket.py), and this is its shape:
|
|
101
|
+
|
|
102
|
+
```python
|
|
103
|
+
import litelink
|
|
104
|
+
import pyarrow as pa
|
|
105
|
+
|
|
106
|
+
# A trade feed: durable the moment it arrives, queryable a moment later.
|
|
107
|
+
schema = pa.schema([
|
|
108
|
+
pa.field("trade_id", pa.int64()),
|
|
109
|
+
pa.field("event_ts", pa.int64()), # microseconds, as the exchange sends them
|
|
110
|
+
pa.field("price", pa.float64()),
|
|
111
|
+
pa.field("amount", pa.float64()),
|
|
112
|
+
pa.field("side", pa.int64()), # 0 buy, 1 sell
|
|
113
|
+
])
|
|
114
|
+
|
|
115
|
+
# new() takes the shape, fixed at creation. open() takes none of it — schema,
|
|
116
|
+
# sort order, config and archive all come from the log itself.
|
|
117
|
+
log = litelink.new("data", "trades", schema=schema, sort_by=("event_ts",))
|
|
118
|
+
log.append({"trade_id": 624438572, "event_ts": 1787772776240000,
|
|
119
|
+
"price": 78501.62, "amount": 0.0076, "side": 0}) # durable on return
|
|
120
|
+
|
|
121
|
+
# extend() commits the whole group in ONE transaction — one fsync for the batch,
|
|
122
|
+
# not one per row. That call size is the write throughput lever, and a call-site
|
|
123
|
+
# choice: no LogConfig setting tunes it.
|
|
124
|
+
log.extend(group_of_rows) # append(row) is extend([row])
|
|
125
|
+
|
|
126
|
+
log = litelink.open("data", "trades")
|
|
127
|
+
reader = litelink.open("data", "trades", read_only=True) # alongside a live writer, no write surface
|
|
128
|
+
|
|
129
|
+
recent = log.scan(where="event_ts > 1787772776000000").read_all()
|
|
130
|
+
log.maintain() # compact, evict, expire
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
That is the whole API surface for local capture. Everything below is optional, and every
|
|
134
|
+
public call is in [`docs/API.md`](docs/API.md) — one page, forty-one of them, and most
|
|
135
|
+
deployments use six.
|
|
136
|
+
|
|
137
|
+
## More demos
|
|
138
|
+
|
|
139
|
+
A synthetic feed you can drive as hard as you like, with one process per storage role:
|
|
140
|
+
|
|
141
|
+
```bash
|
|
142
|
+
just demo-capture # append continuously — the hot path, and nothing else
|
|
143
|
+
just demo-maintain # in another terminal: seal, compact, evict, expire
|
|
144
|
+
just demo-tail # in a third: watch where the rows are
|
|
145
|
+
```
|
|
146
|
+
|
|
147
|
+
To add the archive tier, against a local S3-compatible store or a real bucket:
|
|
148
|
+
|
|
149
|
+
```bash
|
|
150
|
+
just rustfs # object storage in one container
|
|
151
|
+
just demo-archive # capture, with an archive configured
|
|
152
|
+
just demo-maintain # also pushes to it, and evicts what it has pushed
|
|
153
|
+
|
|
154
|
+
cp .env.example .env # or: set LITELINK_DEMO_ARCHIVE=s3://your-bucket/prefix
|
|
155
|
+
```
|
|
156
|
+
|
|
157
|
+
Credentials are not in that file unless you put them there — the library reads them from the
|
|
158
|
+
environment through the ordinary AWS chain, so a profile, instance metadata or SSO all work
|
|
159
|
+
untouched, and a log directory never carries a key with it.
|
|
160
|
+
|
|
161
|
+
To survive losing the machine, ship the SQLite WAL alongside:
|
|
162
|
+
|
|
163
|
+
```bash
|
|
164
|
+
just demo-replicate # generates litestream.yml from the log, runs it
|
|
165
|
+
```
|
|
166
|
+
|
|
167
|
+
`litelink.restore(root, name, archive=...)` then rebuilds the log on another box, reserving an
|
|
168
|
+
offset window so nothing the dead machine served is reissued. Verified against a local
|
|
169
|
+
S3-compatible store and against AWS. See [`examples/`](examples/).
|
|
170
|
+
|
|
171
|
+
### Run the sidecar as its own process
|
|
172
|
+
|
|
173
|
+
**litelink emits the config; your supervisor runs the binary.** `replication_config()` writes
|
|
174
|
+
a `litestream.yml` describing which databases carry the log's state, and systemd, Kubernetes
|
|
175
|
+
or anything else runs `litestream replicate -config …` beside the writer. litelink never
|
|
176
|
+
starts it and never supervises it, and that is a design commitment rather than an omission:
|
|
177
|
+
|
|
178
|
+
- **It keeps the network out of the write path.** A sidecar reads the WAL from outside the
|
|
179
|
+
process. If litelink owned the replicator, a stalled upload could push back on `append`,
|
|
180
|
+
and "durable when it returns, with no network in the write path" is the property the whole
|
|
181
|
+
design rests on.
|
|
182
|
+
- **The lifecycles are the wrong way round otherwise.** A writer that owns its replicator
|
|
183
|
+
kills it at exactly the moment you need the last frames shipped. A separate process keeps
|
|
184
|
+
draining what is already on disk.
|
|
185
|
+
- **§1 allows one writer per log.** Two `Log` handles each spawning a replicator would run
|
|
186
|
+
two litestream instances against one database, which litestream forbids.
|
|
187
|
+
|
|
188
|
+
litelink *does* run litestream in one place — `restore`, which shells out to it once and waits.
|
|
189
|
+
That is a batch call, not a supervised daemon, and it is why the binary ships in the wheel
|
|
190
|
+
rather than being left to PATH: the alternative is discovering it is missing during a
|
|
191
|
+
failover.
|
|
192
|
+
|
|
193
|
+
Check the machine before you need it:
|
|
194
|
+
|
|
195
|
+
```bash
|
|
196
|
+
python -m litelink # extensions, litestream
|
|
197
|
+
python -m litelink s3://bucket/prefix trades # ...and the archive is reachable
|
|
198
|
+
```
|
|
199
|
+
|
|
200
|
+
Run that as the user and from the process manager that will own the log. A systemd user unit
|
|
201
|
+
does not inherit a login shell's PATH, so `which litestream` succeeding in your terminal says
|
|
202
|
+
nothing about the unit that will actually perform the restore.
|
|
203
|
+
|
|
204
|
+
## Reading it from another machine
|
|
205
|
+
|
|
206
|
+
The demos above are the *writer's* read. Everywhere else reads the archive, which is an
|
|
207
|
+
ordinary Iceberg table publishing `version-hint.text` at every commit — so an engine pointed
|
|
208
|
+
at the prefix resolves the current metadata itself, with no catalog service, no `archive.db`,
|
|
209
|
+
no local root and no litelink install:
|
|
210
|
+
|
|
211
|
+
```sql
|
|
212
|
+
SELECT count(*), max(litelink_offset)
|
|
213
|
+
FROM iceberg_scan('s3://bucket/prefix/litelink/trades',
|
|
214
|
+
version_name_format = '%s%s.metadata.json');
|
|
215
|
+
```
|
|
216
|
+
|
|
217
|
+
`litelink_offset` is monotonic and never reused, so a reader keeps the highest one it has seen
|
|
218
|
+
and asks for what came after — which is how you follow an archive that `sync` is publishing
|
|
219
|
+
into. The extensions, the credential shapes, why `version_name_format` is not optional, and
|
|
220
|
+
the polling pattern in full are in [`docs/API.md`](docs/API.md).
|
|
221
|
+
|
|
222
|
+
That read is only as fresh as the last `sync`. When you have a WAL sidecar running
|
|
223
|
+
(`wal_replication`), `litelink.follow` does better: it restores the writer's buffer alongside the
|
|
224
|
+
archive and merges them, so the reader sees down to the replication lag instead.
|
|
225
|
+
|
|
226
|
+
```python
|
|
227
|
+
with litelink.follow("trades", archive="s3://bucket/prefix", s3=opts) as reader:
|
|
228
|
+
reader.coverage() # Coverage(archive=(1, 1928), buffered=(1929, 2100), gap=None, ...)
|
|
229
|
+
reader.scan(where="side = 0", columns=["event_ts", "price"])
|
|
230
|
+
```
|
|
231
|
+
|
|
232
|
+
It never writes anything the primary shares and cannot append — a `LogHandle` has no write
|
|
233
|
+
surface at all, rather than one that raises. It is a **snapshot, not a subscription**:
|
|
234
|
+
refreshing means assembling another one, and the root it builds is a temporary directory
|
|
235
|
+
removed on close. `coverage()` is how it stays honest about what it can and cannot serve.
|
|
236
|
+
See [§3b](docs/SPEC.md) for why a gap in that report is not necessarily loss.
|
|
237
|
+
|
|
238
|
+
## How it works
|
|
239
|
+
|
|
240
|
+
- **Iceberg is used, not reimplemented.** Manifests, per-file column statistics, schema with
|
|
241
|
+
field IDs, and atomic snapshot commits all come from it.
|
|
242
|
+
- **The library owns exactly one column**, `litelink_offset` — monotonic, never reused. It is
|
|
243
|
+
the boundary mechanism between tiers. Everything else is the caller's schema.
|
|
244
|
+
- **Sync is a watermark, not CDC.** There are no updates or deletes to replicate.
|
|
245
|
+
- **Parts are sealed once and never rewritten.** Rewriting a growing partition costs ~144x
|
|
246
|
+
write amplification and buys nothing, because the local WAL already made the row durable.
|
|
247
|
+
- **Read boundaries are derived from committed table state**, never from a stored flag — so
|
|
248
|
+
no seal window can double-count or drop.
|
|
249
|
+
- **The seal cut is chosen by the appender**, in the transaction that crosses
|
|
250
|
+
`target_seal_size`, and queued. A sealer that falls behind therefore writes several
|
|
251
|
+
correctly-sized files rather than one oversized one.
|
|
252
|
+
- **Sizing is two targets, not one.** A seal wants to be small, because the buffer is what a
|
|
253
|
+
hot read scans; a file wants to be large, because per-file overhead dominates scans and
|
|
254
|
+
uploads. Compaction bridges them, on local disk, at 8× the seal size by default.
|
|
255
|
+
|
|
256
|
+
Read performance is the cost of reading Parquet, plus ~4 ms of fixed overhead. The numbers
|
|
257
|
+
behind all of this are [`docs/SPEC.md`](docs/SPEC.md) §7 and §12; how the pieces run is
|
|
258
|
+
[`docs/RUNTIME.md`](docs/RUNTIME.md); `just bench` is the same measurement on your hardware.
|
|
259
|
+
|
|
260
|
+
## What it is not
|
|
261
|
+
|
|
262
|
+
An OLTP or key-value store. A point lookup is ~1,600x slower than an indexed row store, and no
|
|
263
|
+
configuration closes that gap. It is a **local, in-process, real-time analytics store**: data
|
|
264
|
+
is durable at commit and queryable immediately, so freshness is sub-second *with* durability —
|
|
265
|
+
but "real-time" means fresh, not point-lookup fast.
|
|
266
|
+
|
|
267
|
+
Nor is it an unbounded local archive. **Keeping everything on one machine — `archive=None`
|
|
268
|
+
with no `local_retention` — degrades as the table grows.** A seal's cost tracks what the
|
|
269
|
+
table's metadata holds, and a residue grows with the file count: compaction never revisits a
|
|
270
|
+
file already at the target size, and only eviction removes one. With no retention set nothing
|
|
271
|
+
evicts, and the seal is on the write path, so the cost lands on appends. Configure a
|
|
272
|
+
retention, or an archive to evict into — [`docs/SPEC.md`](docs/SPEC.md) §13.7.
|
|
273
|
+
|
|
274
|
+
## Not implemented yet
|
|
275
|
+
|
|
276
|
+
**Schema evolution** ([`docs/SPEC.md`](docs/SPEC.md) §9) and **blob fields** (§15) are
|
|
277
|
+
specified and unbuilt, and are what the code lacks against its own design. `add_column`,
|
|
278
|
+
`rename_column` and `drop_column` exist and raise; `binary` columns are refused outright,
|
|
279
|
+
because §15 has large payloads bypass the buffer rather than travel through it.
|
|
280
|
+
|
|
281
|
+
Still open: payload encoding, local-disk backpressure, bulk ingest, and extension provisioning
|
|
282
|
+
for embedders. All four in [`docs/SPEC.md`](docs/SPEC.md) §13.
|
|
283
|
+
|
|
284
|
+
## Documentation
|
|
285
|
+
|
|
286
|
+
- [`docs/API.md`](docs/API.md) — every public call, on one page
|
|
287
|
+
- [`docs/SPEC.md`](docs/SPEC.md) — the design, and in places still ahead of the code
|
|
288
|
+
- [`docs/RUNTIME.md`](docs/RUNTIME.md) — writer and maintainer, threads, processes, what crosses between them
|
|
289
|
+
- [`examples/`](examples/) — the websocket capture, and the synthetic feed with one process per role
|
|
290
|
+
- [`benchmarks/`](benchmarks/) — the harness, including what litelink costs over raw SQLite
|
|
291
|
+
- [`CONTRIBUTING.md`](CONTRIBUTING.md) — setup, the gates, and what a good PR here looks like
|
|
292
|
+
- [`SECURITY.md`](SECURITY.md) — what to report privately, and what is a known limit instead
|
|
293
|
+
|
|
294
|
+
## Development
|
|
295
|
+
|
|
296
|
+
```bash
|
|
297
|
+
just bootstrap # uv sync + git hooks + DuckDB extensions
|
|
298
|
+
just check # lint + format-check + typecheck + tests, same as CI
|
|
299
|
+
just --list # the rest
|
|
300
|
+
```
|
|
301
|
+
|
|
302
|
+
`just bootstrap` provisions the `iceberg`, `avro` and `httpfs` DuckDB extensions, which are
|
|
303
|
+
downloaded rather than bundled — see [`docs/SPEC.md`](docs/SPEC.md) §7. Tooling is uv + ruff +
|
|
304
|
+
[ty](https://github.com/astral-sh/ty) + pytest. Commits follow
|
|
305
|
+
[Conventional Commits](https://www.conventionalcommits.org), enforced by a `commit-msg` hook;
|
|
306
|
+
[`CONTRIBUTING.md`](CONTRIBUTING.md) has the types, scopes and style gates.
|
|
307
|
+
|
|
308
|
+
## License
|
|
309
|
+
|
|
310
|
+
Apache License 2.0 — see [LICENSE](LICENSE) and [NOTICE](NOTICE).
|