lodestream 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- lodestream-0.1.0/.gitignore +16 -0
- lodestream-0.1.0/CHANGELOG.md +11 -0
- lodestream-0.1.0/LICENSE +21 -0
- lodestream-0.1.0/PKG-INFO +217 -0
- lodestream-0.1.0/README.md +165 -0
- lodestream-0.1.0/pyproject.toml +76 -0
- lodestream-0.1.0/src/lodestream/__init__.py +19 -0
- lodestream-0.1.0/src/lodestream/cli.py +138 -0
- lodestream-0.1.0/src/lodestream/config.py +54 -0
- lodestream-0.1.0/src/lodestream/examples/cmis_to_filenet.yaml +34 -0
- lodestream-0.1.0/src/lodestream/examples/csv_to_filenet.yaml +34 -0
- lodestream-0.1.0/src/lodestream/examples/fileshare_to_filenet.yaml +27 -0
- lodestream-0.1.0/src/lodestream/examples/s3_to_filenet.yaml +25 -0
- lodestream-0.1.0/src/lodestream/examples/sql_to_filenet.yaml +40 -0
- lodestream-0.1.0/src/lodestream/examples/wss_to_filenet.yaml +57 -0
- lodestream-0.1.0/src/lodestream/manifest.py +270 -0
- lodestream-0.1.0/src/lodestream/mapping.py +97 -0
- lodestream-0.1.0/src/lodestream/models.py +64 -0
- lodestream-0.1.0/src/lodestream/pipeline.py +210 -0
- lodestream-0.1.0/src/lodestream/py.typed +0 -0
- lodestream-0.1.0/src/lodestream/registry.py +36 -0
- lodestream-0.1.0/src/lodestream/sinks/__init__.py +3 -0
- lodestream-0.1.0/src/lodestream/sinks/base.py +63 -0
- lodestream-0.1.0/src/lodestream/sinks/filenet_graphql.py +343 -0
- lodestream-0.1.0/src/lodestream/sinks/local_folder.py +76 -0
- lodestream-0.1.0/src/lodestream/sources/__init__.py +3 -0
- lodestream-0.1.0/src/lodestream/sources/base.py +51 -0
- lodestream-0.1.0/src/lodestream/sources/cmis.py +122 -0
- lodestream-0.1.0/src/lodestream/sources/csv_manifest.py +85 -0
- lodestream-0.1.0/src/lodestream/sources/filesystem.py +76 -0
- lodestream-0.1.0/src/lodestream/sources/s3.py +84 -0
- lodestream-0.1.0/src/lodestream/sources/sql_blob.py +235 -0
- lodestream-0.1.0/src/lodestream/sources/wss_sql.py +100 -0
- lodestream-0.1.0/src/lodestream/stages/__init__.py +3 -0
- lodestream-0.1.0/src/lodestream/stages/base.py +17 -0
- lodestream-0.1.0/src/lodestream/stages/extract.py +72 -0
- lodestream-0.1.0/src/lodestream/stages/filter.py +71 -0
- lodestream-0.1.0/src/lodestream/stages/inventory.py +78 -0
- lodestream-0.1.0/src/lodestream/stages/load.py +189 -0
- lodestream-0.1.0/src/lodestream/stages/verify.py +64 -0
- lodestream-0.1.0/src/lodestream/utils.py +215 -0
- lodestream-0.1.0/tests/test_lodestream.py +342 -0
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
## 0.1.0 (2026-11-18)
|
|
4
|
+
|
|
5
|
+
First release.
|
|
6
|
+
|
|
7
|
+
- Steps: `inventory`, `filter`, `extract`, `load`, `verify`, `delta`. Choose them per run with `--steps` or `Migration.run(steps=[...])`; `rolling` mode runs extract/load/verify in chunks.
|
|
8
|
+
- Sources: `wss_sql` (WSS 3.0 content DB), `sql_blob` (any DB with BLOBs), `filesystem`, `csv_manifest`, `s3`, `cmis`.
|
|
9
|
+
- Sinks: `filenet_graphql` (batched Content Services GraphQL), `local_folder`.
|
|
10
|
+
- SQLite manifest makes runs restartable; failed items retry on later runs; reconciliation report.
|
|
11
|
+
- CLI: `init`, `steps`, `run`, `status`, `estimate`, `retry`.
|
lodestream-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Karthikkeyan
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,217 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: lodestream
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Bulk content migration from any source (WSS/SharePoint, file shares, SQL BLOBs, CSV exports, S3, CMIS) into IBM FileNet: pluggable, restartable, built for 100 TB+.
|
|
5
|
+
Project-URL: Homepage, https://github.com/karthikbd/lodestream
|
|
6
|
+
Project-URL: Source, https://github.com/karthikbd/lodestream
|
|
7
|
+
Project-URL: Issues, https://github.com/karthikbd/lodestream/issues
|
|
8
|
+
Project-URL: Changelog, https://github.com/karthikbd/lodestream/blob/main/CHANGELOG.md
|
|
9
|
+
Author: Karthikkeyan
|
|
10
|
+
License: MIT
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Keywords: bulk-import,cmis,content-management,ecm,etl,filenet,migration,sharepoint,wss
|
|
13
|
+
Classifier: Development Status :: 4 - Beta
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: Intended Audience :: System Administrators
|
|
16
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
17
|
+
Classifier: Operating System :: OS Independent
|
|
18
|
+
Classifier: Programming Language :: Python :: 3
|
|
19
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
23
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
24
|
+
Classifier: Topic :: Office/Business
|
|
25
|
+
Classifier: Topic :: System :: Archiving
|
|
26
|
+
Classifier: Topic :: Utilities
|
|
27
|
+
Classifier: Typing :: Typed
|
|
28
|
+
Requires-Python: >=3.10
|
|
29
|
+
Requires-Dist: pyyaml>=5.4
|
|
30
|
+
Provides-Extra: all
|
|
31
|
+
Requires-Dist: boto3>=1.26; extra == 'all'
|
|
32
|
+
Requires-Dist: pyodbc>=4.0; extra == 'all'
|
|
33
|
+
Requires-Dist: requests>=2.28; extra == 'all'
|
|
34
|
+
Provides-Extra: cmis
|
|
35
|
+
Requires-Dist: requests>=2.28; extra == 'cmis'
|
|
36
|
+
Provides-Extra: dev
|
|
37
|
+
Requires-Dist: build>=1.0; extra == 'dev'
|
|
38
|
+
Requires-Dist: mypy>=1.8; extra == 'dev'
|
|
39
|
+
Requires-Dist: pytest>=8; extra == 'dev'
|
|
40
|
+
Requires-Dist: requests>=2.28; extra == 'dev'
|
|
41
|
+
Requires-Dist: ruff>=0.4; extra == 'dev'
|
|
42
|
+
Requires-Dist: twine>=5.0; extra == 'dev'
|
|
43
|
+
Provides-Extra: filenet
|
|
44
|
+
Requires-Dist: requests>=2.28; extra == 'filenet'
|
|
45
|
+
Provides-Extra: s3
|
|
46
|
+
Requires-Dist: boto3>=1.26; extra == 's3'
|
|
47
|
+
Provides-Extra: sql
|
|
48
|
+
Requires-Dist: pyodbc>=4.0; extra == 'sql'
|
|
49
|
+
Provides-Extra: wss
|
|
50
|
+
Requires-Dist: pyodbc>=4.0; extra == 'wss'
|
|
51
|
+
Description-Content-Type: text/markdown
|
|
52
|
+
|
|
53
|
+
# lodestream
|
|
54
|
+
|
|
55
|
+
[](https://pypi.org/project/lodestream/)
|
|
56
|
+
[](https://pypi.org/project/lodestream/)
|
|
57
|
+
[](https://github.com/karthikbd/lodestream/actions/workflows/tests.yml)
|
|
58
|
+
[](LICENSE)
|
|
59
|
+
|
|
60
|
+
**Bulk content migration from any source into IBM FileNet.** Each step can be chosen at run time, the whole run is restartable, and it is built for very large volumes (designed for 100 TB+ migrations).
|
|
61
|
+
|
|
62
|
+
```
|
|
63
|
+
SOURCE (pluggable) STEPS (pick any, in any order) SINK (pluggable)
|
|
64
|
+
─────────────────── ─────────────────────────────────────────── ─────────────────
|
|
65
|
+
wss_sql (WSS/SharePoint DB) inventory → filter → [extract] → load → verify filenet_graphql
|
|
66
|
+
sql_blob (any DB with BLOBs) ▲ │ local_folder (dry runs)
|
|
67
|
+
filesystem (NFS/SMB/local) └──────── delta (cutover catch-up) ┘ your own plugin
|
|
68
|
+
csv_manifest (export from ANY system)
|
|
69
|
+
s3 (S3 / MinIO / StorageGRID) MANIFEST (SQLite): one row per document version
|
|
70
|
+
cmis (Alfresco, Documentum, OpenText…) PENDING → EXTRACTED → LOADED → VERIFIED
|
|
71
|
+
your own plugin (+ SKIPPED / FAILED / MISMATCH)
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
## Install
|
|
75
|
+
|
|
76
|
+
```bash
|
|
77
|
+
pip install ./lodestream[all] # or pick extras: [filenet] [wss] [sql] [s3] [cmis]
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
## Quick start
|
|
81
|
+
|
|
82
|
+
```bash
|
|
83
|
+
lodestream init wss_to_filenet -o migration.yaml # templates: wss / fileshare / csv / cmis / s3 / sql
|
|
84
|
+
# edit migration.yaml: connection strings, FileNet endpoint, class + property mapping
|
|
85
|
+
lodestream run -c migration.yaml --dry-run --rate 300 --gbps 10 # inventory + filter only, then an estimate
|
|
86
|
+
lodestream run -c migration.yaml # full migration
|
|
87
|
+
lodestream status -c migration.yaml
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
## Choosing steps dynamically
|
|
91
|
+
|
|
92
|
+
The `steps:` list in the config is the default plan. You can override it per run:
|
|
93
|
+
|
|
94
|
+
```bash
|
|
95
|
+
lodestream run -c migration.yaml --steps inventory,filter # scope the migration first
|
|
96
|
+
lodestream run -c migration.yaml --steps extract,load,verify # bulk move
|
|
97
|
+
lodestream run -c migration.yaml --steps delta,filter,load,verify # final catch-up before cutover
|
|
98
|
+
lodestream retry -c migration.yaml && lodestream run -c migration.yaml --steps load,verify
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
Or from Python:
|
|
102
|
+
|
|
103
|
+
```python
|
|
104
|
+
from lodestream import Migration
|
|
105
|
+
m = Migration.from_file("migration.yaml")
|
|
106
|
+
m.run(steps=["inventory", "filter"])
|
|
107
|
+
print(m.estimate(docs_per_sec=400, link_gbps=10))
|
|
108
|
+
Migration.from_file("migration.yaml").run(steps=["load", "verify"])
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
| Step | What it does | Key options |
|
|
112
|
+
|---|---|---|
|
|
113
|
+
| `inventory` | Scans the source and registers every document version as PENDING. Partitions are scanned in parallel. | `workers`, `batch_size` |
|
|
114
|
+
| `filter` | Marks items you don't want as SKIPPED: old versions, junk paths, size, date and type. | `versions: all/latest/last_n`, `keep_versions`, `exclude_patterns`, `min_size`, `modified_after` |
|
|
115
|
+
| `extract` | *Optional.* Copies content to fast staging storage with SHA-256 checksums. Leave it out to stream content straight from the source into FileNet. | `staging_dir`, `workers`, `max_staged_bytes` |
|
|
116
|
+
| `load` | Creates documents in parallel, many per API call. Each document's versions load oldest to newest. | `workers`, `batch_size`, `max_batch_mb`, `max_attempts`, `retry_failed` |
|
|
117
|
+
| `verify` | Checks every loaded item in the target, cleans up staging, and writes a reconciliation report. | `workers`, `cleanup_staging`, `report` |
|
|
118
|
+
| `delta` | Re-scans only what changed since the last inventory. A changed document becomes a **new version** of the document already in FileNet. | `overlap_minutes`, `since` |
|
|
119
|
+
|
|
120
|
+
**Rolling mode** (`rolling: {chunk_docs: 200000}`) runs extract, load and verify in repeating chunks. Staging then holds only a small slice of the data at any time, instead of the full 100 TB.
|
|
121
|
+
|
|
122
|
+
## Sources
|
|
123
|
+
|
|
124
|
+
| Type | Use for | Notes |
|
|
125
|
+
|---|---|---|
|
|
126
|
+
| `wss_sql` | WSS 3.0 / MOSS 2007 content databases | Reads a **restored copy** of the content DB directly, which is many times faster than SharePoint's web services. Streams BLOBs in chunks. Includes version history and AllUserData metadata. Partitions are document libraries. |
|
|
127
|
+
| `sql_blob` | Any database that stores files as BLOBs | You write the SQL. Supports mssql, sqlite, postgres and oracle; chunked streaming; `{since_filter}` for delta. |
|
|
128
|
+
| `filesystem` | File shares, NFS, SMB, local disk | Each top-level folder is a partition. A changed file becomes a new version. |
|
|
129
|
+
| `csv_manifest` | **Any other system** | Export "file path + metadata" to CSV and point lodestream at it. Each CSV file is a partition. |
|
|
130
|
+
| `s3` | S3, MinIO, StorageGRID, Ceph | Optionally migrates every object version. |
|
|
131
|
+
| `cmis` | Alfresco, Documentum, OpenText, Nuxeo, SharePoint CMIS | CMIS 1.1 browser binding. Walks the folder tree; optionally includes versions. |
|
|
132
|
+
|
|
133
|
+
### Add your own source (about 20 lines)
|
|
134
|
+
|
|
135
|
+
```python
|
|
136
|
+
# mycompany/lodestream_ext.py
|
|
137
|
+
from lodestream import Source, DocumentRecord, register_source
|
|
138
|
+
|
|
139
|
+
@register_source("legacy_dms")
|
|
140
|
+
class LegacyDms(Source):
|
|
141
|
+
def partitions(self):
|
|
142
|
+
return api.list_cabinets()
|
|
143
|
+
|
|
144
|
+
def iter_documents(self, partition, since=None):
|
|
145
|
+
for d in api.list_docs(partition, modified_after=since):
|
|
146
|
+
yield DocumentRecord(source_id=f"{d.id}:{d.ver}", series_id=d.id, version_index=d.ver,
|
|
147
|
+
name=d.filename, directory=d.folder, size=d.size,
|
|
148
|
+
modified=d.modified, content_type=d.doctype,
|
|
149
|
+
properties={"PolicyNo": d.policy}, source_ref={"id": d.id, "ver": d.ver})
|
|
150
|
+
|
|
151
|
+
def open_content(self, record):
|
|
152
|
+
return api.download(record.source_ref["id"], record.source_ref["ver"]) # binary stream
|
|
153
|
+
```
|
|
154
|
+
|
|
155
|
+
```yaml
|
|
156
|
+
plugins: [mycompany.lodestream_ext]
|
|
157
|
+
source: {type: legacy_dms, options: {...}}
|
|
158
|
+
```
|
|
159
|
+
|
|
160
|
+
Custom sinks (`@register_sink`) and custom steps (`@register_stage`) work the same way.
|
|
161
|
+
|
|
162
|
+
## FileNet sink (`filenet_graphql`)
|
|
163
|
+
|
|
164
|
+
- Uses the **Content Services GraphQL API**. Each batch of N documents goes in **one** multipart request with N aliased `createDocument` mutations.
|
|
165
|
+
- New versions use batched `checkoutDocument` and `checkinDocument` calls.
|
|
166
|
+
- Creates folders as needed (`file_in_folder: false` skips folder filing for maximum speed).
|
|
167
|
+
- Sends `ECM-CS-XSRF-Token` automatically if the server issues one.
|
|
168
|
+
- **Validate on your server first.** GraphQL field names differ slightly between FileNet and CP4BA releases. Every query is a template you can override (`create_template`, `checkin_template`, `verify_template`, and so on). Run a 10-document pilot and check the results in Administration Console.
|
|
169
|
+
- Above roughly 500 docs/s, implement the same `Sink` contract over the CE Java API (EJB transport), for example as a small Java loader service. The rest of the pipeline stays the same.
|
|
170
|
+
|
|
171
|
+
## Mapping
|
|
172
|
+
|
|
173
|
+
```yaml
|
|
174
|
+
mapping:
|
|
175
|
+
default_class: Document
|
|
176
|
+
classes: {Invoice: Invoice, Contract: Contract} # source content type / library / CMIS type -> class
|
|
177
|
+
properties:
|
|
178
|
+
Title: DocumentTitle
|
|
179
|
+
InvoiceDate: {target: InvoiceDate, type: datetime}
|
|
180
|
+
_source_id: MigrationSourceId # built-ins: _name _path _directory _size _modified
|
|
181
|
+
# _source_id _series_id _version_index _partition
|
|
182
|
+
folder: "/Migrated/{directory}" # null = unfiled
|
|
183
|
+
strip_path_prefixes: ["sites/finance"]
|
|
184
|
+
```
|
|
185
|
+
|
|
186
|
+
Create a `MigrationSourceId` string property in FileNet and map `_source_id` to it. That gives you an exact link from every FileNet document back to its source for audit and reconciliation.
|
|
187
|
+
|
|
188
|
+
## 100 TB playbook
|
|
189
|
+
|
|
190
|
+
1. **Move databases, not files.** Ship compressed SQL backups (or disks) to the FileNet data center and restore them there.
|
|
191
|
+
2. Run `--dry-run` to get document counts and an ETA. Use the `filter` step to drop old versions and junk before moving anything.
|
|
192
|
+
3. Prepare FileNet for the load: disable CBR indexing, subscriptions, workflows and auditing. Pre-size tablespaces and add CPE nodes. Hooks (`hooks.before.load`) can call your own admin scripts for this.
|
|
193
|
+
4. Bulk run with `rolling` chunks. Scale `load.workers` until the FileNet database or CPE CPU is saturated. Watch progress with `lodestream status` and `lodestream estimate`.
|
|
194
|
+
5. Fix or `retry` failures. Review MISMATCH items by hand: they are never retried automatically, to avoid duplicates.
|
|
195
|
+
6. Cutover: freeze the source, run `--steps delta,filter,load,verify`, then switch users over.
|
|
196
|
+
7. Re-enable CBR and reindex. Keep `report.json` and the manifest as the audit record.
|
|
197
|
+
|
|
198
|
+
## Operations notes
|
|
199
|
+
|
|
200
|
+
- **Restartable.** Stop the run at any time (Ctrl+C) and run it again; it continues from the manifest. Failed items are retried on later runs, up to `max_attempts`.
|
|
201
|
+
- **Manifest storage.** Keep the manifest on local SSD. SQLite in WAL mode handles 100M+ rows.
|
|
202
|
+
- **Batch memory.** Uploads with the `requests` library buffer each batch in memory, so `max_batch_mb` caps memory use per worker.
|
|
203
|
+
- **WSS schema.** The default WSS queries target the WSS 3.0 schema. Validate them on one library first and override them in the config if your patch level differs.
|
|
204
|
+
|
|
205
|
+
## Tests
|
|
206
|
+
|
|
207
|
+
```bash
|
|
208
|
+
pip install -e .[dev] && pytest
|
|
209
|
+
```
|
|
210
|
+
|
|
211
|
+
The suite includes a simulated WSS 3.0 content database, a fake FileNet GraphQL endpoint, versioning, delta, rolling chunks, failure and retry, CSV, generic SQL, and the CLI.
|
|
212
|
+
|
|
213
|
+
## Releasing
|
|
214
|
+
|
|
215
|
+
Bump the version in `pyproject.toml` and `src/lodestream/__init__.py`, update `CHANGELOG.md`, push `main`, then publish a GitHub release `vX.Y.Z`. `.github/workflows/python-publish.yml` runs the tests and publishes to PyPI with Trusted Publishing (environment `release`, no stored token), then attaches Sigstore-signed wheel and sdist files to the release.
|
|
216
|
+
|
|
217
|
+
For an internal artifact repository (JFrog Artifactory, Nexus): `python -m build && twine upload --repository-url https://<artifactory>/api/pypi/<repo> dist/*`.
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
# lodestream
|
|
2
|
+
|
|
3
|
+
[](https://pypi.org/project/lodestream/)
|
|
4
|
+
[](https://pypi.org/project/lodestream/)
|
|
5
|
+
[](https://github.com/karthikbd/lodestream/actions/workflows/tests.yml)
|
|
6
|
+
[](LICENSE)
|
|
7
|
+
|
|
8
|
+
**Bulk content migration from any source into IBM FileNet.** Each step can be chosen at run time, the whole run is restartable, and it is built for very large volumes (designed for 100 TB+ migrations).
|
|
9
|
+
|
|
10
|
+
```
|
|
11
|
+
SOURCE (pluggable) STEPS (pick any, in any order) SINK (pluggable)
|
|
12
|
+
─────────────────── ─────────────────────────────────────────── ─────────────────
|
|
13
|
+
wss_sql (WSS/SharePoint DB) inventory → filter → [extract] → load → verify filenet_graphql
|
|
14
|
+
sql_blob (any DB with BLOBs) ▲ │ local_folder (dry runs)
|
|
15
|
+
filesystem (NFS/SMB/local) └──────── delta (cutover catch-up) ┘ your own plugin
|
|
16
|
+
csv_manifest (export from ANY system)
|
|
17
|
+
s3 (S3 / MinIO / StorageGRID) MANIFEST (SQLite): one row per document version
|
|
18
|
+
cmis (Alfresco, Documentum, OpenText…) PENDING → EXTRACTED → LOADED → VERIFIED
|
|
19
|
+
your own plugin (+ SKIPPED / FAILED / MISMATCH)
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
## Install
|
|
23
|
+
|
|
24
|
+
```bash
|
|
25
|
+
pip install ./lodestream[all] # or pick extras: [filenet] [wss] [sql] [s3] [cmis]
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
## Quick start
|
|
29
|
+
|
|
30
|
+
```bash
|
|
31
|
+
lodestream init wss_to_filenet -o migration.yaml # templates: wss / fileshare / csv / cmis / s3 / sql
|
|
32
|
+
# edit migration.yaml: connection strings, FileNet endpoint, class + property mapping
|
|
33
|
+
lodestream run -c migration.yaml --dry-run --rate 300 --gbps 10 # inventory + filter only, then an estimate
|
|
34
|
+
lodestream run -c migration.yaml # full migration
|
|
35
|
+
lodestream status -c migration.yaml
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
## Choosing steps dynamically
|
|
39
|
+
|
|
40
|
+
The `steps:` list in the config is the default plan. You can override it per run:
|
|
41
|
+
|
|
42
|
+
```bash
|
|
43
|
+
lodestream run -c migration.yaml --steps inventory,filter # scope the migration first
|
|
44
|
+
lodestream run -c migration.yaml --steps extract,load,verify # bulk move
|
|
45
|
+
lodestream run -c migration.yaml --steps delta,filter,load,verify # final catch-up before cutover
|
|
46
|
+
lodestream retry -c migration.yaml && lodestream run -c migration.yaml --steps load,verify
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
Or from Python:
|
|
50
|
+
|
|
51
|
+
```python
|
|
52
|
+
from lodestream import Migration
|
|
53
|
+
m = Migration.from_file("migration.yaml")
|
|
54
|
+
m.run(steps=["inventory", "filter"])
|
|
55
|
+
print(m.estimate(docs_per_sec=400, link_gbps=10))
|
|
56
|
+
Migration.from_file("migration.yaml").run(steps=["load", "verify"])
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
| Step | What it does | Key options |
|
|
60
|
+
|---|---|---|
|
|
61
|
+
| `inventory` | Scans the source and registers every document version as PENDING. Partitions are scanned in parallel. | `workers`, `batch_size` |
|
|
62
|
+
| `filter` | Marks items you don't want as SKIPPED: old versions, junk paths, size, date and type. | `versions: all/latest/last_n`, `keep_versions`, `exclude_patterns`, `min_size`, `modified_after` |
|
|
63
|
+
| `extract` | *Optional.* Copies content to fast staging storage with SHA-256 checksums. Leave it out to stream content straight from the source into FileNet. | `staging_dir`, `workers`, `max_staged_bytes` |
|
|
64
|
+
| `load` | Creates documents in parallel, many per API call. Each document's versions load oldest to newest. | `workers`, `batch_size`, `max_batch_mb`, `max_attempts`, `retry_failed` |
|
|
65
|
+
| `verify` | Checks every loaded item in the target, cleans up staging, and writes a reconciliation report. | `workers`, `cleanup_staging`, `report` |
|
|
66
|
+
| `delta` | Re-scans only what changed since the last inventory. A changed document becomes a **new version** of the document already in FileNet. | `overlap_minutes`, `since` |
|
|
67
|
+
|
|
68
|
+
**Rolling mode** (`rolling: {chunk_docs: 200000}`) runs extract, load and verify in repeating chunks. Staging then holds only a small slice of the data at any time, instead of the full 100 TB.
|
|
69
|
+
|
|
70
|
+
## Sources
|
|
71
|
+
|
|
72
|
+
| Type | Use for | Notes |
|
|
73
|
+
|---|---|---|
|
|
74
|
+
| `wss_sql` | WSS 3.0 / MOSS 2007 content databases | Reads a **restored copy** of the content DB directly, which is many times faster than SharePoint's web services. Streams BLOBs in chunks. Includes version history and AllUserData metadata. Partitions are document libraries. |
|
|
75
|
+
| `sql_blob` | Any database that stores files as BLOBs | You write the SQL. Supports mssql, sqlite, postgres and oracle; chunked streaming; `{since_filter}` for delta. |
|
|
76
|
+
| `filesystem` | File shares, NFS, SMB, local disk | Each top-level folder is a partition. A changed file becomes a new version. |
|
|
77
|
+
| `csv_manifest` | **Any other system** | Export "file path + metadata" to CSV and point lodestream at it. Each CSV file is a partition. |
|
|
78
|
+
| `s3` | S3, MinIO, StorageGRID, Ceph | Optionally migrates every object version. |
|
|
79
|
+
| `cmis` | Alfresco, Documentum, OpenText, Nuxeo, SharePoint CMIS | CMIS 1.1 browser binding. Walks the folder tree; optionally includes versions. |
|
|
80
|
+
|
|
81
|
+
### Add your own source (about 20 lines)
|
|
82
|
+
|
|
83
|
+
```python
|
|
84
|
+
# mycompany/lodestream_ext.py
|
|
85
|
+
from lodestream import Source, DocumentRecord, register_source
|
|
86
|
+
|
|
87
|
+
@register_source("legacy_dms")
|
|
88
|
+
class LegacyDms(Source):
|
|
89
|
+
def partitions(self):
|
|
90
|
+
return api.list_cabinets()
|
|
91
|
+
|
|
92
|
+
def iter_documents(self, partition, since=None):
|
|
93
|
+
for d in api.list_docs(partition, modified_after=since):
|
|
94
|
+
yield DocumentRecord(source_id=f"{d.id}:{d.ver}", series_id=d.id, version_index=d.ver,
|
|
95
|
+
name=d.filename, directory=d.folder, size=d.size,
|
|
96
|
+
modified=d.modified, content_type=d.doctype,
|
|
97
|
+
properties={"PolicyNo": d.policy}, source_ref={"id": d.id, "ver": d.ver})
|
|
98
|
+
|
|
99
|
+
def open_content(self, record):
|
|
100
|
+
return api.download(record.source_ref["id"], record.source_ref["ver"]) # binary stream
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
```yaml
|
|
104
|
+
plugins: [mycompany.lodestream_ext]
|
|
105
|
+
source: {type: legacy_dms, options: {...}}
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
Custom sinks (`@register_sink`) and custom steps (`@register_stage`) work the same way.
|
|
109
|
+
|
|
110
|
+
## FileNet sink (`filenet_graphql`)
|
|
111
|
+
|
|
112
|
+
- Uses the **Content Services GraphQL API**. Each batch of N documents goes in **one** multipart request with N aliased `createDocument` mutations.
|
|
113
|
+
- New versions use batched `checkoutDocument` and `checkinDocument` calls.
|
|
114
|
+
- Creates folders as needed (`file_in_folder: false` skips folder filing for maximum speed).
|
|
115
|
+
- Sends `ECM-CS-XSRF-Token` automatically if the server issues one.
|
|
116
|
+
- **Validate on your server first.** GraphQL field names differ slightly between FileNet and CP4BA releases. Every query is a template you can override (`create_template`, `checkin_template`, `verify_template`, and so on). Run a 10-document pilot and check the results in Administration Console.
|
|
117
|
+
- Above roughly 500 docs/s, implement the same `Sink` contract over the CE Java API (EJB transport), for example as a small Java loader service. The rest of the pipeline stays the same.
|
|
118
|
+
|
|
119
|
+
## Mapping
|
|
120
|
+
|
|
121
|
+
```yaml
|
|
122
|
+
mapping:
|
|
123
|
+
default_class: Document
|
|
124
|
+
classes: {Invoice: Invoice, Contract: Contract} # source content type / library / CMIS type -> class
|
|
125
|
+
properties:
|
|
126
|
+
Title: DocumentTitle
|
|
127
|
+
InvoiceDate: {target: InvoiceDate, type: datetime}
|
|
128
|
+
_source_id: MigrationSourceId # built-ins: _name _path _directory _size _modified
|
|
129
|
+
# _source_id _series_id _version_index _partition
|
|
130
|
+
folder: "/Migrated/{directory}" # null = unfiled
|
|
131
|
+
strip_path_prefixes: ["sites/finance"]
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
Create a `MigrationSourceId` string property in FileNet and map `_source_id` to it. That gives you an exact link from every FileNet document back to its source for audit and reconciliation.
|
|
135
|
+
|
|
136
|
+
## 100 TB playbook
|
|
137
|
+
|
|
138
|
+
1. **Move databases, not files.** Ship compressed SQL backups (or disks) to the FileNet data center and restore them there.
|
|
139
|
+
2. Run `--dry-run` to get document counts and an ETA. Use the `filter` step to drop old versions and junk before moving anything.
|
|
140
|
+
3. Prepare FileNet for the load: disable CBR indexing, subscriptions, workflows and auditing. Pre-size tablespaces and add CPE nodes. Hooks (`hooks.before.load`) can call your own admin scripts for this.
|
|
141
|
+
4. Bulk run with `rolling` chunks. Scale `load.workers` until the FileNet database or CPE CPU is saturated. Watch progress with `lodestream status` and `lodestream estimate`.
|
|
142
|
+
5. Fix or `retry` failures. Review MISMATCH items by hand: they are never retried automatically, to avoid duplicates.
|
|
143
|
+
6. Cutover: freeze the source, run `--steps delta,filter,load,verify`, then switch users over.
|
|
144
|
+
7. Re-enable CBR and reindex. Keep `report.json` and the manifest as the audit record.
|
|
145
|
+
|
|
146
|
+
## Operations notes
|
|
147
|
+
|
|
148
|
+
- **Restartable.** Stop the run at any time (Ctrl+C) and run it again; it continues from the manifest. Failed items are retried on later runs, up to `max_attempts`.
|
|
149
|
+
- **Manifest storage.** Keep the manifest on local SSD. SQLite in WAL mode handles 100M+ rows.
|
|
150
|
+
- **Batch memory.** Uploads with the `requests` library buffer each batch in memory, so `max_batch_mb` caps memory use per worker.
|
|
151
|
+
- **WSS schema.** The default WSS queries target the WSS 3.0 schema. Validate them on one library first and override them in the config if your patch level differs.
|
|
152
|
+
|
|
153
|
+
## Tests
|
|
154
|
+
|
|
155
|
+
```bash
|
|
156
|
+
pip install -e .[dev] && pytest
|
|
157
|
+
```
|
|
158
|
+
|
|
159
|
+
The suite includes a simulated WSS 3.0 content database, a fake FileNet GraphQL endpoint, versioning, delta, rolling chunks, failure and retry, CSV, generic SQL, and the CLI.
|
|
160
|
+
|
|
161
|
+
## Releasing
|
|
162
|
+
|
|
163
|
+
Bump the version in `pyproject.toml` and `src/lodestream/__init__.py`, update `CHANGELOG.md`, push `main`, then publish a GitHub release `vX.Y.Z`. `.github/workflows/python-publish.yml` runs the tests and publishes to PyPI with Trusted Publishing (environment `release`, no stored token), then attaches Sigstore-signed wheel and sdist files to the release.
|
|
164
|
+
|
|
165
|
+
For an internal artifact repository (JFrog Artifactory, Nexus): `python -m build && twine upload --repository-url https://<artifactory>/api/pypi/<repo> dist/*`.
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling>=1.24"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "lodestream"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Bulk content migration from any source (WSS/SharePoint, file shares, SQL BLOBs, CSV exports, S3, CMIS) into IBM FileNet: pluggable, restartable, built for 100 TB+."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.10"
|
|
11
|
+
license = { text = "MIT" }
|
|
12
|
+
authors = [{ name = "Karthikkeyan" }]
|
|
13
|
+
keywords = ["filenet", "ecm", "migration", "sharepoint", "wss", "cmis", "content-management", "etl", "bulk-import"]
|
|
14
|
+
classifiers = [
|
|
15
|
+
"Development Status :: 4 - Beta",
|
|
16
|
+
"Intended Audience :: Developers",
|
|
17
|
+
"Intended Audience :: System Administrators",
|
|
18
|
+
"License :: OSI Approved :: MIT License",
|
|
19
|
+
"Operating System :: OS Independent",
|
|
20
|
+
"Programming Language :: Python :: 3",
|
|
21
|
+
"Programming Language :: Python :: 3 :: Only",
|
|
22
|
+
"Programming Language :: Python :: 3.10",
|
|
23
|
+
"Programming Language :: Python :: 3.11",
|
|
24
|
+
"Programming Language :: Python :: 3.12",
|
|
25
|
+
"Programming Language :: Python :: 3.13",
|
|
26
|
+
"Topic :: Office/Business",
|
|
27
|
+
"Topic :: System :: Archiving",
|
|
28
|
+
"Topic :: Utilities",
|
|
29
|
+
"Typing :: Typed",
|
|
30
|
+
]
|
|
31
|
+
# Deliberately small, mainstream dependency set to ease internal artifact-repository approval.
|
|
32
|
+
dependencies = ["pyyaml>=5.4"]
|
|
33
|
+
|
|
34
|
+
[project.optional-dependencies]
|
|
35
|
+
filenet = ["requests>=2.28"]
|
|
36
|
+
wss = ["pyodbc>=4.0"]
|
|
37
|
+
sql = ["pyodbc>=4.0"]
|
|
38
|
+
s3 = ["boto3>=1.26"]
|
|
39
|
+
cmis = ["requests>=2.28"]
|
|
40
|
+
all = ["requests>=2.28", "pyodbc>=4.0", "boto3>=1.26"]
|
|
41
|
+
dev = ["pytest>=8", "requests>=2.28", "ruff>=0.4", "mypy>=1.8", "build>=1.0", "twine>=5.0"]
|
|
42
|
+
|
|
43
|
+
[project.urls]
|
|
44
|
+
Homepage = "https://github.com/karthikbd/lodestream"
|
|
45
|
+
Source = "https://github.com/karthikbd/lodestream"
|
|
46
|
+
Issues = "https://github.com/karthikbd/lodestream/issues"
|
|
47
|
+
Changelog = "https://github.com/karthikbd/lodestream/blob/main/CHANGELOG.md"
|
|
48
|
+
|
|
49
|
+
[project.scripts]
|
|
50
|
+
lodestream = "lodestream.cli:main"
|
|
51
|
+
|
|
52
|
+
[tool.hatch.build.targets.wheel]
|
|
53
|
+
packages = ["src/lodestream"]
|
|
54
|
+
|
|
55
|
+
[tool.hatch.build.targets.sdist]
|
|
56
|
+
include = ["src/lodestream", "tests", "README.md", "LICENSE", "CHANGELOG.md"]
|
|
57
|
+
|
|
58
|
+
[tool.ruff]
|
|
59
|
+
line-length = 120
|
|
60
|
+
target-version = "py310"
|
|
61
|
+
src = ["src"]
|
|
62
|
+
|
|
63
|
+
[tool.ruff.lint]
|
|
64
|
+
select = ["E", "F", "W", "B", "SIM"]
|
|
65
|
+
ignore = ["B008", "B023", "B027", "E501", "E731", "SIM105", "SIM115"]
|
|
66
|
+
|
|
67
|
+
[tool.ruff.lint.per-file-ignores]
|
|
68
|
+
"src/lodestream/registry.py" = ["F401"]
|
|
69
|
+
|
|
70
|
+
[tool.mypy]
|
|
71
|
+
python_version = "3.10"
|
|
72
|
+
strict = false
|
|
73
|
+
ignore_missing_imports = true
|
|
74
|
+
|
|
75
|
+
[tool.pytest.ini_options]
|
|
76
|
+
testpaths = ["tests"]
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
"""lodestream: bulk content migration from any source into IBM FileNet.
|
|
2
|
+
|
|
3
|
+
source (WSS SQL, file share, SQL BLOBs, CSV export, S3, CMIS, or your plugin)
|
|
4
|
+
-> inventory -> filter -> [extract] -> load -> verify (+ delta for cutover)
|
|
5
|
+
-> sink (FileNet CS GraphQL, local folder, or your plugin)
|
|
6
|
+
"""
|
|
7
|
+
from .models import DocumentRecord, Status
|
|
8
|
+
from .pipeline import Migration
|
|
9
|
+
from .registry import register_sink, register_source, register_stage
|
|
10
|
+
from .sinks.base import LoadItem, LoadResult, Sink
|
|
11
|
+
from .sources.base import Source
|
|
12
|
+
from .stages.base import Stage
|
|
13
|
+
|
|
14
|
+
__version__ = "0.1.0"
|
|
15
|
+
|
|
16
|
+
__all__ = [
|
|
17
|
+
"Migration", "DocumentRecord", "Status", "Source", "Sink", "Stage", "LoadItem", "LoadResult",
|
|
18
|
+
"register_source", "register_sink", "register_stage", "__version__",
|
|
19
|
+
]
|