lodestream 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. lodestream-0.1.0/.gitignore +16 -0
  2. lodestream-0.1.0/CHANGELOG.md +11 -0
  3. lodestream-0.1.0/LICENSE +21 -0
  4. lodestream-0.1.0/PKG-INFO +217 -0
  5. lodestream-0.1.0/README.md +165 -0
  6. lodestream-0.1.0/pyproject.toml +76 -0
  7. lodestream-0.1.0/src/lodestream/__init__.py +19 -0
  8. lodestream-0.1.0/src/lodestream/cli.py +138 -0
  9. lodestream-0.1.0/src/lodestream/config.py +54 -0
  10. lodestream-0.1.0/src/lodestream/examples/cmis_to_filenet.yaml +34 -0
  11. lodestream-0.1.0/src/lodestream/examples/csv_to_filenet.yaml +34 -0
  12. lodestream-0.1.0/src/lodestream/examples/fileshare_to_filenet.yaml +27 -0
  13. lodestream-0.1.0/src/lodestream/examples/s3_to_filenet.yaml +25 -0
  14. lodestream-0.1.0/src/lodestream/examples/sql_to_filenet.yaml +40 -0
  15. lodestream-0.1.0/src/lodestream/examples/wss_to_filenet.yaml +57 -0
  16. lodestream-0.1.0/src/lodestream/manifest.py +270 -0
  17. lodestream-0.1.0/src/lodestream/mapping.py +97 -0
  18. lodestream-0.1.0/src/lodestream/models.py +64 -0
  19. lodestream-0.1.0/src/lodestream/pipeline.py +210 -0
  20. lodestream-0.1.0/src/lodestream/py.typed +0 -0
  21. lodestream-0.1.0/src/lodestream/registry.py +36 -0
  22. lodestream-0.1.0/src/lodestream/sinks/__init__.py +3 -0
  23. lodestream-0.1.0/src/lodestream/sinks/base.py +63 -0
  24. lodestream-0.1.0/src/lodestream/sinks/filenet_graphql.py +343 -0
  25. lodestream-0.1.0/src/lodestream/sinks/local_folder.py +76 -0
  26. lodestream-0.1.0/src/lodestream/sources/__init__.py +3 -0
  27. lodestream-0.1.0/src/lodestream/sources/base.py +51 -0
  28. lodestream-0.1.0/src/lodestream/sources/cmis.py +122 -0
  29. lodestream-0.1.0/src/lodestream/sources/csv_manifest.py +85 -0
  30. lodestream-0.1.0/src/lodestream/sources/filesystem.py +76 -0
  31. lodestream-0.1.0/src/lodestream/sources/s3.py +84 -0
  32. lodestream-0.1.0/src/lodestream/sources/sql_blob.py +235 -0
  33. lodestream-0.1.0/src/lodestream/sources/wss_sql.py +100 -0
  34. lodestream-0.1.0/src/lodestream/stages/__init__.py +3 -0
  35. lodestream-0.1.0/src/lodestream/stages/base.py +17 -0
  36. lodestream-0.1.0/src/lodestream/stages/extract.py +72 -0
  37. lodestream-0.1.0/src/lodestream/stages/filter.py +71 -0
  38. lodestream-0.1.0/src/lodestream/stages/inventory.py +78 -0
  39. lodestream-0.1.0/src/lodestream/stages/load.py +189 -0
  40. lodestream-0.1.0/src/lodestream/stages/verify.py +64 -0
  41. lodestream-0.1.0/src/lodestream/utils.py +215 -0
  42. lodestream-0.1.0/tests/test_lodestream.py +342 -0
@@ -0,0 +1,16 @@
1
+ __pycache__/
2
+ *.pyc
3
+ .venv/
4
+ dist/
5
+ build/
6
+ *.egg-info/
7
+ .pytest_cache/
8
+ .ruff_cache/
9
+ .mypy_cache/
10
+ state/
11
+ staging/
12
+ lodestream-out/
13
+ *.db
14
+ *.db-wal
15
+ *.db-shm
16
+ migration.yaml
@@ -0,0 +1,11 @@
1
+ # Changelog
2
+
3
+ ## 0.1.0 (2026-11-18)
4
+
5
+ First release.
6
+
7
+ - Steps: `inventory`, `filter`, `extract`, `load`, `verify`, `delta`. Choose them per run with `--steps` or `Migration.run(steps=[...])`; `rolling` mode runs extract/load/verify in chunks.
8
+ - Sources: `wss_sql` (WSS 3.0 content DB), `sql_blob` (any DB with BLOBs), `filesystem`, `csv_manifest`, `s3`, `cmis`.
9
+ - Sinks: `filenet_graphql` (batched Content Services GraphQL), `local_folder`.
10
+ - SQLite manifest makes runs restartable; failed items retry on later runs; reconciliation report.
11
+ - CLI: `init`, `steps`, `run`, `status`, `estimate`, `retry`.
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Karthikkeyan
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,217 @@
1
+ Metadata-Version: 2.5
2
+ Name: lodestream
3
+ Version: 0.1.0
4
+ Summary: Bulk content migration from any source (WSS/SharePoint, file shares, SQL BLOBs, CSV exports, S3, CMIS) into IBM FileNet: pluggable, restartable, built for 100 TB+.
5
+ Project-URL: Homepage, https://github.com/karthikbd/lodestream
6
+ Project-URL: Source, https://github.com/karthikbd/lodestream
7
+ Project-URL: Issues, https://github.com/karthikbd/lodestream/issues
8
+ Project-URL: Changelog, https://github.com/karthikbd/lodestream/blob/main/CHANGELOG.md
9
+ Author: Karthikkeyan
10
+ License: MIT
11
+ License-File: LICENSE
12
+ Keywords: bulk-import,cmis,content-management,ecm,etl,filenet,migration,sharepoint,wss
13
+ Classifier: Development Status :: 4 - Beta
14
+ Classifier: Intended Audience :: Developers
15
+ Classifier: Intended Audience :: System Administrators
16
+ Classifier: License :: OSI Approved :: MIT License
17
+ Classifier: Operating System :: OS Independent
18
+ Classifier: Programming Language :: Python :: 3
19
+ Classifier: Programming Language :: Python :: 3 :: Only
20
+ Classifier: Programming Language :: Python :: 3.10
21
+ Classifier: Programming Language :: Python :: 3.11
22
+ Classifier: Programming Language :: Python :: 3.12
23
+ Classifier: Programming Language :: Python :: 3.13
24
+ Classifier: Topic :: Office/Business
25
+ Classifier: Topic :: System :: Archiving
26
+ Classifier: Topic :: Utilities
27
+ Classifier: Typing :: Typed
28
+ Requires-Python: >=3.10
29
+ Requires-Dist: pyyaml>=5.4
30
+ Provides-Extra: all
31
+ Requires-Dist: boto3>=1.26; extra == 'all'
32
+ Requires-Dist: pyodbc>=4.0; extra == 'all'
33
+ Requires-Dist: requests>=2.28; extra == 'all'
34
+ Provides-Extra: cmis
35
+ Requires-Dist: requests>=2.28; extra == 'cmis'
36
+ Provides-Extra: dev
37
+ Requires-Dist: build>=1.0; extra == 'dev'
38
+ Requires-Dist: mypy>=1.8; extra == 'dev'
39
+ Requires-Dist: pytest>=8; extra == 'dev'
40
+ Requires-Dist: requests>=2.28; extra == 'dev'
41
+ Requires-Dist: ruff>=0.4; extra == 'dev'
42
+ Requires-Dist: twine>=5.0; extra == 'dev'
43
+ Provides-Extra: filenet
44
+ Requires-Dist: requests>=2.28; extra == 'filenet'
45
+ Provides-Extra: s3
46
+ Requires-Dist: boto3>=1.26; extra == 's3'
47
+ Provides-Extra: sql
48
+ Requires-Dist: pyodbc>=4.0; extra == 'sql'
49
+ Provides-Extra: wss
50
+ Requires-Dist: pyodbc>=4.0; extra == 'wss'
51
+ Description-Content-Type: text/markdown
52
+
53
+ # lodestream
54
+
55
+ [![PyPI version](https://badge.fury.io/py/lodestream.svg)](https://pypi.org/project/lodestream/)
56
+ [![Python versions](https://img.shields.io/pypi/pyversions/lodestream.svg)](https://pypi.org/project/lodestream/)
57
+ [![Tests](https://github.com/karthikbd/lodestream/actions/workflows/tests.yml/badge.svg)](https://github.com/karthikbd/lodestream/actions/workflows/tests.yml)
58
+ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
59
+
60
+ **Bulk content migration from any source into IBM FileNet.** Each step can be chosen at run time, the whole run is restartable, and it is built for very large volumes (designed for 100 TB+ migrations).
61
+
62
+ ```
63
+ SOURCE (pluggable) STEPS (pick any, in any order) SINK (pluggable)
64
+ ─────────────────── ─────────────────────────────────────────── ─────────────────
65
+ wss_sql (WSS/SharePoint DB) inventory → filter → [extract] → load → verify filenet_graphql
66
+ sql_blob (any DB with BLOBs) ▲ │ local_folder (dry runs)
67
+ filesystem (NFS/SMB/local) └──────── delta (cutover catch-up) ┘ your own plugin
68
+ csv_manifest (export from ANY system)
69
+ s3 (S3 / MinIO / StorageGRID) MANIFEST (SQLite): one row per document version
70
+ cmis (Alfresco, Documentum, OpenText…) PENDING → EXTRACTED → LOADED → VERIFIED
71
+ your own plugin (+ SKIPPED / FAILED / MISMATCH)
72
+ ```
73
+
74
+ ## Install
75
+
76
+ ```bash
77
+ pip install ./lodestream[all] # or pick extras: [filenet] [wss] [sql] [s3] [cmis]
78
+ ```
79
+
80
+ ## Quick start
81
+
82
+ ```bash
83
+ lodestream init wss_to_filenet -o migration.yaml # templates: wss / fileshare / csv / cmis / s3 / sql
84
+ # edit migration.yaml: connection strings, FileNet endpoint, class + property mapping
85
+ lodestream run -c migration.yaml --dry-run --rate 300 --gbps 10 # inventory + filter only, then an estimate
86
+ lodestream run -c migration.yaml # full migration
87
+ lodestream status -c migration.yaml
88
+ ```
89
+
90
+ ## Choosing steps dynamically
91
+
92
+ The `steps:` list in the config is the default plan. You can override it per run:
93
+
94
+ ```bash
95
+ lodestream run -c migration.yaml --steps inventory,filter # scope the migration first
96
+ lodestream run -c migration.yaml --steps extract,load,verify # bulk move
97
+ lodestream run -c migration.yaml --steps delta,filter,load,verify # final catch-up before cutover
98
+ lodestream retry -c migration.yaml && lodestream run -c migration.yaml --steps load,verify
99
+ ```
100
+
101
+ Or from Python:
102
+
103
+ ```python
104
+ from lodestream import Migration
105
+ m = Migration.from_file("migration.yaml")
106
+ m.run(steps=["inventory", "filter"])
107
+ print(m.estimate(docs_per_sec=400, link_gbps=10))
108
+ Migration.from_file("migration.yaml").run(steps=["load", "verify"])
109
+ ```
110
+
111
+ | Step | What it does | Key options |
112
+ |---|---|---|
113
+ | `inventory` | Scans the source and registers every document version as PENDING. Partitions are scanned in parallel. | `workers`, `batch_size` |
114
+ | `filter` | Marks items you don't want as SKIPPED: old versions, junk paths, size, date and type. | `versions: all/latest/last_n`, `keep_versions`, `exclude_patterns`, `min_size`, `modified_after` |
115
+ | `extract` | *Optional.* Copies content to fast staging storage with SHA-256 checksums. Leave it out to stream content straight from the source into FileNet. | `staging_dir`, `workers`, `max_staged_bytes` |
116
+ | `load` | Creates documents in parallel, many per API call. Each document's versions load oldest to newest. | `workers`, `batch_size`, `max_batch_mb`, `max_attempts`, `retry_failed` |
117
+ | `verify` | Checks every loaded item in the target, cleans up staging, and writes a reconciliation report. | `workers`, `cleanup_staging`, `report` |
118
+ | `delta` | Re-scans only what changed since the last inventory. A changed document becomes a **new version** of the document already in FileNet. | `overlap_minutes`, `since` |
119
+
120
+ **Rolling mode** (`rolling: {chunk_docs: 200000}`) runs extract, load and verify in repeating chunks. Staging then holds only a small slice of the data at any time, instead of the full 100 TB.
121
+
122
+ ## Sources
123
+
124
+ | Type | Use for | Notes |
125
+ |---|---|---|
126
+ | `wss_sql` | WSS 3.0 / MOSS 2007 content databases | Reads a **restored copy** of the content DB directly, which is many times faster than SharePoint's web services. Streams BLOBs in chunks. Includes version history and AllUserData metadata. Partitions are document libraries. |
127
+ | `sql_blob` | Any database that stores files as BLOBs | You write the SQL. Supports mssql, sqlite, postgres and oracle; chunked streaming; `{since_filter}` for delta. |
128
+ | `filesystem` | File shares, NFS, SMB, local disk | Each top-level folder is a partition. A changed file becomes a new version. |
129
+ | `csv_manifest` | **Any other system** | Export "file path + metadata" to CSV and point lodestream at it. Each CSV file is a partition. |
130
+ | `s3` | S3, MinIO, StorageGRID, Ceph | Optionally migrates every object version. |
131
+ | `cmis` | Alfresco, Documentum, OpenText, Nuxeo, SharePoint CMIS | CMIS 1.1 browser binding. Walks the folder tree; optionally includes versions. |
132
+
133
+ ### Add your own source (about 20 lines)
134
+
135
+ ```python
136
+ # mycompany/lodestream_ext.py
137
+ from lodestream import Source, DocumentRecord, register_source
138
+
139
+ @register_source("legacy_dms")
140
+ class LegacyDms(Source):
141
+ def partitions(self):
142
+ return api.list_cabinets()
143
+
144
+ def iter_documents(self, partition, since=None):
145
+ for d in api.list_docs(partition, modified_after=since):
146
+ yield DocumentRecord(source_id=f"{d.id}:{d.ver}", series_id=d.id, version_index=d.ver,
147
+ name=d.filename, directory=d.folder, size=d.size,
148
+ modified=d.modified, content_type=d.doctype,
149
+ properties={"PolicyNo": d.policy}, source_ref={"id": d.id, "ver": d.ver})
150
+
151
+ def open_content(self, record):
152
+ return api.download(record.source_ref["id"], record.source_ref["ver"]) # binary stream
153
+ ```
154
+
155
+ ```yaml
156
+ plugins: [mycompany.lodestream_ext]
157
+ source: {type: legacy_dms, options: {...}}
158
+ ```
159
+
160
+ Custom sinks (`@register_sink`) and custom steps (`@register_stage`) work the same way.
161
+
162
+ ## FileNet sink (`filenet_graphql`)
163
+
164
+ - Uses the **Content Services GraphQL API**. Each batch of N documents goes in **one** multipart request with N aliased `createDocument` mutations.
165
+ - New versions use batched `checkoutDocument` and `checkinDocument` calls.
166
+ - Creates folders as needed (`file_in_folder: false` skips folder filing for maximum speed).
167
+ - Sends `ECM-CS-XSRF-Token` automatically if the server issues one.
168
+ - **Validate on your server first.** GraphQL field names differ slightly between FileNet and CP4BA releases. Every query is a template you can override (`create_template`, `checkin_template`, `verify_template`, and so on). Run a 10-document pilot and check the results in Administration Console.
169
+ - Above roughly 500 docs/s, implement the same `Sink` contract over the CE Java API (EJB transport), for example as a small Java loader service. The rest of the pipeline stays the same.
170
+
171
+ ## Mapping
172
+
173
+ ```yaml
174
+ mapping:
175
+ default_class: Document
176
+ classes: {Invoice: Invoice, Contract: Contract} # source content type / library / CMIS type -> class
177
+ properties:
178
+ Title: DocumentTitle
179
+ InvoiceDate: {target: InvoiceDate, type: datetime}
180
+ _source_id: MigrationSourceId # built-ins: _name _path _directory _size _modified
181
+ # _source_id _series_id _version_index _partition
182
+ folder: "/Migrated/{directory}" # null = unfiled
183
+ strip_path_prefixes: ["sites/finance"]
184
+ ```
185
+
186
+ Create a `MigrationSourceId` string property in FileNet and map `_source_id` to it. That gives you an exact link from every FileNet document back to its source for audit and reconciliation.
187
+
188
+ ## 100 TB playbook
189
+
190
+ 1. **Move databases, not files.** Ship compressed SQL backups (or disks) to the FileNet data center and restore them there.
191
+ 2. Run `--dry-run` to get document counts and an ETA. Use the `filter` step to drop old versions and junk before moving anything.
192
+ 3. Prepare FileNet for the load: disable CBR indexing, subscriptions, workflows and auditing. Pre-size tablespaces and add CPE nodes. Hooks (`hooks.before.load`) can call your own admin scripts for this.
193
+ 4. Bulk run with `rolling` chunks. Scale `load.workers` until the FileNet database or CPE CPU is saturated. Watch progress with `lodestream status` and `lodestream estimate`.
194
+ 5. Fix or `retry` failures. Review MISMATCH items by hand: they are never retried automatically, to avoid duplicates.
195
+ 6. Cutover: freeze the source, run `--steps delta,filter,load,verify`, then switch users over.
196
+ 7. Re-enable CBR and reindex. Keep `report.json` and the manifest as the audit record.
197
+
198
+ ## Operations notes
199
+
200
+ - **Restartable.** Stop the run at any time (Ctrl+C) and run it again; it continues from the manifest. Failed items are retried on later runs, up to `max_attempts`.
201
+ - **Manifest storage.** Keep the manifest on local SSD. SQLite in WAL mode handles 100M+ rows.
202
+ - **Batch memory.** Uploads with the `requests` library buffer each batch in memory, so `max_batch_mb` caps memory use per worker.
203
+ - **WSS schema.** The default WSS queries target the WSS 3.0 schema. Validate them on one library first and override them in the config if your patch level differs.
204
+
205
+ ## Tests
206
+
207
+ ```bash
208
+ pip install -e .[dev] && pytest
209
+ ```
210
+
211
+ The suite includes a simulated WSS 3.0 content database, a fake FileNet GraphQL endpoint, versioning, delta, rolling chunks, failure and retry, CSV, generic SQL, and the CLI.
212
+
213
+ ## Releasing
214
+
215
+ Bump the version in `pyproject.toml` and `src/lodestream/__init__.py`, update `CHANGELOG.md`, push `main`, then publish a GitHub release `vX.Y.Z`. `.github/workflows/python-publish.yml` runs the tests and publishes to PyPI with Trusted Publishing (environment `release`, no stored token), then attaches Sigstore-signed wheel and sdist files to the release.
216
+
217
+ For an internal artifact repository (JFrog Artifactory, Nexus): `python -m build && twine upload --repository-url https://<artifactory>/api/pypi/<repo> dist/*`.
@@ -0,0 +1,165 @@
1
+ # lodestream
2
+
3
+ [![PyPI version](https://badge.fury.io/py/lodestream.svg)](https://pypi.org/project/lodestream/)
4
+ [![Python versions](https://img.shields.io/pypi/pyversions/lodestream.svg)](https://pypi.org/project/lodestream/)
5
+ [![Tests](https://github.com/karthikbd/lodestream/actions/workflows/tests.yml/badge.svg)](https://github.com/karthikbd/lodestream/actions/workflows/tests.yml)
6
+ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
7
+
8
+ **Bulk content migration from any source into IBM FileNet.** Each step can be chosen at run time, the whole run is restartable, and it is built for very large volumes (designed for 100 TB+ migrations).
9
+
10
+ ```
11
+ SOURCE (pluggable) STEPS (pick any, in any order) SINK (pluggable)
12
+ ─────────────────── ─────────────────────────────────────────── ─────────────────
13
+ wss_sql (WSS/SharePoint DB) inventory → filter → [extract] → load → verify filenet_graphql
14
+ sql_blob (any DB with BLOBs) ▲ │ local_folder (dry runs)
15
+ filesystem (NFS/SMB/local) └──────── delta (cutover catch-up) ┘ your own plugin
16
+ csv_manifest (export from ANY system)
17
+ s3 (S3 / MinIO / StorageGRID) MANIFEST (SQLite): one row per document version
18
+ cmis (Alfresco, Documentum, OpenText…) PENDING → EXTRACTED → LOADED → VERIFIED
19
+ your own plugin (+ SKIPPED / FAILED / MISMATCH)
20
+ ```
21
+
22
+ ## Install
23
+
24
+ ```bash
25
+ pip install ./lodestream[all] # or pick extras: [filenet] [wss] [sql] [s3] [cmis]
26
+ ```
27
+
28
+ ## Quick start
29
+
30
+ ```bash
31
+ lodestream init wss_to_filenet -o migration.yaml # templates: wss / fileshare / csv / cmis / s3 / sql
32
+ # edit migration.yaml: connection strings, FileNet endpoint, class + property mapping
33
+ lodestream run -c migration.yaml --dry-run --rate 300 --gbps 10 # inventory + filter only, then an estimate
34
+ lodestream run -c migration.yaml # full migration
35
+ lodestream status -c migration.yaml
36
+ ```
37
+
38
+ ## Choosing steps dynamically
39
+
40
+ The `steps:` list in the config is the default plan. You can override it per run:
41
+
42
+ ```bash
43
+ lodestream run -c migration.yaml --steps inventory,filter # scope the migration first
44
+ lodestream run -c migration.yaml --steps extract,load,verify # bulk move
45
+ lodestream run -c migration.yaml --steps delta,filter,load,verify # final catch-up before cutover
46
+ lodestream retry -c migration.yaml && lodestream run -c migration.yaml --steps load,verify
47
+ ```
48
+
49
+ Or from Python:
50
+
51
+ ```python
52
+ from lodestream import Migration
53
+ m = Migration.from_file("migration.yaml")
54
+ m.run(steps=["inventory", "filter"])
55
+ print(m.estimate(docs_per_sec=400, link_gbps=10))
56
+ Migration.from_file("migration.yaml").run(steps=["load", "verify"])
57
+ ```
58
+
59
+ | Step | What it does | Key options |
60
+ |---|---|---|
61
+ | `inventory` | Scans the source and registers every document version as PENDING. Partitions are scanned in parallel. | `workers`, `batch_size` |
62
+ | `filter` | Marks items you don't want as SKIPPED: old versions, junk paths, size, date and type. | `versions: all/latest/last_n`, `keep_versions`, `exclude_patterns`, `min_size`, `modified_after` |
63
+ | `extract` | *Optional.* Copies content to fast staging storage with SHA-256 checksums. Leave it out to stream content straight from the source into FileNet. | `staging_dir`, `workers`, `max_staged_bytes` |
64
+ | `load` | Creates documents in parallel, many per API call. Each document's versions load oldest to newest. | `workers`, `batch_size`, `max_batch_mb`, `max_attempts`, `retry_failed` |
65
+ | `verify` | Checks every loaded item in the target, cleans up staging, and writes a reconciliation report. | `workers`, `cleanup_staging`, `report` |
66
+ | `delta` | Re-scans only what changed since the last inventory. A changed document becomes a **new version** of the document already in FileNet. | `overlap_minutes`, `since` |
67
+
68
+ **Rolling mode** (`rolling: {chunk_docs: 200000}`) runs extract, load and verify in repeating chunks. Staging then holds only a small slice of the data at any time, instead of the full 100 TB.
69
+
70
+ ## Sources
71
+
72
+ | Type | Use for | Notes |
73
+ |---|---|---|
74
+ | `wss_sql` | WSS 3.0 / MOSS 2007 content databases | Reads a **restored copy** of the content DB directly, which is many times faster than SharePoint's web services. Streams BLOBs in chunks. Includes version history and AllUserData metadata. Partitions are document libraries. |
75
+ | `sql_blob` | Any database that stores files as BLOBs | You write the SQL. Supports mssql, sqlite, postgres and oracle; chunked streaming; `{since_filter}` for delta. |
76
+ | `filesystem` | File shares, NFS, SMB, local disk | Each top-level folder is a partition. A changed file becomes a new version. |
77
+ | `csv_manifest` | **Any other system** | Export "file path + metadata" to CSV and point lodestream at it. Each CSV file is a partition. |
78
+ | `s3` | S3, MinIO, StorageGRID, Ceph | Optionally migrates every object version. |
79
+ | `cmis` | Alfresco, Documentum, OpenText, Nuxeo, SharePoint CMIS | CMIS 1.1 browser binding. Walks the folder tree; optionally includes versions. |
80
+
81
+ ### Add your own source (about 20 lines)
82
+
83
+ ```python
84
+ # mycompany/lodestream_ext.py
85
+ from lodestream import Source, DocumentRecord, register_source
86
+
87
+ @register_source("legacy_dms")
88
+ class LegacyDms(Source):
89
+ def partitions(self):
90
+ return api.list_cabinets()
91
+
92
+ def iter_documents(self, partition, since=None):
93
+ for d in api.list_docs(partition, modified_after=since):
94
+ yield DocumentRecord(source_id=f"{d.id}:{d.ver}", series_id=d.id, version_index=d.ver,
95
+ name=d.filename, directory=d.folder, size=d.size,
96
+ modified=d.modified, content_type=d.doctype,
97
+ properties={"PolicyNo": d.policy}, source_ref={"id": d.id, "ver": d.ver})
98
+
99
+ def open_content(self, record):
100
+ return api.download(record.source_ref["id"], record.source_ref["ver"]) # binary stream
101
+ ```
102
+
103
+ ```yaml
104
+ plugins: [mycompany.lodestream_ext]
105
+ source: {type: legacy_dms, options: {...}}
106
+ ```
107
+
108
+ Custom sinks (`@register_sink`) and custom steps (`@register_stage`) work the same way.
109
+
110
+ ## FileNet sink (`filenet_graphql`)
111
+
112
+ - Uses the **Content Services GraphQL API**. Each batch of N documents goes in **one** multipart request with N aliased `createDocument` mutations.
113
+ - New versions use batched `checkoutDocument` and `checkinDocument` calls.
114
+ - Creates folders as needed (`file_in_folder: false` skips folder filing for maximum speed).
115
+ - Sends `ECM-CS-XSRF-Token` automatically if the server issues one.
116
+ - **Validate on your server first.** GraphQL field names differ slightly between FileNet and CP4BA releases. Every query is a template you can override (`create_template`, `checkin_template`, `verify_template`, and so on). Run a 10-document pilot and check the results in Administration Console.
117
+ - Above roughly 500 docs/s, implement the same `Sink` contract over the CE Java API (EJB transport), for example as a small Java loader service. The rest of the pipeline stays the same.
118
+
119
+ ## Mapping
120
+
121
+ ```yaml
122
+ mapping:
123
+ default_class: Document
124
+ classes: {Invoice: Invoice, Contract: Contract} # source content type / library / CMIS type -> class
125
+ properties:
126
+ Title: DocumentTitle
127
+ InvoiceDate: {target: InvoiceDate, type: datetime}
128
+ _source_id: MigrationSourceId # built-ins: _name _path _directory _size _modified
129
+ # _source_id _series_id _version_index _partition
130
+ folder: "/Migrated/{directory}" # null = unfiled
131
+ strip_path_prefixes: ["sites/finance"]
132
+ ```
133
+
134
+ Create a `MigrationSourceId` string property in FileNet and map `_source_id` to it. That gives you an exact link from every FileNet document back to its source for audit and reconciliation.
135
+
136
+ ## 100 TB playbook
137
+
138
+ 1. **Move databases, not files.** Ship compressed SQL backups (or disks) to the FileNet data center and restore them there.
139
+ 2. Run `--dry-run` to get document counts and an ETA. Use the `filter` step to drop old versions and junk before moving anything.
140
+ 3. Prepare FileNet for the load: disable CBR indexing, subscriptions, workflows and auditing. Pre-size tablespaces and add CPE nodes. Hooks (`hooks.before.load`) can call your own admin scripts for this.
141
+ 4. Bulk run with `rolling` chunks. Scale `load.workers` until the FileNet database or CPE CPU is saturated. Watch progress with `lodestream status` and `lodestream estimate`.
142
+ 5. Fix or `retry` failures. Review MISMATCH items by hand: they are never retried automatically, to avoid duplicates.
143
+ 6. Cutover: freeze the source, run `--steps delta,filter,load,verify`, then switch users over.
144
+ 7. Re-enable CBR and reindex. Keep `report.json` and the manifest as the audit record.
145
+
146
+ ## Operations notes
147
+
148
+ - **Restartable.** Stop the run at any time (Ctrl+C) and run it again; it continues from the manifest. Failed items are retried on later runs, up to `max_attempts`.
149
+ - **Manifest storage.** Keep the manifest on local SSD. SQLite in WAL mode handles 100M+ rows.
150
+ - **Batch memory.** Uploads with the `requests` library buffer each batch in memory, so `max_batch_mb` caps memory use per worker.
151
+ - **WSS schema.** The default WSS queries target the WSS 3.0 schema. Validate them on one library first and override them in the config if your patch level differs.
152
+
153
+ ## Tests
154
+
155
+ ```bash
156
+ pip install -e .[dev] && pytest
157
+ ```
158
+
159
+ The suite includes a simulated WSS 3.0 content database, a fake FileNet GraphQL endpoint, versioning, delta, rolling chunks, failure and retry, CSV, generic SQL, and the CLI.
160
+
161
+ ## Releasing
162
+
163
+ Bump the version in `pyproject.toml` and `src/lodestream/__init__.py`, update `CHANGELOG.md`, push `main`, then publish a GitHub release `vX.Y.Z`. `.github/workflows/python-publish.yml` runs the tests and publishes to PyPI with Trusted Publishing (environment `release`, no stored token), then attaches Sigstore-signed wheel and sdist files to the release.
164
+
165
+ For an internal artifact repository (JFrog Artifactory, Nexus): `python -m build && twine upload --repository-url https://<artifactory>/api/pypi/<repo> dist/*`.
@@ -0,0 +1,76 @@
1
+ [build-system]
2
+ requires = ["hatchling>=1.24"]
3
+ build-backend = "hatchling.build"
4
+
5
+ [project]
6
+ name = "lodestream"
7
+ version = "0.1.0"
8
+ description = "Bulk content migration from any source (WSS/SharePoint, file shares, SQL BLOBs, CSV exports, S3, CMIS) into IBM FileNet: pluggable, restartable, built for 100 TB+."
9
+ readme = "README.md"
10
+ requires-python = ">=3.10"
11
+ license = { text = "MIT" }
12
+ authors = [{ name = "Karthikkeyan" }]
13
+ keywords = ["filenet", "ecm", "migration", "sharepoint", "wss", "cmis", "content-management", "etl", "bulk-import"]
14
+ classifiers = [
15
+ "Development Status :: 4 - Beta",
16
+ "Intended Audience :: Developers",
17
+ "Intended Audience :: System Administrators",
18
+ "License :: OSI Approved :: MIT License",
19
+ "Operating System :: OS Independent",
20
+ "Programming Language :: Python :: 3",
21
+ "Programming Language :: Python :: 3 :: Only",
22
+ "Programming Language :: Python :: 3.10",
23
+ "Programming Language :: Python :: 3.11",
24
+ "Programming Language :: Python :: 3.12",
25
+ "Programming Language :: Python :: 3.13",
26
+ "Topic :: Office/Business",
27
+ "Topic :: System :: Archiving",
28
+ "Topic :: Utilities",
29
+ "Typing :: Typed",
30
+ ]
31
+ # Deliberately small, mainstream dependency set to ease internal artifact-repository approval.
32
+ dependencies = ["pyyaml>=5.4"]
33
+
34
+ [project.optional-dependencies]
35
+ filenet = ["requests>=2.28"]
36
+ wss = ["pyodbc>=4.0"]
37
+ sql = ["pyodbc>=4.0"]
38
+ s3 = ["boto3>=1.26"]
39
+ cmis = ["requests>=2.28"]
40
+ all = ["requests>=2.28", "pyodbc>=4.0", "boto3>=1.26"]
41
+ dev = ["pytest>=8", "requests>=2.28", "ruff>=0.4", "mypy>=1.8", "build>=1.0", "twine>=5.0"]
42
+
43
+ [project.urls]
44
+ Homepage = "https://github.com/karthikbd/lodestream"
45
+ Source = "https://github.com/karthikbd/lodestream"
46
+ Issues = "https://github.com/karthikbd/lodestream/issues"
47
+ Changelog = "https://github.com/karthikbd/lodestream/blob/main/CHANGELOG.md"
48
+
49
+ [project.scripts]
50
+ lodestream = "lodestream.cli:main"
51
+
52
+ [tool.hatch.build.targets.wheel]
53
+ packages = ["src/lodestream"]
54
+
55
+ [tool.hatch.build.targets.sdist]
56
+ include = ["src/lodestream", "tests", "README.md", "LICENSE", "CHANGELOG.md"]
57
+
58
+ [tool.ruff]
59
+ line-length = 120
60
+ target-version = "py310"
61
+ src = ["src"]
62
+
63
+ [tool.ruff.lint]
64
+ select = ["E", "F", "W", "B", "SIM"]
65
+ ignore = ["B008", "B023", "B027", "E501", "E731", "SIM105", "SIM115"]
66
+
67
+ [tool.ruff.lint.per-file-ignores]
68
+ "src/lodestream/registry.py" = ["F401"]
69
+
70
+ [tool.mypy]
71
+ python_version = "3.10"
72
+ strict = false
73
+ ignore_missing_imports = true
74
+
75
+ [tool.pytest.ini_options]
76
+ testpaths = ["tests"]
@@ -0,0 +1,19 @@
1
+ """lodestream: bulk content migration from any source into IBM FileNet.
2
+
3
+ source (WSS SQL, file share, SQL BLOBs, CSV export, S3, CMIS, or your plugin)
4
+ -> inventory -> filter -> [extract] -> load -> verify (+ delta for cutover)
5
+ -> sink (FileNet CS GraphQL, local folder, or your plugin)
6
+ """
7
+ from .models import DocumentRecord, Status
8
+ from .pipeline import Migration
9
+ from .registry import register_sink, register_source, register_stage
10
+ from .sinks.base import LoadItem, LoadResult, Sink
11
+ from .sources.base import Source
12
+ from .stages.base import Stage
13
+
14
+ __version__ = "0.1.0"
15
+
16
+ __all__ = [
17
+ "Migration", "DocumentRecord", "Status", "Source", "Sink", "Stage", "LoadItem", "LoadResult",
18
+ "register_source", "register_sink", "register_stage", "__version__",
19
+ ]