lodestream 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. lodestream/__init__.py +19 -0
  2. lodestream/cli.py +138 -0
  3. lodestream/config.py +54 -0
  4. lodestream/examples/cmis_to_filenet.yaml +34 -0
  5. lodestream/examples/csv_to_filenet.yaml +34 -0
  6. lodestream/examples/fileshare_to_filenet.yaml +27 -0
  7. lodestream/examples/s3_to_filenet.yaml +25 -0
  8. lodestream/examples/sql_to_filenet.yaml +40 -0
  9. lodestream/examples/wss_to_filenet.yaml +57 -0
  10. lodestream/manifest.py +270 -0
  11. lodestream/mapping.py +97 -0
  12. lodestream/models.py +64 -0
  13. lodestream/pipeline.py +210 -0
  14. lodestream/py.typed +0 -0
  15. lodestream/registry.py +36 -0
  16. lodestream/sinks/__init__.py +3 -0
  17. lodestream/sinks/base.py +63 -0
  18. lodestream/sinks/filenet_graphql.py +343 -0
  19. lodestream/sinks/local_folder.py +76 -0
  20. lodestream/sources/__init__.py +3 -0
  21. lodestream/sources/base.py +51 -0
  22. lodestream/sources/cmis.py +122 -0
  23. lodestream/sources/csv_manifest.py +85 -0
  24. lodestream/sources/filesystem.py +76 -0
  25. lodestream/sources/s3.py +84 -0
  26. lodestream/sources/sql_blob.py +235 -0
  27. lodestream/sources/wss_sql.py +100 -0
  28. lodestream/stages/__init__.py +3 -0
  29. lodestream/stages/base.py +17 -0
  30. lodestream/stages/extract.py +72 -0
  31. lodestream/stages/filter.py +71 -0
  32. lodestream/stages/inventory.py +78 -0
  33. lodestream/stages/load.py +189 -0
  34. lodestream/stages/verify.py +64 -0
  35. lodestream/utils.py +215 -0
  36. lodestream-0.1.0.dist-info/METADATA +217 -0
  37. lodestream-0.1.0.dist-info/RECORD +40 -0
  38. lodestream-0.1.0.dist-info/WHEEL +4 -0
  39. lodestream-0.1.0.dist-info/entry_points.txt +2 -0
  40. lodestream-0.1.0.dist-info/licenses/LICENSE +21 -0
lodestream/__init__.py ADDED
@@ -0,0 +1,19 @@
1
+ """lodestream: bulk content migration from any source into IBM FileNet.
2
+
3
+ source (WSS SQL, file share, SQL BLOBs, CSV export, S3, CMIS, or your plugin)
4
+ -> inventory -> filter -> [extract] -> load -> verify (+ delta for cutover)
5
+ -> sink (FileNet CS GraphQL, local folder, or your plugin)
6
+ """
7
+ from .models import DocumentRecord, Status
8
+ from .pipeline import Migration
9
+ from .registry import register_sink, register_source, register_stage
10
+ from .sinks.base import LoadItem, LoadResult, Sink
11
+ from .sources.base import Source
12
+ from .stages.base import Stage
13
+
14
+ __version__ = "0.1.0"
15
+
16
+ __all__ = [
17
+ "Migration", "DocumentRecord", "Status", "Source", "Sink", "Stage", "LoadItem", "LoadResult",
18
+ "register_source", "register_sink", "register_stage", "__version__",
19
+ ]
lodestream/cli.py ADDED
@@ -0,0 +1,138 @@
1
+ """Command line interface: lodestream <command> -c migration.yaml"""
2
+ from __future__ import annotations
3
+
4
+ import argparse
5
+ import json
6
+ import logging
7
+ import os
8
+ import shutil
9
+ import sys
10
+
11
+ from .pipeline import Migration
12
+ from .registry import SINKS, SOURCES, load_builtins
13
+ from .utils import human_bytes
14
+
15
+ _EXAMPLES = os.path.join(os.path.dirname(__file__), "examples")
16
+
17
+
18
+ def _setup_logging(verbose: bool) -> None:
19
+ logging.basicConfig(level=logging.DEBUG if verbose else logging.INFO,
20
+ format="%(asctime)s %(levelname)-7s %(message)s", datefmt="%H:%M:%S")
21
+
22
+
23
+ def cmd_steps(args) -> int:
24
+ load_builtins()
25
+ if args.config:
26
+ Migration.from_file(args.config) # loads plugins declared in the config
27
+ print("Steps (run any subset, in any order, with --steps):")
28
+ for name, desc in Migration.available_steps().items():
29
+ print(f" {name:<10} {desc}")
30
+ print("\nSources:", ", ".join(sorted(SOURCES)))
31
+ print("Sinks: ", ", ".join(sorted(SINKS)))
32
+ return 0
33
+
34
+
35
+ def cmd_run(args) -> int:
36
+ m = Migration.from_file(args.config)
37
+ steps = [s.strip() for s in args.steps.split(",")] if args.steps else None
38
+ results = m.run(steps=steps, dry_run=args.dry_run)
39
+ print(json.dumps(results, indent=2, default=str))
40
+ if args.dry_run:
41
+ print(json.dumps(Migration.from_file(args.config).estimate(args.rate, args.gbps), indent=2))
42
+ failed = sum(r.get("failed", 0) for r in results)
43
+ return 2 if failed else 0
44
+
45
+
46
+ def cmd_status(args) -> int:
47
+ st = Migration.from_file(args.config).status()
48
+ if args.json:
49
+ print(json.dumps(st, indent=2, default=str))
50
+ return 0
51
+ print(f"Total documents/versions: {st['total_docs']:,}")
52
+ for status, v in st["by_status"].items():
53
+ print(f" {status:<10} {v['count']:>14,} {human_bytes(v['bytes']):>12}")
54
+ print("\nRecent runs:")
55
+ for r in st["recent_runs"]:
56
+ s = r["stats"]
57
+ keys = ("new", "skipped", "extracted", "loaded", "verified", "failed", "mismatch", "docs_per_sec")
58
+ summary = ", ".join(f"{k}={s[k]}" for k in keys if k in s)
59
+ print(f" #{r['run_id']:<4} {r['step']:<10} {r['started']} {summary}")
60
+ return 0
61
+
62
+
63
+ def cmd_estimate(args) -> int:
64
+ print(json.dumps(Migration.from_file(args.config).estimate(args.rate, args.gbps), indent=2))
65
+ return 0
66
+
67
+
68
+ def cmd_retry(args) -> int:
69
+ n = Migration.from_file(args.config).manifest.reset_failed(reset_attempts=not args.keep_attempts)
70
+ print(f"Reset {n:,} FAILED item(s); run `lodestream run` (load step) to retry them.")
71
+ return 0
72
+
73
+
74
+ def cmd_init(args) -> int:
75
+ src = os.path.join(_EXAMPLES, f"{args.template}.yaml")
76
+ if not os.path.exists(src):
77
+ choices = sorted(f[:-5] for f in os.listdir(_EXAMPLES) if f.endswith(".yaml"))
78
+ print(f"Unknown template {args.template!r}. Choose from: {', '.join(choices)}")
79
+ return 1
80
+ if os.path.exists(args.output) and not args.force:
81
+ print(f"{args.output} exists (use --force to overwrite)")
82
+ return 1
83
+ shutil.copyfile(src, args.output)
84
+ print(f"Wrote {args.output} from template '{args.template}'. Edit it, then: lodestream run -c {args.output}")
85
+ return 0
86
+
87
+
88
+ def main(argv=None) -> int:
89
+ p = argparse.ArgumentParser(prog="lodestream", description="Bulk content migration from any source into IBM FileNet.")
90
+ p.add_argument("-v", "--verbose", action="store_true")
91
+ sub = p.add_subparsers(dest="cmd", required=True)
92
+
93
+ s = sub.add_parser("init", help="write a starter config from a template")
94
+ s.add_argument("template", nargs="?", default="wss_to_filenet",
95
+ help="wss_to_filenet | fileshare_to_filenet | csv_to_filenet | cmis_to_filenet | s3_to_filenet | sql_to_filenet")
96
+ s.add_argument("-o", "--output", default="migration.yaml")
97
+ s.add_argument("--force", action="store_true")
98
+ s.set_defaults(func=cmd_init)
99
+
100
+ s = sub.add_parser("steps", help="list available steps, sources and sinks")
101
+ s.add_argument("-c", "--config")
102
+ s.set_defaults(func=cmd_steps)
103
+
104
+ s = sub.add_parser("run", help="run the migration (all configured steps or --steps)")
105
+ s.add_argument("-c", "--config", required=True)
106
+ s.add_argument("--steps", help="comma-separated, e.g. inventory,filter or delta,load,verify")
107
+ s.add_argument("--dry-run", action="store_true", help="inventory/filter only, then estimate")
108
+ s.add_argument("--rate", type=float, help="docs/sec for the estimate (dry run)")
109
+ s.add_argument("--gbps", type=float, help="network link speed for the estimate (dry run)")
110
+ s.set_defaults(func=cmd_run)
111
+
112
+ s = sub.add_parser("status", help="counts by status and recent runs")
113
+ s.add_argument("-c", "--config", required=True)
114
+ s.add_argument("--json", action="store_true")
115
+ s.set_defaults(func=cmd_status)
116
+
117
+ s = sub.add_parser("estimate", help="time remaining at observed or given rate")
118
+ s.add_argument("-c", "--config", required=True)
119
+ s.add_argument("--rate", type=float)
120
+ s.add_argument("--gbps", type=float)
121
+ s.set_defaults(func=cmd_estimate)
122
+
123
+ s = sub.add_parser("retry", help="send FAILED items back to the queue")
124
+ s.add_argument("-c", "--config", required=True)
125
+ s.add_argument("--keep-attempts", action="store_true")
126
+ s.set_defaults(func=cmd_retry)
127
+
128
+ args = p.parse_args(argv)
129
+ _setup_logging(args.verbose)
130
+ try:
131
+ return args.func(args)
132
+ except BrokenPipeError: # e.g. `lodestream status ... | head`
133
+ sys.stderr.close()
134
+ return 0
135
+
136
+
137
+ if __name__ == "__main__": # pragma: no cover
138
+ sys.exit(main())
lodestream/config.py ADDED
@@ -0,0 +1,54 @@
1
+ """Config loading (YAML or JSON) with ${ENV_VAR} expansion."""
2
+ from __future__ import annotations
3
+
4
+ import json
5
+ import os
6
+ import re
7
+ from typing import Any, Dict, List, Tuple
8
+
9
+ _ENV = re.compile(r"\$\{([A-Za-z_][A-Za-z0-9_]*)(?::-([^}]*))?\}")
10
+
11
+ DEFAULT_STEPS = ["inventory", "filter", "extract", "load", "verify"]
12
+
13
+
14
+ def _expand(value: Any) -> Any:
15
+ if isinstance(value, str):
16
+ return _ENV.sub(lambda m: os.environ.get(m.group(1), m.group(2) or ""), value)
17
+ if isinstance(value, list):
18
+ return [_expand(v) for v in value]
19
+ if isinstance(value, dict):
20
+ return {k: _expand(v) for k, v in value.items()}
21
+ return value
22
+
23
+
24
+ def load_config(path: str) -> Dict[str, Any]:
25
+ with open(path, "r", encoding="utf-8") as fh:
26
+ text = fh.read()
27
+ if path.endswith((".yaml", ".yml")):
28
+ try:
29
+ import yaml
30
+ except ImportError as exc: # pragma: no cover
31
+ raise RuntimeError("PyYAML is required for YAML configs: pip install pyyaml") from exc
32
+ data = yaml.safe_load(text) or {}
33
+ else:
34
+ data = json.loads(text)
35
+ return _expand(data)
36
+
37
+
38
+ def normalize_steps(steps: Any) -> List[Tuple[str, Dict[str, Any]]]:
39
+ """Accepts: ["inventory", "load"]
40
+ [{"inventory": {...}}, {"load": {...}}]
41
+ [{"name": "load", "workers": 8}]"""
42
+ out: List[Tuple[str, Dict[str, Any]]] = []
43
+ for item in steps or DEFAULT_STEPS:
44
+ if isinstance(item, str):
45
+ out.append((item, {}))
46
+ elif isinstance(item, dict) and "name" in item:
47
+ opts = {k: v for k, v in item.items() if k != "name"}
48
+ out.append((item["name"], opts))
49
+ elif isinstance(item, dict) and len(item) == 1:
50
+ name, opts = next(iter(item.items()))
51
+ out.append((name, dict(opts or {})))
52
+ else:
53
+ raise ValueError(f"Cannot understand step entry: {item!r}")
54
+ return out
@@ -0,0 +1,34 @@
1
+ # lodestream: CMIS repository (Alfresco, Documentum, OpenText, Nuxeo, SharePoint CMIS) -> IBM FileNet
2
+ name: cmis-to-filenet
3
+ manifest: ./state/manifest.db
4
+
5
+ source:
6
+ type: cmis
7
+ options:
8
+ url: https://alfresco.example.com/alfresco/api/-default-/public/cmis/versions/1.1/browser
9
+ username: "${CMIS_USER}"
10
+ password: "${CMIS_PASSWORD}"
11
+ folders: ["/Sites/finance/documentLibrary", "/Sites/legal/documentLibrary"]
12
+ include_versions: true
13
+
14
+ sink:
15
+ type: filenet_graphql
16
+ options:
17
+ endpoint: https://cpe.example.com/content-services-graphql/graphql
18
+ repository: OS1
19
+ auth: {type: basic, username: "${FN_USER}", password: "${FN_PASSWORD}"}
20
+
21
+ mapping:
22
+ default_class: Document
23
+ classes: {"D:acme:invoice": Invoice, "cmis:document": Document}
24
+ properties:
25
+ "acme:invoiceNo": InvoiceNumber
26
+ _source_id: MigrationSourceId
27
+ folder: "/Migrated/{directory}"
28
+
29
+ steps:
30
+ - inventory: {workers: 4}
31
+ - filter: {versions: latest}
32
+ - extract: {staging_dir: /mnt/staging, workers: 8}
33
+ - load: {workers: 16}
34
+ - verify: {workers: 8}
@@ -0,0 +1,34 @@
1
+ # lodestream: ANY system that can export "file + metadata" as CSV -> IBM FileNet
2
+ name: csv-to-filenet
3
+ manifest: ./state/manifest.db
4
+
5
+ source:
6
+ type: csv_manifest
7
+ options:
8
+ path: ./exports/*.csv # each CSV is a partition (parallel)
9
+ content_root: /mnt/export/files
10
+ columns: # your CSV header -> lodestream field; other columns = properties
11
+ id: DocumentId
12
+ version: VersionNo
13
+ name: FileName
14
+ directory: FolderPath
15
+ file: StoragePath
16
+ modified: LastModified
17
+ content_type: DocType
18
+
19
+ sink:
20
+ type: filenet_graphql
21
+ options:
22
+ endpoint: https://cpe.example.com/content-services-graphql/graphql
23
+ repository: OS1
24
+ auth: {type: basic, username: "${FN_USER}", password: "${FN_PASSWORD}"}
25
+
26
+ mapping:
27
+ default_class: Document
28
+ classes: {INV: Invoice, CON: Contract}
29
+ properties:
30
+ CustomerNo: CustomerNumber
31
+ _source_id: MigrationSourceId
32
+ folder: "/Migrated/{content_type}/{directory}"
33
+
34
+ steps: [inventory, filter, load, verify]
@@ -0,0 +1,27 @@
1
+ # lodestream: file share / NFS / SMB mount -> IBM FileNet (streams directly, no staging)
2
+ name: fileshare-to-filenet
3
+ manifest: ./state/manifest.db
4
+
5
+ source:
6
+ type: filesystem
7
+ options:
8
+ root: /mnt/legacy_share # each top-level folder is scanned in parallel
9
+
10
+ sink:
11
+ type: filenet_graphql
12
+ options:
13
+ endpoint: https://cpe.example.com/content-services-graphql/graphql
14
+ repository: OS1
15
+ auth: {type: basic, username: "${FN_USER}", password: "${FN_PASSWORD}"}
16
+
17
+ mapping:
18
+ default_class: Document
19
+ properties:
20
+ _source_id: MigrationSourceId
21
+ folder: "/Migrated/{directory}"
22
+
23
+ steps:
24
+ - inventory: {workers: 8}
25
+ - filter: {exclude_patterns: ["*/thumbs.db", "*.tmp", "*/~$*"], min_size: 1}
26
+ - load: {workers: 32, batch_size: 50}
27
+ - verify: {workers: 16, report: ./state/report.json}
@@ -0,0 +1,25 @@
1
+ # lodestream: S3 / S3-compatible object store -> IBM FileNet
2
+ name: s3-to-filenet
3
+ manifest: ./state/manifest.db
4
+
5
+ source:
6
+ type: s3
7
+ options:
8
+ bucket: legacy-archive
9
+ prefix: documents/
10
+ # endpoint_url: https://minio.internal:9000
11
+ include_versions: false
12
+
13
+ sink:
14
+ type: filenet_graphql
15
+ options:
16
+ endpoint: https://cpe.example.com/content-services-graphql/graphql
17
+ repository: OS1
18
+ auth: {type: bearer, token: "${FN_TOKEN}"}
19
+ file_in_folder: false # unfiled = fastest load
20
+
21
+ mapping:
22
+ default_class: Document
23
+ properties: {_path: SourcePath, _source_id: MigrationSourceId}
24
+
25
+ steps: [inventory, load, verify]
@@ -0,0 +1,40 @@
1
+ # lodestream: any relational DB holding files as BLOBs (home-grown DMS, imaging system...) -> FileNet
2
+ name: sql-to-filenet
3
+ manifest: ./state/manifest.db
4
+
5
+ source:
6
+ type: sql_blob
7
+ options:
8
+ connection_string: "DRIVER={ODBC Driver 18 for SQL Server};SERVER=legacydb;DATABASE=DMS;Trusted_Connection=yes"
9
+ dialect: mssql
10
+ partitions_query: "SELECT DISTINCT CabinetId FROM Documents"
11
+ documents_queries:
12
+ - sql: >
13
+ SELECT d.DocId AS id, d.Version AS version, d.FileName AS name, d.FolderPath AS directory,
14
+ d.ByteSize AS size, d.ModifiedOn AS modified, d.DocType AS content_type,
15
+ d.CustomerNo, d.PolicyNo
16
+ FROM Documents d WHERE d.CabinetId = ? {since_filter}
17
+ since_column: d.ModifiedOn
18
+ content:
19
+ default:
20
+ length_query: "SELECT {length}(Data) FROM DocumentData WHERE DocId = ? AND Version = ?"
21
+ chunk_query: "SELECT {substr}(Data, ?, ?) FROM DocumentData WHERE DocId = ? AND Version = ?"
22
+ params: [id, version]
23
+
24
+ sink:
25
+ type: filenet_graphql
26
+ options:
27
+ endpoint: https://cpe.example.com/content-services-graphql/graphql
28
+ repository: OS1
29
+ auth: {type: basic, username: "${FN_USER}", password: "${FN_PASSWORD}"}
30
+
31
+ mapping:
32
+ classes: {POLICY: PolicyDocument, CLAIM: ClaimDocument}
33
+ properties:
34
+ CustomerNo: CustomerNumber
35
+ PolicyNo: PolicyNumber
36
+ _source_id: MigrationSourceId
37
+ folder: "/Migrated/{content_type}"
38
+
39
+ steps: [inventory, filter, extract, load, verify]
40
+ rolling: {chunk_docs: 100000}
@@ -0,0 +1,57 @@
1
+ # lodestream: WSS (Windows SharePoint Services 3.0) -> IBM FileNet
2
+ # Point the source at a RESTORED COPY of the WSS content database, never the live one.
3
+ name: wss-to-filenet
4
+ manifest: ./state/manifest.db # keep on local SSD
5
+
6
+ source:
7
+ type: wss_sql
8
+ options:
9
+ connection_string: "DRIVER={ODBC Driver 18 for SQL Server};SERVER=sqlstage01;DATABASE=WSS_Content_Restored;Trusted_Connection=yes;TrustServerCertificate=yes"
10
+ include_versions: true # migrate version history (filter step can trim it)
11
+ chunk_size_mb: 8 # BLOBs are streamed in 8 MB windows
12
+ userdata_columns: # AllUserData column -> property name (check your list schema)
13
+ nvarchar1: Title
14
+ tp_ContentType: ContentType
15
+ content_type_field: ContentType # use the SharePoint content type to pick the FileNet class
16
+ # site_ids: ["..."] # optional: limit to site collections
17
+ # list_ids: ["..."] # optional: limit to document libraries
18
+
19
+ sink:
20
+ type: filenet_graphql
21
+ options:
22
+ endpoint: https://cpe.example.com/content-services-graphql/graphql
23
+ repository: OS1
24
+ auth: {type: basic, username: "${FN_USER}", password: "${FN_PASSWORD}"}
25
+ file_in_folder: true
26
+ create_folders: true
27
+
28
+ mapping:
29
+ default_class: Document
30
+ classes: # source content type -> FileNet document class
31
+ Invoice: Invoice
32
+ Contract: Contract
33
+ properties: # source field -> FileNet property
34
+ Title: DocumentTitle
35
+ _source_id: MigrationSourceId # create this string property in FileNet first (reconciliation)
36
+ _modified: {target: SourceModified, type: datetime}
37
+ title_property: DocumentTitle
38
+ folder: "/Migrated/{directory}"
39
+ strip_path_prefixes: ["sites"]
40
+
41
+ steps:
42
+ - inventory: {workers: 8}
43
+ - filter:
44
+ versions: last_n
45
+ keep_versions: 3
46
+ exclude_patterns: ["*/forms/*", "*/_catalogs/*", "*/_vti_*", "*.tmp", "*/~$*"]
47
+ min_size: 1
48
+ - extract: {staging_dir: /mnt/staging, workers: 16, max_staged_bytes: 2e12}
49
+ - load: {workers: 32, batch_size: 50, max_batch_mb: 256}
50
+ - verify: {workers: 16, cleanup_staging: true, report: ./state/report.json}
51
+
52
+ rolling:
53
+ chunk_docs: 200000 # extract -> load -> verify in rolling chunks (staging stays small)
54
+
55
+ # hooks: # your own functions, called as fn(ctx, step, stats)
56
+ # before: {load: ["my_fn_admin:disable_cbr_and_subscriptions"]}
57
+ # after: {verify: ["my_fn_admin:enable_cbr_and_reindex"]}