lodestream 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- lodestream/__init__.py +19 -0
- lodestream/cli.py +138 -0
- lodestream/config.py +54 -0
- lodestream/examples/cmis_to_filenet.yaml +34 -0
- lodestream/examples/csv_to_filenet.yaml +34 -0
- lodestream/examples/fileshare_to_filenet.yaml +27 -0
- lodestream/examples/s3_to_filenet.yaml +25 -0
- lodestream/examples/sql_to_filenet.yaml +40 -0
- lodestream/examples/wss_to_filenet.yaml +57 -0
- lodestream/manifest.py +270 -0
- lodestream/mapping.py +97 -0
- lodestream/models.py +64 -0
- lodestream/pipeline.py +210 -0
- lodestream/py.typed +0 -0
- lodestream/registry.py +36 -0
- lodestream/sinks/__init__.py +3 -0
- lodestream/sinks/base.py +63 -0
- lodestream/sinks/filenet_graphql.py +343 -0
- lodestream/sinks/local_folder.py +76 -0
- lodestream/sources/__init__.py +3 -0
- lodestream/sources/base.py +51 -0
- lodestream/sources/cmis.py +122 -0
- lodestream/sources/csv_manifest.py +85 -0
- lodestream/sources/filesystem.py +76 -0
- lodestream/sources/s3.py +84 -0
- lodestream/sources/sql_blob.py +235 -0
- lodestream/sources/wss_sql.py +100 -0
- lodestream/stages/__init__.py +3 -0
- lodestream/stages/base.py +17 -0
- lodestream/stages/extract.py +72 -0
- lodestream/stages/filter.py +71 -0
- lodestream/stages/inventory.py +78 -0
- lodestream/stages/load.py +189 -0
- lodestream/stages/verify.py +64 -0
- lodestream/utils.py +215 -0
- lodestream-0.1.0.dist-info/METADATA +217 -0
- lodestream-0.1.0.dist-info/RECORD +40 -0
- lodestream-0.1.0.dist-info/WHEEL +4 -0
- lodestream-0.1.0.dist-info/entry_points.txt +2 -0
- lodestream-0.1.0.dist-info/licenses/LICENSE +21 -0
lodestream/__init__.py
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
"""lodestream: bulk content migration from any source into IBM FileNet.
|
|
2
|
+
|
|
3
|
+
source (WSS SQL, file share, SQL BLOBs, CSV export, S3, CMIS, or your plugin)
|
|
4
|
+
-> inventory -> filter -> [extract] -> load -> verify (+ delta for cutover)
|
|
5
|
+
-> sink (FileNet CS GraphQL, local folder, or your plugin)
|
|
6
|
+
"""
|
|
7
|
+
from .models import DocumentRecord, Status
|
|
8
|
+
from .pipeline import Migration
|
|
9
|
+
from .registry import register_sink, register_source, register_stage
|
|
10
|
+
from .sinks.base import LoadItem, LoadResult, Sink
|
|
11
|
+
from .sources.base import Source
|
|
12
|
+
from .stages.base import Stage
|
|
13
|
+
|
|
14
|
+
__version__ = "0.1.0"
|
|
15
|
+
|
|
16
|
+
__all__ = [
|
|
17
|
+
"Migration", "DocumentRecord", "Status", "Source", "Sink", "Stage", "LoadItem", "LoadResult",
|
|
18
|
+
"register_source", "register_sink", "register_stage", "__version__",
|
|
19
|
+
]
|
lodestream/cli.py
ADDED
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
"""Command line interface: lodestream <command> -c migration.yaml"""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import argparse
|
|
5
|
+
import json
|
|
6
|
+
import logging
|
|
7
|
+
import os
|
|
8
|
+
import shutil
|
|
9
|
+
import sys
|
|
10
|
+
|
|
11
|
+
from .pipeline import Migration
|
|
12
|
+
from .registry import SINKS, SOURCES, load_builtins
|
|
13
|
+
from .utils import human_bytes
|
|
14
|
+
|
|
15
|
+
_EXAMPLES = os.path.join(os.path.dirname(__file__), "examples")
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _setup_logging(verbose: bool) -> None:
|
|
19
|
+
logging.basicConfig(level=logging.DEBUG if verbose else logging.INFO,
|
|
20
|
+
format="%(asctime)s %(levelname)-7s %(message)s", datefmt="%H:%M:%S")
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def cmd_steps(args) -> int:
|
|
24
|
+
load_builtins()
|
|
25
|
+
if args.config:
|
|
26
|
+
Migration.from_file(args.config) # loads plugins declared in the config
|
|
27
|
+
print("Steps (run any subset, in any order, with --steps):")
|
|
28
|
+
for name, desc in Migration.available_steps().items():
|
|
29
|
+
print(f" {name:<10} {desc}")
|
|
30
|
+
print("\nSources:", ", ".join(sorted(SOURCES)))
|
|
31
|
+
print("Sinks: ", ", ".join(sorted(SINKS)))
|
|
32
|
+
return 0
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def cmd_run(args) -> int:
|
|
36
|
+
m = Migration.from_file(args.config)
|
|
37
|
+
steps = [s.strip() for s in args.steps.split(",")] if args.steps else None
|
|
38
|
+
results = m.run(steps=steps, dry_run=args.dry_run)
|
|
39
|
+
print(json.dumps(results, indent=2, default=str))
|
|
40
|
+
if args.dry_run:
|
|
41
|
+
print(json.dumps(Migration.from_file(args.config).estimate(args.rate, args.gbps), indent=2))
|
|
42
|
+
failed = sum(r.get("failed", 0) for r in results)
|
|
43
|
+
return 2 if failed else 0
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def cmd_status(args) -> int:
|
|
47
|
+
st = Migration.from_file(args.config).status()
|
|
48
|
+
if args.json:
|
|
49
|
+
print(json.dumps(st, indent=2, default=str))
|
|
50
|
+
return 0
|
|
51
|
+
print(f"Total documents/versions: {st['total_docs']:,}")
|
|
52
|
+
for status, v in st["by_status"].items():
|
|
53
|
+
print(f" {status:<10} {v['count']:>14,} {human_bytes(v['bytes']):>12}")
|
|
54
|
+
print("\nRecent runs:")
|
|
55
|
+
for r in st["recent_runs"]:
|
|
56
|
+
s = r["stats"]
|
|
57
|
+
keys = ("new", "skipped", "extracted", "loaded", "verified", "failed", "mismatch", "docs_per_sec")
|
|
58
|
+
summary = ", ".join(f"{k}={s[k]}" for k in keys if k in s)
|
|
59
|
+
print(f" #{r['run_id']:<4} {r['step']:<10} {r['started']} {summary}")
|
|
60
|
+
return 0
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def cmd_estimate(args) -> int:
|
|
64
|
+
print(json.dumps(Migration.from_file(args.config).estimate(args.rate, args.gbps), indent=2))
|
|
65
|
+
return 0
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def cmd_retry(args) -> int:
|
|
69
|
+
n = Migration.from_file(args.config).manifest.reset_failed(reset_attempts=not args.keep_attempts)
|
|
70
|
+
print(f"Reset {n:,} FAILED item(s); run `lodestream run` (load step) to retry them.")
|
|
71
|
+
return 0
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def cmd_init(args) -> int:
|
|
75
|
+
src = os.path.join(_EXAMPLES, f"{args.template}.yaml")
|
|
76
|
+
if not os.path.exists(src):
|
|
77
|
+
choices = sorted(f[:-5] for f in os.listdir(_EXAMPLES) if f.endswith(".yaml"))
|
|
78
|
+
print(f"Unknown template {args.template!r}. Choose from: {', '.join(choices)}")
|
|
79
|
+
return 1
|
|
80
|
+
if os.path.exists(args.output) and not args.force:
|
|
81
|
+
print(f"{args.output} exists (use --force to overwrite)")
|
|
82
|
+
return 1
|
|
83
|
+
shutil.copyfile(src, args.output)
|
|
84
|
+
print(f"Wrote {args.output} from template '{args.template}'. Edit it, then: lodestream run -c {args.output}")
|
|
85
|
+
return 0
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def main(argv=None) -> int:
|
|
89
|
+
p = argparse.ArgumentParser(prog="lodestream", description="Bulk content migration from any source into IBM FileNet.")
|
|
90
|
+
p.add_argument("-v", "--verbose", action="store_true")
|
|
91
|
+
sub = p.add_subparsers(dest="cmd", required=True)
|
|
92
|
+
|
|
93
|
+
s = sub.add_parser("init", help="write a starter config from a template")
|
|
94
|
+
s.add_argument("template", nargs="?", default="wss_to_filenet",
|
|
95
|
+
help="wss_to_filenet | fileshare_to_filenet | csv_to_filenet | cmis_to_filenet | s3_to_filenet | sql_to_filenet")
|
|
96
|
+
s.add_argument("-o", "--output", default="migration.yaml")
|
|
97
|
+
s.add_argument("--force", action="store_true")
|
|
98
|
+
s.set_defaults(func=cmd_init)
|
|
99
|
+
|
|
100
|
+
s = sub.add_parser("steps", help="list available steps, sources and sinks")
|
|
101
|
+
s.add_argument("-c", "--config")
|
|
102
|
+
s.set_defaults(func=cmd_steps)
|
|
103
|
+
|
|
104
|
+
s = sub.add_parser("run", help="run the migration (all configured steps or --steps)")
|
|
105
|
+
s.add_argument("-c", "--config", required=True)
|
|
106
|
+
s.add_argument("--steps", help="comma-separated, e.g. inventory,filter or delta,load,verify")
|
|
107
|
+
s.add_argument("--dry-run", action="store_true", help="inventory/filter only, then estimate")
|
|
108
|
+
s.add_argument("--rate", type=float, help="docs/sec for the estimate (dry run)")
|
|
109
|
+
s.add_argument("--gbps", type=float, help="network link speed for the estimate (dry run)")
|
|
110
|
+
s.set_defaults(func=cmd_run)
|
|
111
|
+
|
|
112
|
+
s = sub.add_parser("status", help="counts by status and recent runs")
|
|
113
|
+
s.add_argument("-c", "--config", required=True)
|
|
114
|
+
s.add_argument("--json", action="store_true")
|
|
115
|
+
s.set_defaults(func=cmd_status)
|
|
116
|
+
|
|
117
|
+
s = sub.add_parser("estimate", help="time remaining at observed or given rate")
|
|
118
|
+
s.add_argument("-c", "--config", required=True)
|
|
119
|
+
s.add_argument("--rate", type=float)
|
|
120
|
+
s.add_argument("--gbps", type=float)
|
|
121
|
+
s.set_defaults(func=cmd_estimate)
|
|
122
|
+
|
|
123
|
+
s = sub.add_parser("retry", help="send FAILED items back to the queue")
|
|
124
|
+
s.add_argument("-c", "--config", required=True)
|
|
125
|
+
s.add_argument("--keep-attempts", action="store_true")
|
|
126
|
+
s.set_defaults(func=cmd_retry)
|
|
127
|
+
|
|
128
|
+
args = p.parse_args(argv)
|
|
129
|
+
_setup_logging(args.verbose)
|
|
130
|
+
try:
|
|
131
|
+
return args.func(args)
|
|
132
|
+
except BrokenPipeError: # e.g. `lodestream status ... | head`
|
|
133
|
+
sys.stderr.close()
|
|
134
|
+
return 0
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
if __name__ == "__main__": # pragma: no cover
|
|
138
|
+
sys.exit(main())
|
lodestream/config.py
ADDED
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
"""Config loading (YAML or JSON) with ${ENV_VAR} expansion."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import json
|
|
5
|
+
import os
|
|
6
|
+
import re
|
|
7
|
+
from typing import Any, Dict, List, Tuple
|
|
8
|
+
|
|
9
|
+
_ENV = re.compile(r"\$\{([A-Za-z_][A-Za-z0-9_]*)(?::-([^}]*))?\}")
|
|
10
|
+
|
|
11
|
+
DEFAULT_STEPS = ["inventory", "filter", "extract", "load", "verify"]
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def _expand(value: Any) -> Any:
|
|
15
|
+
if isinstance(value, str):
|
|
16
|
+
return _ENV.sub(lambda m: os.environ.get(m.group(1), m.group(2) or ""), value)
|
|
17
|
+
if isinstance(value, list):
|
|
18
|
+
return [_expand(v) for v in value]
|
|
19
|
+
if isinstance(value, dict):
|
|
20
|
+
return {k: _expand(v) for k, v in value.items()}
|
|
21
|
+
return value
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def load_config(path: str) -> Dict[str, Any]:
|
|
25
|
+
with open(path, "r", encoding="utf-8") as fh:
|
|
26
|
+
text = fh.read()
|
|
27
|
+
if path.endswith((".yaml", ".yml")):
|
|
28
|
+
try:
|
|
29
|
+
import yaml
|
|
30
|
+
except ImportError as exc: # pragma: no cover
|
|
31
|
+
raise RuntimeError("PyYAML is required for YAML configs: pip install pyyaml") from exc
|
|
32
|
+
data = yaml.safe_load(text) or {}
|
|
33
|
+
else:
|
|
34
|
+
data = json.loads(text)
|
|
35
|
+
return _expand(data)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def normalize_steps(steps: Any) -> List[Tuple[str, Dict[str, Any]]]:
|
|
39
|
+
"""Accepts: ["inventory", "load"]
|
|
40
|
+
[{"inventory": {...}}, {"load": {...}}]
|
|
41
|
+
[{"name": "load", "workers": 8}]"""
|
|
42
|
+
out: List[Tuple[str, Dict[str, Any]]] = []
|
|
43
|
+
for item in steps or DEFAULT_STEPS:
|
|
44
|
+
if isinstance(item, str):
|
|
45
|
+
out.append((item, {}))
|
|
46
|
+
elif isinstance(item, dict) and "name" in item:
|
|
47
|
+
opts = {k: v for k, v in item.items() if k != "name"}
|
|
48
|
+
out.append((item["name"], opts))
|
|
49
|
+
elif isinstance(item, dict) and len(item) == 1:
|
|
50
|
+
name, opts = next(iter(item.items()))
|
|
51
|
+
out.append((name, dict(opts or {})))
|
|
52
|
+
else:
|
|
53
|
+
raise ValueError(f"Cannot understand step entry: {item!r}")
|
|
54
|
+
return out
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
# lodestream: CMIS repository (Alfresco, Documentum, OpenText, Nuxeo, SharePoint CMIS) -> IBM FileNet
|
|
2
|
+
name: cmis-to-filenet
|
|
3
|
+
manifest: ./state/manifest.db
|
|
4
|
+
|
|
5
|
+
source:
|
|
6
|
+
type: cmis
|
|
7
|
+
options:
|
|
8
|
+
url: https://alfresco.example.com/alfresco/api/-default-/public/cmis/versions/1.1/browser
|
|
9
|
+
username: "${CMIS_USER}"
|
|
10
|
+
password: "${CMIS_PASSWORD}"
|
|
11
|
+
folders: ["/Sites/finance/documentLibrary", "/Sites/legal/documentLibrary"]
|
|
12
|
+
include_versions: true
|
|
13
|
+
|
|
14
|
+
sink:
|
|
15
|
+
type: filenet_graphql
|
|
16
|
+
options:
|
|
17
|
+
endpoint: https://cpe.example.com/content-services-graphql/graphql
|
|
18
|
+
repository: OS1
|
|
19
|
+
auth: {type: basic, username: "${FN_USER}", password: "${FN_PASSWORD}"}
|
|
20
|
+
|
|
21
|
+
mapping:
|
|
22
|
+
default_class: Document
|
|
23
|
+
classes: {"D:acme:invoice": Invoice, "cmis:document": Document}
|
|
24
|
+
properties:
|
|
25
|
+
"acme:invoiceNo": InvoiceNumber
|
|
26
|
+
_source_id: MigrationSourceId
|
|
27
|
+
folder: "/Migrated/{directory}"
|
|
28
|
+
|
|
29
|
+
steps:
|
|
30
|
+
- inventory: {workers: 4}
|
|
31
|
+
- filter: {versions: latest}
|
|
32
|
+
- extract: {staging_dir: /mnt/staging, workers: 8}
|
|
33
|
+
- load: {workers: 16}
|
|
34
|
+
- verify: {workers: 8}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
# lodestream: ANY system that can export "file + metadata" as CSV -> IBM FileNet
|
|
2
|
+
name: csv-to-filenet
|
|
3
|
+
manifest: ./state/manifest.db
|
|
4
|
+
|
|
5
|
+
source:
|
|
6
|
+
type: csv_manifest
|
|
7
|
+
options:
|
|
8
|
+
path: ./exports/*.csv # each CSV is a partition (parallel)
|
|
9
|
+
content_root: /mnt/export/files
|
|
10
|
+
columns: # your CSV header -> lodestream field; other columns = properties
|
|
11
|
+
id: DocumentId
|
|
12
|
+
version: VersionNo
|
|
13
|
+
name: FileName
|
|
14
|
+
directory: FolderPath
|
|
15
|
+
file: StoragePath
|
|
16
|
+
modified: LastModified
|
|
17
|
+
content_type: DocType
|
|
18
|
+
|
|
19
|
+
sink:
|
|
20
|
+
type: filenet_graphql
|
|
21
|
+
options:
|
|
22
|
+
endpoint: https://cpe.example.com/content-services-graphql/graphql
|
|
23
|
+
repository: OS1
|
|
24
|
+
auth: {type: basic, username: "${FN_USER}", password: "${FN_PASSWORD}"}
|
|
25
|
+
|
|
26
|
+
mapping:
|
|
27
|
+
default_class: Document
|
|
28
|
+
classes: {INV: Invoice, CON: Contract}
|
|
29
|
+
properties:
|
|
30
|
+
CustomerNo: CustomerNumber
|
|
31
|
+
_source_id: MigrationSourceId
|
|
32
|
+
folder: "/Migrated/{content_type}/{directory}"
|
|
33
|
+
|
|
34
|
+
steps: [inventory, filter, load, verify]
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
# lodestream: file share / NFS / SMB mount -> IBM FileNet (streams directly, no staging)
|
|
2
|
+
name: fileshare-to-filenet
|
|
3
|
+
manifest: ./state/manifest.db
|
|
4
|
+
|
|
5
|
+
source:
|
|
6
|
+
type: filesystem
|
|
7
|
+
options:
|
|
8
|
+
root: /mnt/legacy_share # each top-level folder is scanned in parallel
|
|
9
|
+
|
|
10
|
+
sink:
|
|
11
|
+
type: filenet_graphql
|
|
12
|
+
options:
|
|
13
|
+
endpoint: https://cpe.example.com/content-services-graphql/graphql
|
|
14
|
+
repository: OS1
|
|
15
|
+
auth: {type: basic, username: "${FN_USER}", password: "${FN_PASSWORD}"}
|
|
16
|
+
|
|
17
|
+
mapping:
|
|
18
|
+
default_class: Document
|
|
19
|
+
properties:
|
|
20
|
+
_source_id: MigrationSourceId
|
|
21
|
+
folder: "/Migrated/{directory}"
|
|
22
|
+
|
|
23
|
+
steps:
|
|
24
|
+
- inventory: {workers: 8}
|
|
25
|
+
- filter: {exclude_patterns: ["*/thumbs.db", "*.tmp", "*/~$*"], min_size: 1}
|
|
26
|
+
- load: {workers: 32, batch_size: 50}
|
|
27
|
+
- verify: {workers: 16, report: ./state/report.json}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
# lodestream: S3 / S3-compatible object store -> IBM FileNet
|
|
2
|
+
name: s3-to-filenet
|
|
3
|
+
manifest: ./state/manifest.db
|
|
4
|
+
|
|
5
|
+
source:
|
|
6
|
+
type: s3
|
|
7
|
+
options:
|
|
8
|
+
bucket: legacy-archive
|
|
9
|
+
prefix: documents/
|
|
10
|
+
# endpoint_url: https://minio.internal:9000
|
|
11
|
+
include_versions: false
|
|
12
|
+
|
|
13
|
+
sink:
|
|
14
|
+
type: filenet_graphql
|
|
15
|
+
options:
|
|
16
|
+
endpoint: https://cpe.example.com/content-services-graphql/graphql
|
|
17
|
+
repository: OS1
|
|
18
|
+
auth: {type: bearer, token: "${FN_TOKEN}"}
|
|
19
|
+
file_in_folder: false # unfiled = fastest load
|
|
20
|
+
|
|
21
|
+
mapping:
|
|
22
|
+
default_class: Document
|
|
23
|
+
properties: {_path: SourcePath, _source_id: MigrationSourceId}
|
|
24
|
+
|
|
25
|
+
steps: [inventory, load, verify]
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
# lodestream: any relational DB holding files as BLOBs (home-grown DMS, imaging system...) -> FileNet
|
|
2
|
+
name: sql-to-filenet
|
|
3
|
+
manifest: ./state/manifest.db
|
|
4
|
+
|
|
5
|
+
source:
|
|
6
|
+
type: sql_blob
|
|
7
|
+
options:
|
|
8
|
+
connection_string: "DRIVER={ODBC Driver 18 for SQL Server};SERVER=legacydb;DATABASE=DMS;Trusted_Connection=yes"
|
|
9
|
+
dialect: mssql
|
|
10
|
+
partitions_query: "SELECT DISTINCT CabinetId FROM Documents"
|
|
11
|
+
documents_queries:
|
|
12
|
+
- sql: >
|
|
13
|
+
SELECT d.DocId AS id, d.Version AS version, d.FileName AS name, d.FolderPath AS directory,
|
|
14
|
+
d.ByteSize AS size, d.ModifiedOn AS modified, d.DocType AS content_type,
|
|
15
|
+
d.CustomerNo, d.PolicyNo
|
|
16
|
+
FROM Documents d WHERE d.CabinetId = ? {since_filter}
|
|
17
|
+
since_column: d.ModifiedOn
|
|
18
|
+
content:
|
|
19
|
+
default:
|
|
20
|
+
length_query: "SELECT {length}(Data) FROM DocumentData WHERE DocId = ? AND Version = ?"
|
|
21
|
+
chunk_query: "SELECT {substr}(Data, ?, ?) FROM DocumentData WHERE DocId = ? AND Version = ?"
|
|
22
|
+
params: [id, version]
|
|
23
|
+
|
|
24
|
+
sink:
|
|
25
|
+
type: filenet_graphql
|
|
26
|
+
options:
|
|
27
|
+
endpoint: https://cpe.example.com/content-services-graphql/graphql
|
|
28
|
+
repository: OS1
|
|
29
|
+
auth: {type: basic, username: "${FN_USER}", password: "${FN_PASSWORD}"}
|
|
30
|
+
|
|
31
|
+
mapping:
|
|
32
|
+
classes: {POLICY: PolicyDocument, CLAIM: ClaimDocument}
|
|
33
|
+
properties:
|
|
34
|
+
CustomerNo: CustomerNumber
|
|
35
|
+
PolicyNo: PolicyNumber
|
|
36
|
+
_source_id: MigrationSourceId
|
|
37
|
+
folder: "/Migrated/{content_type}"
|
|
38
|
+
|
|
39
|
+
steps: [inventory, filter, extract, load, verify]
|
|
40
|
+
rolling: {chunk_docs: 100000}
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
# lodestream: WSS (Windows SharePoint Services 3.0) -> IBM FileNet
|
|
2
|
+
# Point the source at a RESTORED COPY of the WSS content database, never the live one.
|
|
3
|
+
name: wss-to-filenet
|
|
4
|
+
manifest: ./state/manifest.db # keep on local SSD
|
|
5
|
+
|
|
6
|
+
source:
|
|
7
|
+
type: wss_sql
|
|
8
|
+
options:
|
|
9
|
+
connection_string: "DRIVER={ODBC Driver 18 for SQL Server};SERVER=sqlstage01;DATABASE=WSS_Content_Restored;Trusted_Connection=yes;TrustServerCertificate=yes"
|
|
10
|
+
include_versions: true # migrate version history (filter step can trim it)
|
|
11
|
+
chunk_size_mb: 8 # BLOBs are streamed in 8 MB windows
|
|
12
|
+
userdata_columns: # AllUserData column -> property name (check your list schema)
|
|
13
|
+
nvarchar1: Title
|
|
14
|
+
tp_ContentType: ContentType
|
|
15
|
+
content_type_field: ContentType # use the SharePoint content type to pick the FileNet class
|
|
16
|
+
# site_ids: ["..."] # optional: limit to site collections
|
|
17
|
+
# list_ids: ["..."] # optional: limit to document libraries
|
|
18
|
+
|
|
19
|
+
sink:
|
|
20
|
+
type: filenet_graphql
|
|
21
|
+
options:
|
|
22
|
+
endpoint: https://cpe.example.com/content-services-graphql/graphql
|
|
23
|
+
repository: OS1
|
|
24
|
+
auth: {type: basic, username: "${FN_USER}", password: "${FN_PASSWORD}"}
|
|
25
|
+
file_in_folder: true
|
|
26
|
+
create_folders: true
|
|
27
|
+
|
|
28
|
+
mapping:
|
|
29
|
+
default_class: Document
|
|
30
|
+
classes: # source content type -> FileNet document class
|
|
31
|
+
Invoice: Invoice
|
|
32
|
+
Contract: Contract
|
|
33
|
+
properties: # source field -> FileNet property
|
|
34
|
+
Title: DocumentTitle
|
|
35
|
+
_source_id: MigrationSourceId # create this string property in FileNet first (reconciliation)
|
|
36
|
+
_modified: {target: SourceModified, type: datetime}
|
|
37
|
+
title_property: DocumentTitle
|
|
38
|
+
folder: "/Migrated/{directory}"
|
|
39
|
+
strip_path_prefixes: ["sites"]
|
|
40
|
+
|
|
41
|
+
steps:
|
|
42
|
+
- inventory: {workers: 8}
|
|
43
|
+
- filter:
|
|
44
|
+
versions: last_n
|
|
45
|
+
keep_versions: 3
|
|
46
|
+
exclude_patterns: ["*/forms/*", "*/_catalogs/*", "*/_vti_*", "*.tmp", "*/~$*"]
|
|
47
|
+
min_size: 1
|
|
48
|
+
- extract: {staging_dir: /mnt/staging, workers: 16, max_staged_bytes: 2e12}
|
|
49
|
+
- load: {workers: 32, batch_size: 50, max_batch_mb: 256}
|
|
50
|
+
- verify: {workers: 16, cleanup_staging: true, report: ./state/report.json}
|
|
51
|
+
|
|
52
|
+
rolling:
|
|
53
|
+
chunk_docs: 200000 # extract -> load -> verify in rolling chunks (staging stays small)
|
|
54
|
+
|
|
55
|
+
# hooks: # your own functions, called as fn(ctx, step, stats)
|
|
56
|
+
# before: {load: ["my_fn_admin:disable_cbr_and_subscriptions"]}
|
|
57
|
+
# after: {verify: ["my_fn_admin:enable_cbr_and_reindex"]}
|