PyReverseETL 3.1.4__tar.gz → 3.1.5__tar.gz

{pyreverseetl-3.1.4 → pyreverseetl-3.1.5}/Cargo.lock

@@ -3199,7 +3199,7 @@ dependencies = [
 
 [[package]]
 name = "pyreverseetl"
-version = "3.1.4"
+version = "3.1.5"
 dependencies = [
  "chrono",
  "pyo3",
@@ -3210,7 +3210,7 @@ dependencies = [
 
 [[package]]
 name = "pyreverseetl-core"
-version = "3.1.4"
+version = "3.1.5"
 dependencies = [
  "async-trait",
  "aws-config",

{pyreverseetl-3.1.4 → pyreverseetl-3.1.5}/Cargo.toml

@@ -3,7 +3,7 @@ members = ["core", "python"]
 resolver = "2"
 
 [workspace.package]
-version = "3.1.4"
+version = "3.1.5"
 authors = ["Georgi Mammen Mullassery <mullassery@gmail.com>"]
 license = "Apache-2.0"
 repository = "https://github.com/Mullassery/PyReverseETL"

{pyreverseetl-3.1.4 → pyreverseetl-3.1.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: PyReverseETL
-Version: 3.1.4
+Version: 3.1.5
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: Programming Language :: Python :: 3
@@ -103,7 +103,7 @@ repeated here to avoid implying vendor endorsement.
 
 ## Project status
 
-**Beta.** Version 3.1.4. The Postgres/MySQL/S3/webhook paths are real and verified against
+**Beta.** Version 3.1.5. The Postgres/MySQL/S3/webhook paths are real and verified against
 real services; the `web` source connector (MudFish-backed site crawling) is implemented
 and unit-tested but not yet exercised end-to-end against a live site; the CRM/marketing
 adapters are real but mock-verified only; several

{pyreverseetl-3.1.4 → pyreverseetl-3.1.5}/README.md

@@ -80,7 +80,7 @@ repeated here to avoid implying vendor endorsement.
 
 ## Project status
 
-**Beta.** Version 3.1.4. The Postgres/MySQL/S3/webhook paths are real and verified against
+**Beta.** Version 3.1.5. The Postgres/MySQL/S3/webhook paths are real and verified against
 real services; the `web` source connector (MudFish-backed site crawling) is implemented
 and unit-tested but not yet exercised end-to-end against a live site; the CRM/marketing
 adapters are real but mock-verified only; several

{pyreverseetl-3.1.4 → pyreverseetl-3.1.5}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "maturin"
 
 [project]
 name = "PyReverseETL"
-version = "3.1.4"
+version = "3.1.5"
 description = "The Open Operational Data Activation Runtime"
 readme = "README.md"
 license = "Apache-2.0"

{pyreverseetl-3.1.4 → pyreverseetl-3.1.5}/python/src/lib.rs

@@ -2,7 +2,7 @@ use pyo3::exceptions::PyValueError;
 use pyo3::prelude::*;
 use pyreverseetl_core::adapters::AuthMethod as AdapterAuth;
 use pyreverseetl_core::connectors::{
-    MySQLConfig, ObjectStorageConfig, PostgreSQLConfig, TableFormat,
+    MySQLConfig, ObjectStorageConfig, PostgreSQLConfig, TableFormat, WebConfig,
 };
 use pyreverseetl_core::destination::DestinationType;
 use pyreverseetl_core::entity::EntityType;
@@ -301,8 +301,27 @@ fn build_source_spec(source_type: &str, config: &Value) -> PyResult<SourceSpec>
             Ok(SourceSpec::MySQL(cfg))
         }
         "s3" => Ok(SourceSpec::S3(build_object_storage_config(config)?)),
+        "web" => {
+            let mut cfg = WebConfig {
+                seed_url: get_str(config, "seed_url")?.to_string(),
+                ..Default::default()
+            };
+            if let Some(v) = config.get("max_depth").and_then(|v| v.as_u64()) {
+                cfg.max_depth = v as usize;
+            }
+            if let Some(v) = config.get("max_urls").and_then(|v| v.as_u64()) {
+                cfg.max_urls = v as usize;
+            }
+            if let Some(v) = config.get("same_domain").and_then(|v| v.as_bool()) {
+                cfg.same_domain = v;
+            }
+            if let Some(v) = config.get("render_js").and_then(|v| v.as_bool()) {
+                cfg.render_js = v;
+            }
+            Ok(SourceSpec::Web(cfg))
+        }
         other => Err(py_err(format!(
-            "unsupported source_type '{other}' (supported: postgres, mysql, s3)"
+            "unsupported source_type '{other}' (supported: postgres, mysql, s3, web)"
         ))),
     }
 }
@@ -489,7 +508,9 @@ pub struct PySyncResult {
 /// Run a real sync: read from `source_type`/`source_config` (JSON string),
 /// apply `compliance_rules` (JSON string, optional), write to
 /// `destination_type`/`destination_config` (JSON string), and record a real
-/// lineage edge. Supported `source_type`: `postgres`, `mysql`, `s3`.
+/// lineage edge. Supported `source_type`: `postgres`, `mysql`, `s3`, `web`
+/// (MudFish-backed site crawl; `source_config` takes `seed_url` plus optional
+/// `max_depth`, `max_urls`, `same_domain`, `render_js`).
 /// Supported `destination_type`: all of those plus `webhook`, `salesforce`,
 /// `hubspot`, `marketo`. With `dry_run=True`, records are still read and run
 /// through the real compliance engine, but the destination is never