xfmr-zem 0.2.7__py3-none-any.whl → 0.2.8__py3-none-any.whl

xfmr_zem/client.py

@@ -51,22 +51,48 @@ class PipelineClient:
                 items.append((new_key, v))
         return dict(items)
 
+    def _unflatten_params(self, flat_dict: Dict[str, Any]) -> Dict[str, Any]:
+        """Expand dot-notation keys into nested dictionaries."""
+        nested = {}
+        for key, value in flat_dict.items():
+            if "." in key:
+                parts = key.split(".")
+                d = nested
+                for part in parts[:-1]:
+                    if part not in d or not isinstance(d[part], dict):
+                        d[part] = {}
+                    d = d[part]
+                d[parts[-1]] = value
+            else:
+                if isinstance(value, dict) and key in nested and isinstance(nested[key], dict):
+                    nested[key].update(value)
+                else:
+                    nested[key] = value
+        return nested
+
     def _load_config_dict(self, path: Path) -> Dict[str, Any]:
         """Load YAML config and perform substitution."""
         with open(path, "r") as f:
             raw_content = f.read()
             
-        self.params = self._load_params(None)
+        # 1. Load parameters from file
+        base_params = self._load_params(None)
+        
+        # 2. Add custom parameters file if provided
+        if self.params_path:
+            custom_params = self._load_params(self.params_path)
+            base_params.update(custom_params)
+            
+        # 3. Load internal parameters from the config file itself
         preliminary_dict = yaml.safe_load(raw_content) or {}
         internal_params = preliminary_dict.get("parameters", {})
         if internal_params:
-            self.params.update(internal_params)
-            
-        if self.params_path:
-            custom_params = self._load_params(self.params_path)
-            self.params.update(custom_params)
+            base_params.update(internal_params)
             
-        # Flatten params for template substitution
+        # Store unflattened parameters for hierarchical lookup
+        self.params = self._unflatten_params(base_params)
+        
+        # 4. Flatten all params for template substitution ({{ key }})
         flat_params = self._flatten_params(self.params)
         
         content = raw_content
@@ -105,11 +131,12 @@ class PipelineClient:
             env["PYTHONPATH"] = f"{src_path}:{current_pythonpath}" if current_pythonpath else src_path
 
             server_specific_params = {}
-            prefix = f"{name}."
             for key, value in self.params.items():
-                if key.startswith(prefix):
-                    server_specific_params[key[len(prefix):]] = value
-                else:
+                if key == name and isinstance(value, dict):
+                    # Direct match: ocr -> { ... }
+                    server_specific_params.update(value)
+                elif not isinstance(value, dict):
+                    # Global scalars
                     server_specific_params[key] = value
             
             env["ZEM_PARAMETERS"] = yaml.dump(server_specific_params)

xfmr_zem/servers/ocr/server.py

@@ -43,6 +43,7 @@ def extract_pdf_pages(
             # Temporary save for engine compatibility (engines expect path)
             temp_path = os.path.join(temp_dir, f"ocr_page_{os.getpid()}_{page_num}.png")
             img.save(temp_path)
+            logger.debug(f"Saved temporary page image to: {temp_path}")
             
             try:
                 ocr_result = ocr_engine.process(temp_path)

{xfmr_zem-0.2.7.dist-info → xfmr_zem-0.2.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: xfmr-zem
-Version: 0.2.7
+Version: 0.2.8
 Summary: Zem: Unified Data Pipeline Framework (ZenML + NeMo Curator + DataJuicer) for multi-domain processing
 Project-URL: Homepage, https://github.com/OAI-Labs/xfmr-zem
 Project-URL: Repository, https://github.com/OAI-Labs/xfmr-zem

{xfmr_zem-0.2.7.dist-info → xfmr_zem-0.2.8.dist-info}/RECORD

@@ -1,6 +1,6 @@
 xfmr_zem/__init__.py,sha256=Abx2BepsZu-e7E93N2lOgu9w0b4TBZLN6MEzCzDCn_A,1138
 xfmr_zem/cli.py,sha256=5oz4qxXthU4mXu7bSbfKreVkAvCqrieXpGoKhJBXBvk,12538
-xfmr_zem/client.py,sha256=wf9N_fILDBvWd-08TnNq3B1PqKQPhR0pvVuJq0vidk0,11435
+xfmr_zem/client.py,sha256=2PkJavZ8kMVq0dXoeZvpRODO96tWiXyT1alZLcw5RH0,12601
 xfmr_zem/schemas.py,sha256=0tHM0ftOWTWxNiqmAZn_MyIYJwF2p9brHK0MHlOMlKY,494
 xfmr_zem/server.py,sha256=EeohfqhUiCm0cGnV85H2ODZ4FLXjcTjbkdHrHuGHW4I,8363
 xfmr_zem/zenml_wrapper.py,sha256=LHgDewuPBjCl4EiU6JZVU-_lyEi-ATURDSG9Vf7PbEY,6739
@@ -18,7 +18,7 @@ xfmr_zem/servers/nemo_curator/server.py,sha256=zcHoSwxxoK_rMaDIAbEy1s8qfdp68Ue4B
 xfmr_zem/servers/ocr/engines.py,sha256=zScn4Qjxbpl2nB8UXEf3kd9l8z84TEwGs6bV5ka8Lks,10295
 xfmr_zem/servers/ocr/install_models.py,sha256=t02zpoy8djVhITOLEaRJ2mjiMrFfA9H6fpeHD3hXuio,2135
 xfmr_zem/servers/ocr/parameters.yml,sha256=UTMwtTu0Eeit0tFkYcZOxpuzD78UBlpONXZIx6STYwc,144
-xfmr_zem/servers/ocr/server.py,sha256=eJtQnMVBFX6PLZMxZITNlNEXGarjsvkz003-uT1iIo0,4369
+xfmr_zem/servers/ocr/server.py,sha256=wfk9L1776TOpFNlmc73jknEMDDobfcFgqBUhcVX2elc,4441
 xfmr_zem/servers/ocr/deepdoc_vietocr/__init__.py,sha256=XJE7RnOu5oo5p902HPWPDBd7FhVQXetmnr2-kWEG0nI,2419
 xfmr_zem/servers/ocr/deepdoc_vietocr/implementations.py,sha256=79fYr76fx8yZda3HaFcK1d5G-4sDVf1JFHNW_OBQAk8,47348
 xfmr_zem/servers/ocr/deepdoc_vietocr/layout_recognizer.py,sha256=7BeLHzf9FQUkkHMb5jDpggruJmfXVMU78MF_EeZ9PG4,10462
@@ -51,8 +51,8 @@ xfmr_zem/servers/sinks/parameters.yml,sha256=9HAnv84Utw2qWsVZH8uOjVE62lnAKBkzv4P
 xfmr_zem/servers/sinks/server.py,sha256=jI_r4sq_U_avNwF1PiE0alpaDrYpzOI-qPeLU7hgHP0,1589
 xfmr_zem/servers/unstructured/parameters.yml,sha256=N31cmc56GTr3rkVhbni4yOpbnHISReN8f-KnRZTDbBc,118
 xfmr_zem/servers/unstructured/server.py,sha256=0XmXWMAUNEJboX-J4bn_8EBUfMHIqu_ylNC_s9YOZdk,1996
-xfmr_zem-0.2.7.dist-info/METADATA,sha256=Iv77eb-eHw6rdJhG1LfoNY4Hf9I7oFlIsx1K3K7_sH0,6379
-xfmr_zem-0.2.7.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-xfmr_zem-0.2.7.dist-info/entry_points.txt,sha256=uxs-IXFxpSakHivpFN3mEr13cz-z-0vkeSF_4dEBMa4,65
-xfmr_zem-0.2.7.dist-info/licenses/LICENSE,sha256=kf_ILr0zLkSy5-EBu0VF2PGaOykYo83z3UijI-bZeAE,11342
-xfmr_zem-0.2.7.dist-info/RECORD,,
+xfmr_zem-0.2.8.dist-info/METADATA,sha256=sv4boGlSzTYgE1MlKIZieIVvRoioVKoWwTOPXhrqKeE,6379
+xfmr_zem-0.2.8.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+xfmr_zem-0.2.8.dist-info/entry_points.txt,sha256=uxs-IXFxpSakHivpFN3mEr13cz-z-0vkeSF_4dEBMa4,65
+xfmr_zem-0.2.8.dist-info/licenses/LICENSE,sha256=kf_ILr0zLkSy5-EBu0VF2PGaOykYo83z3UijI-bZeAE,11342
+xfmr_zem-0.2.8.dist-info/RECORD,,