unstructured-ingest 0.3.15__py3-none-any.whl → 0.4.1__py3-none-any.whl

test/integration/connectors/test_confluence.py

@@ -30,10 +30,10 @@ async def test_confluence_source(temp_dir):
     spaces = ["testteamsp", "MFS"]
 
     # Create connection and indexer configurations
-    access_config = ConfluenceAccessConfig(api_token=api_token)
+    access_config = ConfluenceAccessConfig(password=api_token)
     connection_config = ConfluenceConnectionConfig(
         url=confluence_url,
-        user_email=user_email,
+        username=user_email,
         access_config=access_config,
     )
     index_config = ConfluenceIndexerConfig(
@@ -77,10 +77,10 @@ async def test_confluence_source_large(temp_dir):
     spaces = ["testteamsp1"]
 
     # Create connection and indexer configurations
-    access_config = ConfluenceAccessConfig(api_token=api_token)
+    access_config = ConfluenceAccessConfig(password=api_token)
     connection_config = ConfluenceConnectionConfig(
         url=confluence_url,
-        user_email=user_email,
+        username=user_email,
         access_config=access_config,
     )
     index_config = ConfluenceIndexerConfig(

test/integration/connectors/utils/validation/equality.py

@@ -1,10 +1,11 @@
 import json
 from pathlib import Path
 
-import ndjson
 from bs4 import BeautifulSoup
 from deepdiff import DeepDiff
 
+from unstructured_ingest.utils import ndjson
+
 
 def json_equality_check(expected_filepath: Path, current_filepath: Path) -> bool:
     with expected_filepath.open() as f:

test/unit/v2/connectors/sql/test_sql.py

@@ -0,0 +1,74 @@
+from pathlib import Path
+
+import pytest
+from pytest_mock import MockerFixture
+
+from unstructured_ingest.v2.interfaces.file_data import FileData, SourceIdentifiers
+from unstructured_ingest.v2.processes.connectors.sql.sql import SQLUploadStager
+
+
+@pytest.fixture
+def mock_instance() -> SQLUploadStager:
+    return SQLUploadStager()
+
+
+@pytest.mark.parametrize(
+    ("input_filepath", "output_filename", "expected"),
+    [
+        (
+            "/path/to/input_file.ndjson",
+            "output_file.ndjson",
+            "output_file.ndjson",
+        ),
+        ("input_file.txt", "output_file.json", "output_file.txt"),
+        ("/path/to/input_file.json", "output_file", "output_file.json"),
+    ],
+)
+def test_run_output_filename_suffix(
+    mocker: MockerFixture,
+    mock_instance: SQLUploadStager,
+    input_filepath: str,
+    output_filename: str,
+    expected: str,
+):
+    output_dir = Path("/tmp/test/output_dir")
+
+    # Mocks
+    mock_get_data = mocker.patch(
+        "unstructured_ingest.v2.processes.connectors.sql.sql.get_data",
+        return_value=[{"key": "value"}, {"key": "value2"}],
+    )
+    mock_conform_dict = mocker.patch.object(
+        SQLUploadStager, "conform_dict", side_effect=lambda element_dict, file_data: element_dict
+    )
+    mock_conform_dataframe = mocker.patch.object(
+        SQLUploadStager, "conform_dataframe", side_effect=lambda df: df
+    )
+    mock_get_output_path = mocker.patch.object(
+        SQLUploadStager, "get_output_path", return_value=output_dir / expected
+    )
+    mock_write_output = mocker.patch(
+        "unstructured_ingest.v2.processes.connectors.sql.sql.write_data", return_value=None
+    )
+
+    # Act
+    result = mock_instance.run(
+        elements_filepath=Path(input_filepath),
+        file_data=FileData(
+            identifier="test",
+            connector_type="test",
+            source_identifiers=SourceIdentifiers(filename=input_filepath, fullpath=input_filepath),
+        ),
+        output_dir=output_dir,
+        output_filename=output_filename,
+    )
+
+    # Assert
+    mock_get_data.assert_called_once_with(path=Path(input_filepath))
+    assert mock_conform_dict.call_count == 2
+    mock_conform_dataframe.assert_called_once()
+    mock_get_output_path.assert_called_once_with(output_filename=expected, output_dir=output_dir)
+    mock_write_output.assert_called_once_with(
+        path=output_dir / expected, data=[{"key": "value"}, {"key": "value2"}]
+    )
+    assert result.name == expected

test/unit/v2/connectors/test_confluence.py

@@ -11,10 +11,10 @@ def test_connection_config_multiple_auth():
     with pytest.raises(ValidationError):
         ConfluenceConnectionConfig(
             access_config=ConfluenceAccessConfig(
-                api_token="api_token",
-                access_token="access_token",
+                password="api_token",
+                token="access_token",
             ),
-            user_email="user_email",
+            username="user_email",
             url="url",
         )
 
@@ -26,14 +26,14 @@ def test_connection_config_no_auth():
 
 def test_connection_config_basic_auth():
     ConfluenceConnectionConfig(
-        access_config=ConfluenceAccessConfig(api_token="api_token"),
+        access_config=ConfluenceAccessConfig(password="api_token"),
         url="url",
-        user_email="user_email",
+        username="user_email",
     )
 
 
 def test_connection_config_pat_auth():
     ConfluenceConnectionConfig(
-        access_config=ConfluenceAccessConfig(access_token="access_token"),
+        access_config=ConfluenceAccessConfig(token="access_token"),
         url="url",
     )

unstructured_ingest/__version__.py

@@ -1 +1 @@
-__version__ = "0.3.15"  # pragma: no cover
+__version__ = "0.4.1"  # pragma: no cover

unstructured_ingest/utils/data_prep.py

@@ -4,9 +4,9 @@ from datetime import datetime
 from pathlib import Path
 from typing import Any, Generator, Iterable, Optional, Sequence, TypeVar, cast
 
-import ndjson
 import pandas as pd
 
+from unstructured_ingest.utils import ndjson
 from unstructured_ingest.v2.logger import logger
 
 DATE_FORMATS = ("%Y-%m-%d", "%Y-%m-%dT%H:%M:%S", "%Y-%m-%d+%H:%M:%S", "%Y-%m-%dT%H:%M:%S%z")
@@ -153,6 +153,16 @@ def get_data_by_suffix(path: Path) -> list[dict]:
             raise ValueError(f"Unsupported file type: {path}")
 
 
+def write_data(path: Path, data: list[dict], indent: int = 2) -> None:
+    with path.open("w") as f:
+        if path.suffix == ".json":
+            json.dump(data, f, indent=indent, ensure_ascii=False)
+        elif path.suffix == ".ndjson":
+            ndjson.dump(data, f, ensure_ascii=False)
+        else:
+            raise IOError("Unsupported file type: {path}")
+
+
 def get_data(path: Path) -> list[dict]:
     try:
         return get_data_by_suffix(path=path)
@@ -179,8 +189,6 @@ def get_data(path: Path) -> list[dict]:
         except Exception as e:
             logger.warning(f"failed to read {path} as parquet: {e}")
 
-    raise IOError(f"File could not be parsed: {path}")
-
 
 def get_data_df(path: Path) -> pd.DataFrame:
     with path.open() as f:

unstructured_ingest/utils/html.py

@@ -0,0 +1,109 @@
+import base64
+from pathlib import Path
+from typing import Optional
+from urllib.parse import urlparse
+from uuid import NAMESPACE_DNS, uuid5
+
+import requests
+from bs4 import BeautifulSoup
+from requests import Session
+
+from unstructured_ingest.v2.interfaces import DownloadResponse, FileData, SourceIdentifiers
+from unstructured_ingest.v2.logger import logger
+
+
+def convert_image_tags(url: str, original_html: str, session: Optional[Session] = None) -> str:
+    session = session or requests.Session()
+    parsed_url = urlparse(url)
+    base_url = parsed_url.scheme + "://" + parsed_url.netloc
+    soup = BeautifulSoup(original_html, "html.parser")
+    images = soup.find_all("img")
+    for image in images:
+        current_source = image["src"]
+        if current_source.startswith("//"):
+            source_url = f"{parsed_url.scheme}:{current_source}"
+        elif current_source.startswith("http"):
+            source_url = current_source
+        else:
+            source_url = base_url + current_source
+        try:
+            response = session.get(source_url)
+            response.raise_for_status()
+            image_content = response.content
+            logger.debug(
+                "img tag having src updated from {} to base64 content".format(image["src"])
+            )
+            image["src"] = f"data:image/png;base64,{base64.b64encode(image_content).decode()}"
+        except Exception as e:
+            logger.warning(
+                f"failed to download image content from {source_url}: {e}", exc_info=True
+            )
+    return str(soup)
+
+
+def download_link(
+    download_dir: Path, link: str, session: Optional[Session] = None, force_download: bool = False
+) -> Path:
+    session = session or requests.Session()
+    filename = Path(urlparse(url=link).path).name
+    download_path = download_dir / filename
+    logger.debug(f"downloading file from {link} to {download_path}")
+    if download_path.exists() and download_path.is_file() and not force_download:
+        return download_path
+    with download_path.open("wb") as downloaded_file:
+        response = session.get(link)
+        response.raise_for_status()
+        downloaded_file.write(response.content)
+    return download_path
+
+
+def download_embedded_files(
+    download_dir: Path,
+    original_filedata: FileData,
+    original_html: str,
+    session: Optional[Session] = None,
+    force_download: bool = False,
+) -> list[DownloadResponse]:
+    session = session or requests.Session()
+    url = original_filedata.metadata.url
+    parsed_url = urlparse(url)
+    base_url = parsed_url.scheme + "://" + parsed_url.netloc
+    soup = BeautifulSoup(original_html, "html.parser")
+    tags = soup.find_all("a", href=True)
+    hrefs = [
+        tag["href"]
+        for tag in tags
+        if not tag["href"].startswith("#") and Path(tag["href"]).suffix != ""
+    ]
+    results = []
+    for current_source in hrefs:
+        download_dir.mkdir(parents=True, exist_ok=True)
+        if current_source.startswith("//"):
+            source_url = f"{parsed_url.scheme}:{current_source}"
+        elif current_source.startswith("http"):
+            source_url = current_source
+        else:
+            source_url = base_url + current_source
+        try:
+            downloaded_path = download_link(
+                download_dir=download_dir,
+                link=source_url,
+                session=session,
+                force_download=force_download,
+            )
+        except Exception as e:
+            logger.warning(f"failed to download file content from {source_url}: {e}")
+            continue
+        result_file_data = original_filedata.model_copy(deep=True)
+        result_file_data.metadata.url = source_url
+        result_file_data.metadata.record_locator["parent_url"] = url
+        result_file_data.identifier = str(
+            uuid5(NAMESPACE_DNS, source_url + original_filedata.identifier)
+        )
+        filename = Path(urlparse(url=source_url).path).name
+        result_file_data.source_identifiers = SourceIdentifiers(
+            filename=filename, fullpath=filename
+        )
+        result_file_data.local_download_path = downloaded_path.as_posix()
+        results.append(DownloadResponse(file_data=result_file_data, path=downloaded_path))
+    return results

unstructured_ingest/utils/ndjson.py

@@ -0,0 +1,52 @@
+import json
+from typing import IO, Any
+
+
+def dumps(obj: list[dict[str, Any]], **kwargs) -> str:
+    return "\n".join(json.dumps(each, **kwargs) for each in obj)
+
+
+def dump(obj: list[dict[str, Any]], fp: IO, **kwargs) -> None:
+    # Indent breaks ndjson formatting
+    kwargs["indent"] = None
+    text = dumps(obj, **kwargs)
+    fp.write(text)
+
+
+def loads(s: str, **kwargs) -> list[dict[str, Any]]:
+    return [json.loads(line, **kwargs) for line in s.splitlines()]
+
+
+def load(fp: IO, **kwargs) -> list[dict[str, Any]]:
+    return loads(fp.read(), **kwargs)
+
+
+class writer(object):
+    def __init__(self, f, **kwargs):
+        self.f = f
+        self.kwargs = kwargs
+
+    def write(self, row):
+        stringified = json.dumps(row, **self.kwargs)
+        self.f.write(stringified + "\n")
+
+
+class reader(object):
+    def __init__(self, f, **kwargs):
+        self.f = f
+        self.kwargs = kwargs
+
+    def __iter__(self):
+        return self
+
+    def __next__(self):
+        line = ""
+
+        while line == "":
+            line = next(self.f).strip()
+
+        return json.loads(line, **self.kwargs)
+
+    # NOTE: this is necessary to comply with py27
+    def next(self):
+        return self.__next__()

unstructured_ingest/v2/interfaces/upload_stager.py

@@ -2,11 +2,11 @@ import json
 from abc import ABC
 from dataclasses import dataclass
 from pathlib import Path
-from typing import Any, Optional, TypeVar
+from typing import Any, TypeVar
 
-import ndjson
 from pydantic import BaseModel
 
+from unstructured_ingest.utils import ndjson
 from unstructured_ingest.v2.interfaces.file_data import FileData
 from unstructured_ingest.v2.interfaces.process import BaseProcess
 
@@ -22,16 +22,6 @@ UploadStagerConfigT = TypeVar("UploadStagerConfigT", bound=UploadStagerConfig)
 class UploadStager(BaseProcess, ABC):
     upload_stager_config: UploadStagerConfigT
 
-    def write_output(self, output_path: Path, data: list[dict], indent: Optional[int] = 2) -> None:
-        if output_path.suffix == ".json":
-            with output_path.open("w") as f:
-                json.dump(data, f, indent=indent)
-        elif output_path.suffix == ".ndjson":
-            with output_path.open("w") as f:
-                ndjson.dump(data, f)
-        else:
-            raise ValueError(f"Unsupported output format: {output_path}")
-
     def conform_dict(self, element_dict: dict, file_data: FileData) -> dict:
         return element_dict
 
@@ -49,7 +39,7 @@ class UploadStager(BaseProcess, ABC):
                 writer = ndjson.writer(out_f)
                 for element in reader:
                     conformed_element = self.conform_dict(element_dict=element, file_data=file_data)
-                    writer.writerow(row=conformed_element)
+                    writer.write(row=conformed_element)
                     writer.f.flush()
 
     def process_whole(self, input_file: Path, output_file: Path, file_data: FileData) -> None:

unstructured_ingest/v2/pipeline/steps/chunk.py

@@ -1,10 +1,10 @@
 import asyncio
 import hashlib
-import json
 from dataclasses import dataclass
 from pathlib import Path
 from typing import Callable, Optional, TypedDict
 
+from unstructured_ingest.utils.data_prep import write_data
 from unstructured_ingest.v2.interfaces import FileData
 from unstructured_ingest.v2.interfaces.file_data import file_data_from_file
 from unstructured_ingest.v2.logger import logger
@@ -44,9 +44,8 @@ class ChunkStep(PipelineStep):
         return filepath
 
     def _save_output(self, output_filepath: str, chunked_content: list[dict]):
-        with open(str(output_filepath), "w") as f:
-            logger.debug(f"writing chunker output to: {output_filepath}")
-            json.dump(chunked_content, f, indent=2)
+        logger.debug(f"writing chunker output to: {output_filepath}")
+        write_data(path=Path(output_filepath), data=chunked_content)
 
     async def _run_async(
         self, fn: Callable, path: str, file_data_path: str, **kwargs

unstructured_ingest/v2/pipeline/steps/embed.py

@@ -1,10 +1,10 @@
 import asyncio
 import hashlib
-import json
 from dataclasses import dataclass
 from pathlib import Path
 from typing import Callable, Optional, TypedDict
 
+from unstructured_ingest.utils.data_prep import write_data
 from unstructured_ingest.v2.interfaces import FileData
 from unstructured_ingest.v2.interfaces.file_data import file_data_from_file
 from unstructured_ingest.v2.logger import logger
@@ -44,9 +44,8 @@ class EmbedStep(PipelineStep):
         return filepath
 
     def _save_output(self, output_filepath: str, embedded_content: list[dict]):
-        with open(str(output_filepath), "w") as f:
-            logger.debug(f"writing embedded output to: {output_filepath}")
-            json.dump(embedded_content, f, indent=2)
+        logger.debug(f"writing embedded output to: {output_filepath}")
+        write_data(path=Path(output_filepath), data=embedded_content)
 
     async def _run_async(self, fn: Callable, path: str, file_data_path: str) -> EmbedStepResponse:
         path = Path(path)

unstructured_ingest/v2/pipeline/steps/partition.py

@@ -1,10 +1,10 @@
 import asyncio
 import hashlib
-import json
 from dataclasses import dataclass
 from pathlib import Path
 from typing import Callable, Optional, TypedDict
 
+from unstructured_ingest.utils.data_prep import write_data
 from unstructured_ingest.v2.interfaces import FileData
 from unstructured_ingest.v2.interfaces.file_data import file_data_from_file
 from unstructured_ingest.v2.logger import logger
@@ -44,9 +44,8 @@ class PartitionStep(PipelineStep):
         return filepath
 
     def _save_output(self, output_filepath: str, partitioned_content: list[dict]):
-        with open(str(output_filepath), "w") as f:
-            logger.debug(f"writing partitioned output to: {output_filepath}")
-            json.dump(partitioned_content, f, indent=2)
+        logger.debug(f"writing partitioned output to: {output_filepath}")
+        write_data(path=Path(output_filepath), data=partitioned_content)
 
     async def _run_async(
         self, fn: Callable, path: str, file_data_path: str