docex 0.2.0__tar.gz

docex-0.2.0/.gitignore

@@ -0,0 +1,74 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+venv/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# mypy
+.mypy_cache/
+.dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# VS Code
+.vscode/
+
+# macOS
+.DS_Store
+
+# Local files
+*.db
+*.sqlite3
+temp_download/
+example_docs/ 

docex-0.2.0/CONTRIBUTING.md

@@ -0,0 +1,46 @@
+# Contributing to DocFlow
+
+Thank you for your interest in contributing to DocFlow! We welcome contributions from the community to make this project better.
+
+## How to Contribute
+
+1. **Fork the repository** and create your branch from `main`.
+2. **Clone your fork** and set up a virtual environment.
+3. **Install dependencies** using `pip install -r requirements.txt`.
+4. **Create a new branch** for your feature or bugfix:
+   ```sh
+   git checkout -b my-feature
+   ```
+5. **Make your changes** and add tests as appropriate.
+6. **Run the test suite** to ensure nothing is broken:
+   ```sh
+   pytest
+   ```
+7. **Commit your changes** and push your branch to GitHub.
+8. **Open a Pull Request** with a clear description of your changes.
+
+## Code Style
+
+- Follow [PEP8](https://www.python.org/dev/peps/pep-0008/) for Python code style.
+- Use type hints where possible.
+- Write clear, concise docstrings for all public classes and methods.
+- Please review design document to understand overall architecture concept, in particular, seperation of concerns with layer implementation. 
+
+## Pull Requests
+
+- Ensure your branch is up to date with `main` before submitting a PR.
+- Reference related issues in your PR description.
+- Include tests for new features and bugfixes.
+- Be responsive to code review feedback.
+
+## Reporting Issues
+
+- Use the [GitHub Issues](https://github.com/tommyGPT2S/DocFlow/issues) page to report bugs or request features.
+- Please provide as much detail as possible, including steps to reproduce, logs, and your environment.
+
+## Community
+
+- Be respectful and inclusive in all interactions.
+- Help others by reviewing PRs and answering questions.
+
+Thank you for helping make DocFlow better! 

docex-0.2.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Tommy Jiang
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

docex-0.2.0/MANIFEST.in

@@ -0,0 +1,27 @@
+# Include documentation
+include README.md
+include LICENSE
+include docs/DocFlow_Design.md
+include docs/Developer_Guide.md
+include docs/New Era of Supply Chain/image.png
+
+# Include configuration files
+include docflow/config/default_config.yaml
+
+# Include database schema
+include docflow/db/sqlite_schema.sql
+
+# Include examples
+recursive-include examples *
+
+# Include test files
+include test_simple.py
+include test_db_connection.py
+include test_sqlite.py
+include test_transport_workflow.py
+include verify_routes.py
+
+# Include any other important files
+include .env.example
+include .gitignore
+include CONTRIBUTING.md

docex-0.2.0/PKG-INFO

@@ -0,0 +1,149 @@
+Metadata-Version: 2.4
+Name: docex
+Version: 0.2.0
+Summary: A robust, lightweight, and developer-friendly document management and transport system for Python.
+Home-page: https://github.com/tommyGPT2S/DocFlow
+Author: Tommy Jiang
+Author-email: Tommy Jiang <tommyscosc@scos.ai>
+License: MIT License
+        
+        Copyright (c) 2025 Tommy Jiang
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+        
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: sqlalchemy>=2.0.0
+Requires-Dist: psycopg2-binary>=2.9.0
+Requires-Dist: pyyaml>=6.0.0
+Requires-Dist: pytest>=7.0.0
+Requires-Dist: boto3>=1.26.0
+Requires-Dist: pydantic>=2.0.0
+Requires-Dist: paramiko>=3.4.0
+Requires-Dist: aiohttp>=3.9.0
+Requires-Dist: python-dotenv>=1.0.0
+Requires-Dist: click>=8.0.0
+Dynamic: author
+Dynamic: home-page
+Dynamic: license-file
+Dynamic: requires-python
+
+# DocFlow
+
+<!-- Badges -->
+![License](https://img.shields.io/github/license/tommyGPT2S/DocFlow)
+![Python](https://img.shields.io/pypi/pyversions/docflow)
+![Build](https://github.com/tommyGPT2S/DocFlow/actions/workflows/ci.yml/badge.svg)
+<!-- Add PyPI badge here when ready -->
+
+![DocFlow Architecture](docs/New%20Era%20of%20Supply%20Chain/image.png)
+
+**DocFlow** is a robust, extensible document management and transport system for Python. It supports multiple storage backends, metadata management, and operation tracking, with a unified API for local, SFTP, HTTP, and other protocols.
+
+## Features
+
+- 📁 Document storage and metadata management
+- 🔄 Transport layer with pluggable protocols (local, SFTP, HTTP, etc.)
+- 🛣️ Configurable transport routes and routing rules
+- 📝 Operation and audit tracking
+- 🧩 Extensible architecture for new protocols and workflows
+
+## Installation
+
+Install from PyPI:
+
+```sh
+pip install pydocflow
+```
+
+If you want to use PDF processing features (e.g., custom PDF processors), also install:
+
+```sh
+pip install pdfminer.six
+```
+
+## Quick Start
+
+Before using DocFlow in your code, you must initialize the system using the CLI:
+
+```sh
+# Run this once to set up configuration and database
+$ docflow init
+```
+
+Then you can use the Python API (minimal example):
+
+```python
+from docflow import DocFlow
+from pathlib import Path
+
+# Create DocFlow instance (will check initialization internally)
+docflow = DocFlow()
+
+# Create a basket
+basket = docflow.basket('mybasket')
+
+# Create a simple text file
+hello_file = Path('hello.txt')
+hello_file.write_text('Hello scos.ai!')
+
+# Add the document to the basket
+doc = basket.add(str(hello_file))
+
+# Print document details
+print(doc.get_details())
+
+hello_file.unlink()
+```
+Additional examples can be found in examples/ folder.
+
+## Configuration
+
+Configure routes and storage in `default_config.yaml`:
+
+```yaml
+transport_config:
+  routes:
+    - name: local_backup
+      purpose: backup
+      protocol: local
+      config:
+        type: local
+        name: local_backup_transport
+        base_path: /path/to/backup
+        create_dirs: true
+      can_upload: true
+      can_download: true
+      enabled: true
+  default_route: local_backup
+```
+
+## Documentation
+
+- [Developer Guide](docs/Developer_Guide.md)
+- [Design Document](docs/DocFlow%20Design.md)
+
+## Contributing
+
+Contributions are welcome! Please see [CONTRIBUTING.md](CONTRIBUTING.md) for guidelines.
+
+## License
+
+[MIT License](LICENSE)

docex-0.2.0/README.md

@@ -0,0 +1,102 @@
+# DocFlow
+
+<!-- Badges -->
+![License](https://img.shields.io/github/license/tommyGPT2S/DocFlow)
+![Python](https://img.shields.io/pypi/pyversions/docflow)
+![Build](https://github.com/tommyGPT2S/DocFlow/actions/workflows/ci.yml/badge.svg)
+<!-- Add PyPI badge here when ready -->
+
+![DocFlow Architecture](docs/New%20Era%20of%20Supply%20Chain/image.png)
+
+**DocFlow** is a robust, extensible document management and transport system for Python. It supports multiple storage backends, metadata management, and operation tracking, with a unified API for local, SFTP, HTTP, and other protocols.
+
+## Features
+
+- 📁 Document storage and metadata management
+- 🔄 Transport layer with pluggable protocols (local, SFTP, HTTP, etc.)
+- 🛣️ Configurable transport routes and routing rules
+- 📝 Operation and audit tracking
+- 🧩 Extensible architecture for new protocols and workflows
+
+## Installation
+
+Install from PyPI:
+
+```sh
+pip install pydocflow
+```
+
+If you want to use PDF processing features (e.g., custom PDF processors), also install:
+
+```sh
+pip install pdfminer.six
+```
+
+## Quick Start
+
+Before using DocFlow in your code, you must initialize the system using the CLI:
+
+```sh
+# Run this once to set up configuration and database
+$ docflow init
+```
+
+Then you can use the Python API (minimal example):
+
+```python
+from docflow import DocFlow
+from pathlib import Path
+
+# Create DocFlow instance (will check initialization internally)
+docflow = DocFlow()
+
+# Create a basket
+basket = docflow.basket('mybasket')
+
+# Create a simple text file
+hello_file = Path('hello.txt')
+hello_file.write_text('Hello scos.ai!')
+
+# Add the document to the basket
+doc = basket.add(str(hello_file))
+
+# Print document details
+print(doc.get_details())
+
+hello_file.unlink()
+```
+Additional examples can be found in examples/ folder.
+
+## Configuration
+
+Configure routes and storage in `default_config.yaml`:
+
+```yaml
+transport_config:
+  routes:
+    - name: local_backup
+      purpose: backup
+      protocol: local
+      config:
+        type: local
+        name: local_backup_transport
+        base_path: /path/to/backup
+        create_dirs: true
+      can_upload: true
+      can_download: true
+      enabled: true
+  default_route: local_backup
+```
+
+## Documentation
+
+- [Developer Guide](docs/Developer_Guide.md)
+- [Design Document](docs/DocFlow%20Design.md)
+
+## Contributing
+
+Contributions are welcome! Please see [CONTRIBUTING.md](CONTRIBUTING.md) for guidelines.
+
+## License
+
+[MIT License](LICENSE)

docex-0.2.0/docex/__init__.py

@@ -0,0 +1,32 @@
+"""
+DocEX - Document Management Library
+
+This library provides a robust, extensible document management and transport system.
+"""
+
+from docex.docex import DocEX
+
+# Setup DocEX with configuration
+DocEX.setup(
+    database={
+        'type': 'sqlite',
+        'sqlite': {'path': 'docex.db'}
+    }
+)
+
+# Create DocEX instance
+docex = DocEX()
+basket = docex.create_basket('my_basket')
+
+# Get available metadata keys
+print(DocEX.get_metadata_keys())
+
+__all__ = [
+    'DocEX',          # Main entry point
+    'DocBasket',      # Document basket
+    'Document',       # Document class
+    'Route',          # Transport route
+    'UserContext',    # User context for auditing
+]
+
+__version__ = '0.1.9'

docex-0.2.0/docex/cli.py

@@ -0,0 +1,180 @@
+"""
+DocEX CLI commands
+
+This module provides command-line interface for DocEX operations.
+"""
+
+import click
+import os
+from pathlib import Path
+import yaml
+from docex import DocEX
+from docex.config.docflow_config import DocFlowConfig
+from docex.db.connection import Database
+from docex.db.models import Base
+from docex.transport.models import Base as TransportBase
+import sqlite3
+from datetime import datetime, UTC
+from sqlalchemy import inspect
+import logging
+import sys
+from typing import Optional
+
+# Configure logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+@click.group()
+def cli():
+    """DocEX command-line interface"""
+    pass
+
+@cli.command()
+@click.option('--config', type=click.Path(exists=True), help='Path to configuration file')
+@click.option('--force', is_flag=True, help='Force reinitialization')
+@click.option('--db-type', type=click.Choice(['sqlite', 'postgresql']), help='Database type')
+@click.option('--db-path', type=click.Path(), help='Path to SQLite database file')
+@click.option('--db-host', help='Database host')
+@click.option('--db-port', type=int, help='Database port')
+@click.option('--db-name', help='Database name')
+@click.option('--db-user', help='Database user')
+@click.option('--db-password', help='Database password')
+@click.option('--storage-path', type=click.Path(), help='Path to storage directory')
+@click.option('--log-level', type=click.Choice(['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']), help='Logging level')
+def init(config: Optional[str], force: bool, **kwargs):
+    """Initialize DocEX with configuration"""
+    try:
+        # Check if already initialized
+        is_initialized = DocEX.is_initialized()
+        
+        if is_initialized and not force:
+            if not click.confirm('DocEX is already initialized. Do you want to reinitialize? This will drop all existing data.'):
+                return
+                
+        # Load configuration
+        if config:
+            config_path = Path(config)
+            if not config_path.exists():
+                click.echo(f'Configuration file not found: {config}', err=True)
+                return
+                
+            user_config = DocFlowConfig.from_file(config)
+        else:
+            user_config = DocEX.get_defaults()
+            
+        # Override with command line options
+        if kwargs.get('db_type'):
+            user_config['database']['type'] = kwargs['db_type']
+            
+        if kwargs.get('db_path'):
+            user_config['database']['sqlite']['path'] = kwargs['db_path']
+            
+        if kwargs.get('db_host'):
+            user_config['database']['postgres']['host'] = kwargs['db_host']
+            
+        if kwargs.get('db_port'):
+            user_config['database']['postgres']['port'] = kwargs['db_port']
+            
+        if kwargs.get('db_name'):
+            user_config['database']['postgres']['database'] = kwargs['db_name']
+            
+        if kwargs.get('db_user'):
+            user_config['database']['postgres']['user'] = kwargs['db_user']
+            
+        if kwargs.get('db_password'):
+            user_config['database']['postgres']['password'] = kwargs['db_password']
+            
+        if kwargs.get('storage_path'):
+            user_config['storage']['filesystem']['path'] = kwargs['storage_path']
+            
+        if kwargs.get('log_level'):
+            user_config['logging']['level'] = kwargs['log_level']
+            
+        # Initialize DocEX
+        DocEX.setup(**user_config)
+        
+        # Save configuration
+        config_path = Path.home() / '.docex' / 'config.yaml'
+        config_path.parent.mkdir(parents=True, exist_ok=True)
+        
+        with open(config_path, 'w') as f:
+            yaml.dump(user_config, f)
+            
+        # Create DocEX instance for verification
+        docex = DocEX()
+        
+        click.echo('\nDocEX initialized successfully!')
+        
+    except Exception as e:
+        click.echo(f'Error initializing DocEX: {str(e)}', err=True)
+        sys.exit(1)
+
+@cli.group()
+def processor():
+    """Manage document processors"""
+    pass
+
+@processor.command('register')
+@click.option('--name', required=True, help='Processor class name (must match Python class)')
+@click.option('--type', required=True, help='Processor type (e.g., format_converter, content_processor)')
+@click.option('--description', default='', help='Description of the processor')
+@click.option('--config', default='{}', help='JSON string for processor config')
+@click.option('--enabled/--disabled', default=True, help='Enable or disable the processor')
+def register_processor(name, type, description, config, enabled):
+    """Register a new processor in the database"""
+    from docex.db.models import Processor
+    import json
+    db = Database()
+    with db.session() as session:
+        if session.query(Processor).filter_by(name=name).first():
+            click.echo(f"Processor '{name}' already exists.")
+            return
+        try:
+            config_dict = json.loads(config)
+        except Exception as e:
+            click.echo(f"Invalid config JSON: {e}")
+            return
+        processor = Processor(
+            name=name,
+            type=type,
+            description=description,
+            config=config_dict,
+            enabled=enabled,
+            created_at=datetime.now(UTC),
+            updated_at=datetime.now(UTC)
+        )
+        session.add(processor)
+        session.commit()
+        click.echo(f"Processor '{name}' registered successfully.")
+
+@processor.command('remove')
+@click.option('--name', required=True, help='Processor class name to remove')
+def remove_processor(name):
+    """Remove a processor from the database"""
+    from docex.db.models import Processor
+    db = Database()
+    with db.session() as session:
+        processor = session.query(Processor).filter_by(name=name).first()
+        if not processor:
+            click.echo(f"Processor '{name}' not found.")
+            return
+        session.delete(processor)
+        session.commit()
+        click.echo(f"Processor '{name}' removed successfully.")
+
+@processor.command('list')
+def list_processors():
+    """List all registered processors"""
+    from docex.db.models import Processor
+    db = Database()
+    with db.session() as session:
+        processors = session.query(Processor).all()
+        if not processors:
+            click.echo("No processors registered.")
+            return
+        click.echo("Registered Processors:")
+        for p in processors:
+            click.echo(f"- {p.name} | Type: {p.type} | Enabled: {p.enabled} | Description: {p.description}")
+
+if __name__ == '__main__':
+    cli()