pystou 0.1.0__tar.gz

pystou-0.1.0/.claude/settings.local.json

@@ -0,0 +1,24 @@
+{
+  "permissions": {
+    "allow": [
+      "Bash(make test:*)",
+      "Bash(python3:*)",
+      "Bash(ruff check:*)",
+      "Bash(git add:*)",
+      "Bash(git commit:*)",
+      "Bash(git stash:*)",
+      "Bash(pip install:*)",
+      "Bash(pystou --help:*)",
+      "Bash(pystou dedup --help:*)",
+      "Bash(pystou unarchive --help:*)",
+      "Bash(pip uninstall:*)",
+      "Bash(pystou unarchive:*)",
+      "Bash(PYTHONPATH=/home/dev/Repositories/pystou python3:*)",
+      "Bash(git reset:*)",
+      "Bash(git mv:*)",
+      "Bash(git rm:*)",
+      "Bash(pystou cleanup --help:*)",
+      "Bash(python -m pytest:*)"
+    ]
+  }
+}

pystou-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,47 @@
+name: CI
+
+on:
+  push:
+    branches: ['*']
+  pull_request:
+    branches: ['*']
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Install ruff
+        run: pip install ruff
+
+      - name: Lint
+        run: ruff check cleanup common dedup_folders empty extract identify pystou stats tests
+
+      - name: Format check
+        run: ruff format --check cleanup common dedup_folders empty extract identify pystou stats tests
+
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ['3.9', '3.10', '3.11', '3.12']
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install package
+        run: pip install -e ".[dev]"
+
+      - name: Run tests
+        run: python -m pytest tests -v

pystou-0.1.0/.github/workflows/release.yml

@@ -0,0 +1,28 @@
+name: Release
+
+on:
+  push:
+    branches: [main]
+
+concurrency:
+  group: release
+  cancel-in-progress: false
+
+jobs:
+  release:
+    # Only run on the canonical repo, never on forks.
+    if: github.repository == 'ICIJ/pystou'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Python Semantic Release
+        id: release
+        uses: python-semantic-release/python-semantic-release@v9
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}

pystou-0.1.0/.gitignore

@@ -0,0 +1,121 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.ruff_cache
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyderworkspace
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# SQLite3 database files
+*.sqlite3
+*.db
+
+# Superpowers (local design specs/plans, not tracked)
+docs/superpowers/

pystou-0.1.0/LICENSE

@@ -0,0 +1,7 @@
+Copyright 2024 International Consortium of Investigative Journalists
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

pystou-0.1.0/Makefile

@@ -0,0 +1,76 @@
+.PHONY: help test coverage lint format typecheck clean install build publish release-dry _check-uv
+
+SRC := cleanup common dedup_folders empty extract identify pystou stats
+
+help:
+	@echo "pystou - Python scripts for deduplicating folders and unarchiving files"
+	@echo ""
+	@echo "Usage:"
+	@echo "  make install        Sync the uv environment with dev extras"
+	@echo "  pystou dedup DIR    Find and deduplicate duplicate folders"
+	@echo "  pystou extract DIR  Extract archive files"
+	@echo "  pystou cleanup DIR  Remove junk files (.DS_Store, Thumbs.db, ...)"
+	@echo "  pystou identify DIR Identify file types and detect issues"
+	@echo "  pystou stats DIR    Show directory statistics"
+	@echo "  pystou empty DIR    Find and remove empty directories"
+	@echo ""
+	@echo "Development:"
+	@echo "  make test           Run the test suite"
+	@echo "  make coverage       Run tests with coverage report"
+	@echo "  make lint           Check code style"
+	@echo "  make format         Auto-format code"
+	@echo "  make typecheck      Run mypy"
+	@echo "  make clean          Remove cache and build files"
+	@echo ""
+	@echo "Release:"
+	@echo "  make build          Build sdist and wheel into dist/"
+	@echo "  make publish        Build and upload to PyPI (needs a token)"
+	@echo "  make release-dry    Preview the next semantic-release version"
+
+_check-uv:
+	@command -v uv >/dev/null 2>&1 || { \
+		echo "Error: uv is not installed"; \
+		echo ""; \
+		echo "Install it from https://docs.astral.sh/uv/getting-started/installation/"; \
+		exit 1; \
+	}
+
+test: _check-uv
+	@uv run --extra dev pytest tests -v
+
+coverage: _check-uv
+	@uv run --extra dev pytest --cov=. --cov-report=term-missing tests
+
+lint: _check-uv
+	@uv run --extra dev ruff check $(SRC) tests && uv run --extra dev ruff format --check $(SRC) tests && echo "Lint OK"
+
+format: _check-uv
+	@uv run --extra dev ruff check --fix $(SRC) tests && uv run --extra dev ruff format $(SRC) tests && echo "Format OK"
+
+typecheck: _check-uv
+	@uv run --extra dev mypy $(SRC) && echo "Typecheck OK"
+
+clean:
+	@rm -rf .pytest_cache .mypy_cache .ruff_cache .coverage htmlcov
+	@rm -rf dist build pystou.egg-info
+	@find . -type d -name "__pycache__" -prune -exec rm -rf {} +
+	@find . -type f -name "*.pyc" -delete
+	@echo "Cleaned"
+
+install: _check-uv
+	@uv sync --extra dev
+	@echo "Synced pystou environment with dev extras"
+
+build: _check-uv
+	@rm -rf dist build pystou.egg-info
+	@uv build
+	@echo ""
+	@echo "Built artifacts:"
+	@ls -1 dist
+
+publish: build
+	@uv publish
+	@echo "Published to PyPI"
+
+release-dry: _check-uv
+	@uvx --from python-semantic-release semantic-release version --print

pystou-0.1.0/PKG-INFO

@@ -0,0 +1,392 @@
+Metadata-Version: 2.4
+Name: pystou
+Version: 0.1.0
+Summary: Python scripts for deduplicating folders and unarchiving files.
+Project-URL: Homepage, https://github.com/ICIJ/pystou
+Project-URL: Repository, https://github.com/ICIJ/pystou
+Author: ICIJ
+License-Expression: MIT
+License-File: LICENSE
+Keywords: archive,cli,deduplication,filesystem
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.9
+Provides-Extra: dev
+Requires-Dist: mypy>=1.10; extra == 'dev'
+Requires-Dist: pytest-cov>=5.0; extra == 'dev'
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Requires-Dist: ruff>=0.5; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# PyStou
+
+Welcome to **PyStou** – your ultimate toolkit for keeping your filesystem tidy and organized! Whether you're a developer drowning in duplicate folders or someone who loves archiving files but hates the clutter, PyStou is here to rescue you from chaos with style and efficiency.
+
+**PyStou** is proudly developed by the [International Consortium of Investigative Journalists (ICIJ)](https://www.icij.org/), aiming to empower users with tools to manage and maintain large amounts of files.
+
+## Table of Contents
+
+- [Features](#features)
+- [Installation](#installation)
+  - [Prerequisites](#prerequisites)
+  - [Clone the Repository](#clone-the-repository)
+  - [Install the Package](#install-the-package)
+- [Usage](#usage)
+  - [Deduplicate Folders](#deduplicate-folders)
+  - [Extract Archives](#extract-archives)
+  - [Cleanup Junk Files](#cleanup-junk-files)
+  - [Identify File Types](#identify-file-types)
+  - [Directory Statistics](#directory-statistics)
+  - [Empty Directories](#empty-directories)
+- [Running Tests](#running-tests)
+- [License](#license)
+
+## Features
+
+- Automatically identify and manage duplicate directories, ensuring you only keep what you need.
+- Effortlessly extract a wide range of archive formats, including `.zip`, `.tar.gz`, `.zst`, and `.pst`.
+- Support for split ZIP archives (`.z01`, `.z02`, etc.) with automatic detection.
+- Nested archive extraction for archives containing other archives.
+- Parallel archive extraction for faster processing of multiple archives.
+- Remove junk files (`.DS_Store`, `Thumbs.db`, `__MACOSX`, etc.) with a single command.
+- Detect file type mismatches and encrypted archives.
+- Get comprehensive directory statistics including file counts, sizes, and types.
+- Find and remove empty directories safely.
+- Choose to interact with each file/archive or set default actions for seamless automation.
+- Keep track of all actions with detailed JSON-formatted logs for easy troubleshooting.
+- Pure native Python scripts ready to run out-of-the-box (except for necessary command-line tools).
+
+## Installation
+
+Getting started with PyStou is a breeze! Follow the steps below to install and set up the project on your machine.
+
+### Prerequisites
+
+- **Python 3.7 or higher** is required.
+- **Command-Line Tools:**
+  - **`p7zip-full`**: Required for extracting split ZIP archives (`.z01`, `.z02`, etc.).
+  - **`pst-utils`**: Required for extracting `.pst` files.
+  - **`zstd`**: Required for handling `.zst` files.
+
+### Clone the Repository
+
+```bash
+git clone https://github.com/ICIJ/pystou.git
+cd pystou
+```
+
+### Install the Package
+
+PyStou can be installed using `pip`. It includes all necessary components without additional dependencies.
+
+```bash
+pip install .
+```
+
+> **Note:** You might need to use `pip3` and/or `sudo` depending on your system configuration.
+
+## Usage
+
+PyStou provides a unified command-line interface with several subcommands.
+
+```bash
+pystou --help
+pystou dedup --help
+pystou extract --help
+pystou cleanup --help
+pystou identify --help
+pystou stats --help
+pystou empty --help
+```
+
+### Deduplicate Folders
+
+**Purpose:** Identify and manage duplicate directories to keep your filesystem clean.
+
+**Command:**
+
+```bash
+pystou dedup [directory] [options]
+```
+
+**Parameters:**
+
+- `directory`: (Optional) The root directory to start scanning from. Defaults to the current directory if not specified.
+
+**Options:**
+
+- `-r`, `--recursive`: Recursively process subdirectories.
+- `-l LEVEL`, `--level LEVEL`: Maximum depth level for recursion (default: unlimited).
+- `-c CHOICE`, `--default-choice CHOICE`: Default action to apply to all duplicate groups.
+  - `1`: Delete duplicates.
+  - `2`: Merge contents and delete duplicates.
+  - `3`: Skip (do nothing).
+- `-n`, `--dry-run`: Perform a dry run without making any changes.
+- `--log-dir LOG_DIR`: Directory to store log files (default: current directory).
+- `--db-dir DB_DIR`: Directory to store index database (default: current directory).
+
+**Examples:**
+
+- **Interactive Mode:**
+
+  ```bash
+  pystou dedup /path/to/your/folders -r
+  ```
+
+  *The script will prompt you for each duplicate group found.*
+
+- **Automated Mode with Default Choice (Delete Duplicates):**
+
+  ```bash
+  pystou dedup /path/to/your/folders -r -c 1
+  ```
+
+- **Dry Run Mode:**
+
+  ```bash
+  pystou dedup /path/to/your/folders -r -n
+  ```
+
+### Extract Archives
+
+**Purpose:** Extract various archive formats efficiently and manage them post-extraction.
+
+**Supported Formats:**
+- Standard: `.zip`, `.tar`, `.tar.gz`, `.tgz`, `.tar.bz2`, `.tbz`, `.gz`, `.bz2`
+- Zstandard: `.zst`, `.tar.zst`, `.tzst`
+- Outlook: `.pst`
+- Split ZIP: `.z01`, `.z02`, ... (automatically detected with main `.zip` file)
+
+**Command:**
+
+```bash
+pystou extract [directory] [options]
+```
+
+**Parameters:**
+
+- `directory`: (Optional) The root directory to start searching for archives. Defaults to the current directory if not specified.
+
+**Options:**
+
+- `-r`, `--recursive`: Recursively search subdirectories for archives.
+- `-c CHOICE`, `--default-choice CHOICE`: Default action to apply to all archives.
+  - `1`: Extract archives.
+  - `2`: Skip (do nothing).
+- `-dc DELETE_CHOICE`, `--default-delete-choice DELETE_CHOICE`: Default action when prompted to delete archives after extraction.
+  - `1`: Delete the archive after extraction.
+  - `2`: Keep the archive after extraction.
+- `-p N`, `--parallel N`: Number of parallel extraction workers (default: 1). Requires `-c` flag.
+- `-N`, `--nested`: Recursively extract archives found inside extracted content.
+- `--max-depth N`: Maximum nesting depth for `--nested` (default: 10).
+- `-n`, `--dry-run`: Perform a dry run without making any changes.
+- `--log-dir LOG_DIR`: Directory to store log files (default: current directory).
+- `--db-dir DB_DIR`: Directory to store index database (default: current directory).
+
+**Examples:**
+
+- **Interactive Mode:**
+
+  ```bash
+  pystou extract /path/to/archives -r
+  ```
+
+  *The script will prompt you for each archive found, asking whether to extract or skip.*
+
+- **Automated Mode with Default Choices (Extract and Delete Archives):**
+
+  ```bash
+  pystou extract /path/to/archives -r -c 1 -dc 1
+  ```
+
+- **Parallel Extraction (4 workers):**
+
+  ```bash
+  pystou extract /path/to/archives -r -c 1 -dc 2 -p 4
+  ```
+
+- **Nested Extraction (archives inside archives):**
+
+  ```bash
+  pystou extract /path/to/archives -r -c 1 -dc 1 --nested
+  ```
+
+- **Dry Run Mode:**
+
+  ```bash
+  pystou extract /path/to/archives -r -n
+  ```
+
+### Cleanup Junk Files
+
+**Purpose:** Remove common junk files created by operating systems and applications.
+
+**Removed by default:**
+- macOS: `.DS_Store`, `._.DS_Store`, `._*` files, `__MACOSX`, `.AppleDouble`, `.Spotlight-V100`, `.Trashes`, `.fseventsd`, `.TemporaryItems`, `.LSOverride`
+- Windows: `Thumbs.db`, `ehthumbs.db`, `ehthumbs_vista.db`, `desktop.ini`
+
+**Command:**
+
+```bash
+pystou cleanup [directory] [options]
+```
+
+**Options:**
+
+- `-r`, `--recursive`: Recursively process subdirectories.
+- `--include PATTERN`: Additional file/directory names to remove (can be used multiple times).
+- `--list-only`: Only list junk files without removing them.
+- `-n`, `--dry-run`: Perform a dry run without making any changes.
+
+**Examples:**
+
+- **List junk files:**
+
+  ```bash
+  pystou cleanup /path/to/folder -r --list-only
+  ```
+
+- **Remove junk files:**
+
+  ```bash
+  pystou cleanup /path/to/folder -r
+  ```
+
+- **Remove additional patterns:**
+
+  ```bash
+  pystou cleanup /path/to/folder -r --include ".gitkeep" --include "*.bak"
+  ```
+
+### Identify File Types
+
+**Purpose:** Detect file types and find potential issues like mismatched extensions or encrypted archives.
+
+**Command:**
+
+```bash
+pystou identify [directory] [options]
+```
+
+**Options:**
+
+- `-r`, `--recursive`: Recursively process subdirectories.
+- `--check-mismatch`: Check for files with mismatched extensions.
+- `--check-encrypted`: Check for encrypted ZIP archives.
+- `--check-all`: Run all checks.
+- `--extensions EXT`: Comma-separated list of extensions to check (e.g., `.zip,.pdf`).
+
+**Examples:**
+
+- **Find mismatched extensions:**
+
+  ```bash
+  pystou identify /path/to/folder -r --check-mismatch
+  ```
+
+- **Find encrypted archives:**
+
+  ```bash
+  pystou identify /path/to/folder -r --check-encrypted
+  ```
+
+- **Run all checks on specific extensions:**
+
+  ```bash
+  pystou identify /path/to/folder -r --check-all --extensions ".zip,.pdf,.docx"
+  ```
+
+### Directory Statistics
+
+**Purpose:** Display comprehensive statistics about files and directories.
+
+**Command:**
+
+```bash
+pystou stats [directory] [options]
+```
+
+**Options:**
+
+- `-r`, `--recursive`: Recursively process subdirectories.
+- `--top N`: Number of top items to show (default: 10).
+- `--by-extension`: Show breakdown by file extension.
+- `--by-size`: Show largest files.
+- `--json`: Output statistics in JSON format.
+
+**Examples:**
+
+- **Show directory statistics:**
+
+  ```bash
+  pystou stats /path/to/folder -r
+  ```
+
+- **Show largest files:**
+
+  ```bash
+  pystou stats /path/to/folder -r --by-size --top 20
+  ```
+
+- **Output as JSON:**
+
+  ```bash
+  pystou stats /path/to/folder -r --json
+  ```
+
+### Empty Directories
+
+**Purpose:** Find and remove empty directories.
+
+**Command:**
+
+```bash
+pystou empty [directory] [options]
+```
+
+**Options:**
+
+- `-r`, `--recursive`: Recursively process subdirectories.
+- `--list-only`: Only list empty directories without removing them.
+- `--include-hidden`: Include hidden directories (starting with `.`).
+- `-n`, `--dry-run`: Perform a dry run without making any changes.
+
+**Examples:**
+
+- **List empty directories:**
+
+  ```bash
+  pystou empty /path/to/folder -r --list-only
+  ```
+
+- **Remove empty directories:**
+
+  ```bash
+  pystou empty /path/to/folder -r
+  ```
+
+- **Include hidden directories:**
+
+  ```bash
+  pystou empty /path/to/folder -r --include-hidden
+  ```
+
+## Running Tests
+
+PyStou includes a suite of unit tests to ensure everything works smoothly. Here's how to run them:
+
+```bash
+make test
+```
+
+Or manually:
+
+```bash
+python3 -m unittest discover tests
+```
+
+> **Note:** Ensure you have all necessary command-line tools installed (`readpst`, `zstd`, `7z`) before running tests that involve archive extraction.
+
+## License
+
+Distributed under the [MIT License](LICENSE). See `LICENSE` for more information.