dmcaed 2026.4.18__tar.gz

dmcaed-2026.4.18/.envrc

@@ -0,0 +1,3 @@
+export VIRTUAL_ENV=.venv
+layout python python3.11
+dotenv

dmcaed-2026.4.18/.gitignore

@@ -0,0 +1,5 @@
+DMCA.db
+exports/
+__pycache__/
+dist/
+.env

dmcaed-2026.4.18/.vscode/settings.json

@@ -0,0 +1,6 @@
+{
+    "cSpell.words": [
+        "dmca",
+        "dmcaed"
+    ]
+}

dmcaed-2026.4.18/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Canarduck
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

dmcaed-2026.4.18/PKG-INFO

@@ -0,0 +1,113 @@
+Metadata-Version: 2.4
+Name: dmcaed
+Version: 2026.4.18
+Summary: Check if a software URL is found in takedown notices.
+Project-URL: Homepage, https://codeberg.org/canarduck/dmcaed
+Project-URL: Issues, https://codeberg.org/canarduck/dmcaed/issues
+Author-email: Renaud Canarduck <renaud@canarduck.com>
+License-Expression: MIT
+License-File: LICENSE
+Classifier: Development Status :: 3 - Alpha
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.11
+Provides-Extra: dev
+Requires-Dist: black; extra == 'dev'
+Requires-Dist: mypy; extra == 'dev'
+Provides-Extra: tests
+Requires-Dist: coverage; extra == 'tests'
+Description-Content-Type: text/markdown
+
+# DMCAed
+
+A command-line tool to analyze GitHub DMCA takedown notices. It downloads, parses, and
+indexes takedown data from the [GitHub DMCA repository](https://github.com/github/dmca)
+to allow querying **which repository URL can be found** in takedown requests, please
+note that this only means the repository URL _might_ have been subject to takedown
+requests.
+
+## Installation
+
+DMCAed requires at least python 3.11.
+
+```bash
+pip install dmcaed
+```
+
+## CLI
+
+```bash
+dmcaed [--verbose] [command]
+```
+
+### Commands
+
+- `download`: Download the latest DMCA notices from GitHub
+- `analyze`: Parse downloaded notices and extract takedown data
+- `db`: Rebuild the complete DMCA database (download + analyze)
+- `stats`: Show statistics about the database content
+- `notices <url>`: List takedown notices for a specific repository URL
+
+### Options
+
+- `--verbose`: Enable debug logging
+- `-h`, `--help`: Show help message
+
+## Library Usage
+
+```python
+from dmcaed import DMCAed
+
+dmca = DMCAed()
+notices = dmca.notices("https://github.com/user/repo")
+
+if notices:
+    print(f"https://github.com/user/repo _might_ be subject to a takedown: {notices}")
+```
+
+## How it works
+
+- Download the latest DMCA notices from GitHub
+- Extract repository URLs (path to files, issues, releases, etc. are truncated)
+  mentioned in takedown notices
+- Build a local SQLite database mapping repository URLs to takedown notices
+
+## Build and Release
+
+### Prerequisites
+
+The project uses `hatch` build system and dynamic CalVer versioning based on the 
+current date.
+
+```bash
+pip install hatch hatch-calver
+```
+
+Git clone and install DMCAed.
+
+```bash
+git clone ssh://git@codeberg.org/canarduck/dmcaed.git
+pip install -e .
+```
+
+### Update the package and publish it on PyPI
+
+You first need to fetch DMCAs and create/update the database.
+
+```bash
+dmcaed db
+```
+
+Then use hatch to update version number and build the package.
+
+```bash
+hatch version release
+# use `hatch version release,fix` in case you need to fix an already released package
+```
+
+Build the package and publish it on PyPI.
+
+```bash
+hatch build
+hatch publish  --auth PYPI_AUTH_TOKEN
+```

dmcaed-2026.4.18/README.md

@@ -0,0 +1,93 @@
+# DMCAed
+
+A command-line tool to analyze GitHub DMCA takedown notices. It downloads, parses, and
+indexes takedown data from the [GitHub DMCA repository](https://github.com/github/dmca)
+to allow querying **which repository URL can be found** in takedown requests, please
+note that this only means the repository URL _might_ have been subject to takedown
+requests.
+
+## Installation
+
+DMCAed requires at least python 3.11.
+
+```bash
+pip install dmcaed
+```
+
+## CLI
+
+```bash
+dmcaed [--verbose] [command]
+```
+
+### Commands
+
+- `download`: Download the latest DMCA notices from GitHub
+- `analyze`: Parse downloaded notices and extract takedown data
+- `db`: Rebuild the complete DMCA database (download + analyze)
+- `stats`: Show statistics about the database content
+- `notices <url>`: List takedown notices for a specific repository URL
+
+### Options
+
+- `--verbose`: Enable debug logging
+- `-h`, `--help`: Show help message
+
+## Library Usage
+
+```python
+from dmcaed import DMCAed
+
+dmca = DMCAed()
+notices = dmca.notices("https://github.com/user/repo")
+
+if notices:
+    print(f"https://github.com/user/repo _might_ be subject to a takedown: {notices}")
+```
+
+## How it works
+
+- Download the latest DMCA notices from GitHub
+- Extract repository URLs (path to files, issues, releases, etc. are truncated)
+  mentioned in takedown notices
+- Build a local SQLite database mapping repository URLs to takedown notices
+
+## Build and Release
+
+### Prerequisites
+
+The project uses `hatch` build system and dynamic CalVer versioning based on the 
+current date.
+
+```bash
+pip install hatch hatch-calver
+```
+
+Git clone and install DMCAed.
+
+```bash
+git clone ssh://git@codeberg.org/canarduck/dmcaed.git
+pip install -e .
+```
+
+### Update the package and publish it on PyPI
+
+You first need to fetch DMCAs and create/update the database.
+
+```bash
+dmcaed db
+```
+
+Then use hatch to update version number and build the package.
+
+```bash
+hatch version release
+# use `hatch version release,fix` in case you need to fix an already released package
+```
+
+Build the package and publish it on PyPI.
+
+```bash
+hatch build
+hatch publish  --auth PYPI_AUTH_TOKEN
+```

dmcaed-2026.4.18/dmcaed.py

@@ -0,0 +1,205 @@
+"""DMCAed.
+
+Check if a software URL is found in takedown notices.
+"""
+
+import argparse
+import logging
+import re
+import shutil
+import sqlite3
+import sys
+import urllib.request
+import zipfile
+from pathlib import Path
+from importlib.resources import files
+
+__version__ = "2026.4.18"
+
+logger = logging.getLogger("dmcaed")
+
+LOCAL_DB = Path("DMCA.db")
+EXPORTS = Path("exports")
+
+GH_DMCA_URL = "https://github.com/github/dmca"
+"""GH DMCA repo URL."""
+GH_DMCA_DOWNLOAD_URL = f"{GH_DMCA_URL}/archive/refs/heads/master.zip"
+"""The URL to Github's zip export of their DMCA repo."""
+
+
+class DMCAed:
+    def __init__(self, db: Path | None = None):
+        """Init the db cursor."""
+        if not db:
+            ref = files(__name__) / "DMCA.db"
+            db = Path(str(ref))
+        self.cursor = sqlite3.connect(str(db)).cursor()
+
+    def notices(self, origin_url: str) -> set[str]:
+        """Get the takedown notices associated to `origin_url`.
+
+        The check ignore the URL case.
+
+        Args:
+            origin_url: a software URL.
+
+        Returns:
+            A set of takedown URLs, the set is empty if nothing matches.
+        """
+        self.cursor.execute(
+            "SELECT notice_url FROM takedowns WHERE origin_url = ? COLLATE NOCASE",
+            [origin_url],
+        )
+        return {r[0] for r in self.cursor.fetchall()}
+
+    def stats(self) -> dict:
+        """Some stats on the db.
+
+        Returns:
+            A dict of stats
+        """
+        self.cursor.execute(
+            "SELECT COUNT(DISTINCT origin_url), COUNT(DISTINCT notice_url) FROM takedowns"
+        )
+        origins, notices = self.cursor.fetchone()
+        return {"origins": origins, "notices": notices}
+
+
+def download_notices() -> None:
+    logger.info(f"Cleanup {EXPORTS}")
+    shutil.rmtree(EXPORTS, ignore_errors=True)
+    EXPORTS.mkdir()
+    gh_zip_path = EXPORTS / "gh.zip"
+
+    logger.info(f"Download {GH_DMCA_DOWNLOAD_URL}")
+    urllib.request.urlretrieve(GH_DMCA_DOWNLOAD_URL, gh_zip_path)
+
+    logger.info(f"Extract to {EXPORTS}")
+    with zipfile.ZipFile(gh_zip_path, "r") as z:
+        z.extractall(EXPORTS)
+
+
+def analyze_notices() -> None:
+    logger.info("Create an in-memory database")
+    db = sqlite3.connect(":memory:")
+    db.execute(
+        "CREATE TABLE takedowns (origin_url TEXT NOT NULL, notice_url TEXT NOT NULL)"
+    )
+    db.execute("CREATE INDEX origin_url_idx ON takedowns(origin_url COLLATE NOCASE)")
+
+    logger.info("Extracting repo URLS")
+
+    gh_dmca_path = EXPORTS / "dmca-master"
+
+    notice_paths = list(gh_dmca_path.glob("[0-9][0-9][0-9][0-9]/**/*.md")) + list(
+        gh_dmca_path.glob("[0-9][0-9][0-9][0-9]/**/*.markdown")
+    )
+
+    logger.info(f"{len(notice_paths)} notices to process")
+
+    cursor = db.cursor()
+
+    def gh_path_to_url(path: Path) -> str:
+        return str(path).replace("exports/dmca-master", f"{GH_DMCA_URL}/blob/master")
+
+    origin_pattern = re.compile(r"https?://github\.com/[\w\.\-]+/[\w\.\-\_]+")
+    for notice_path in notice_paths:
+        logger.debug("Processing %s", notice_path)
+        for origin_url in origin_pattern.findall(notice_path.read_text()):
+            logger.debug("- Found %s", origin_url)
+            cursor.execute(
+                "INSERT INTO takedowns(origin_url, notice_url) VALUES(?,?)",
+                [origin_url, gh_path_to_url(notice_path)],
+            )
+    logger.debug("Commit DB")
+    db.commit()
+    logger.debug(f"Cleanup previous {LOCAL_DB}")
+    LOCAL_DB.unlink(missing_ok=True)
+    db.execute(f"VACUUM main INTO '{LOCAL_DB}'")
+    logger.info(f"Wrote {LOCAL_DB}")
+
+
+def build_db() -> None:
+    """Create a new DMCA db.
+
+    This method:
+    1. recreate `LOCAL_DB`
+    2. fetches the latest version of GH DMCA repo to `exports/`
+    3. extract repo URLs from the takedown notices
+    4. store a (repo URL, Takedown URL) pair in the DB
+    """
+    logger.info(f"Rebuilding {LOCAL_DB}")
+    download_notices()
+    analyze_notices()
+
+    dmcaed = DMCAed(LOCAL_DB)
+
+    logger.info(f"Done! {dmcaed.stats()}")
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Check if a software URL is found in takedown notices"
+    )
+    parser.add_argument("--verbose", action="store_true", help="Enable debug logging")
+    parser.add_argument(
+        "--local",
+        type=bool,
+        help="Use the local database file. Uses the packaged one by default.",
+    )
+
+    subparsers = parser.add_subparsers(dest="command", required=True)
+    subparsers.add_parser(
+        "download", help="Download the latest DMCA notices from GitHub"
+    )
+    subparsers.add_parser(
+        "analyze", help="Parse downloaded notices and extract takedown data"
+    )
+    subparsers.add_parser(
+        "db", help="Rebuild the complete DMCA database (download + analyze)"
+    )
+    subparsers.add_parser("stats", help="Show statistics about the database content")
+    parser_notices = subparsers.add_parser(
+        "notices", help="List takedown notices for a URL"
+    )
+    parser_notices.add_argument("url", help="The origin URL to check")
+
+    if len(sys.argv) == 1:
+        print(r"""    ____  __  ____________            __
+   / __ \/  |/  / ____/   | ___  ____/ /
+  / / / / /|_/ / /   / /| |/ _ \/ __  / 
+ / /_/ / /  / / /___/ ___ /  __/ /_/ /  
+/_____/_/  /_/\____/_/  |_\___/\__,_/   
+""")
+        parser.print_help()
+        sys.exit(0)
+
+    args = parser.parse_args()
+
+    logging.basicConfig(
+        level=logging.DEBUG if args.verbose else logging.INFO,
+        format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+    )
+
+    if args.command == "download":
+        download_notices()
+    elif args.command == "analyze":
+        analyze_notices()
+    elif args.command == "db":
+        build_db()
+    elif args.command == "stats":
+        db = LOCAL_DB if args.local else None
+        dmcaed = DMCAed(db)
+        print(dmcaed.stats())
+    elif args.command == "notices":
+        db = LOCAL_DB if args.local else None
+        dmcaed = DMCAed(db)
+        notices = dmcaed.notices(args.url)
+        if notices:
+            print(*notices, sep="\n")
+        else:
+            print("No takedown notices found for the given URL.")
+
+
+if __name__ == "__main__":
+    main()

dmcaed-2026.4.18/pyproject.toml

@@ -0,0 +1,49 @@
+[project]
+name = "dmcaed"
+authors = [
+  { name="Renaud Canarduck", email="renaud@canarduck.com" },
+]
+description = "Check if a software URL is found in takedown notices."
+readme = "README.md"
+requires-python = ">=3.11"
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "Operating System :: OS Independent",
+    "Development Status :: 3 - Alpha"
+]
+license = "MIT"
+license-files = ["LICENSE"]
+dynamic = ["version"]
+
+[project.urls]
+Homepage = "https://codeberg.org/canarduck/dmcaed"
+Issues = "https://codeberg.org/canarduck/dmcaed/issues"
+
+[project.optional-dependencies]
+tests = ["coverage"]
+dev = [
+  "black",
+  "mypy",
+]
+
+[project.scripts]
+dmcaed = "dmcaed:main"
+
+[project.data]
+paths = ["DMCA.db"]
+
+[build-system]
+requires = ["hatchling >= 1.26", "hatch-calver"]
+build-backend = "hatchling.build"
+
+[tool.hatch.version]
+path = "dmcaed.py"
+scheme = "calver"
+calver-scheme = "YYYY.MM.DD"
+
+[tool.coverage.report]
+fail_under = 90
+
+[tool.coverage.run]
+branch = true
+source = ["."]