ofscraper 3.13.dev10__tar.gz → 3.13.dev12__tar.gz

ofscraper-3.13.dev12/.dockerignore

@@ -0,0 +1,28 @@
+# Ignore Python's virtual environment directory (your 'info' folder)
+info/
+.venv/
+venv/
+env/
+
+# Ignore Python cache and artifacts
+__pycache__/
+*.pyc
+*.pyo
+*.pyd
+dist/
+build/
+wiki/
+*.egg-info/
+
+# Ignore editor and OS-specific files
+.vscode/
+.idea/
+*.DS_Store
+
+# Ignore test files if they aren't needed in the final image
+test/
+
+# Ignore local secrets or temp files
+.env
+.tempmedia/
+*.log

{ofscraper-3.13.dev10 → ofscraper-3.13.dev12}/.github/workflows/commit_bundler.yml

@@ -197,7 +197,7 @@ jobs:
         with:
           artifacts: artifacts/*/* # Attach any downloaded artifacts (will include only those that were built successfully)
           # Combined tag for uniqueness and chronological sorting
-          tag: "${{ needs.generate_version.outputs.base_version }}-${{ needs.generate_version.outputs.short_hash }}-${{ needs.generate_version.outputs.commit_timestamp }}"
+          tag: "${{ needs.generate_version.outputs.base_version }}+${{ needs.generate_version.outputs.short_hash }}.${{ needs.generate_version.outputs.commit_timestamp }}"
           # Human-readable name with all key details
           name: "Dev Build ${{ needs.generate_version.outputs.base_version }} (Commit ${{ needs.generate_version.outputs.short_hash }}) - ${{ needs.generate_version.outputs.commit_timestamp }}"
           

{ofscraper-3.13.dev10 → ofscraper-3.13.dev12}/.github/workflows/docker-daily.yml

@@ -6,7 +6,6 @@ on:
       - '**'
     tags-ignore:
       - '**'
-
 permissions:
   contents: read
   packages: write
@@ -28,7 +27,7 @@ jobs:
         with:
           fetch-depth: 0
           fetch-tags: true
-      
+
       - name: Make script executable
         run: chmod +x ./scripts/commit_version.sh
       - name: Generate Version and Git Info
@@ -36,7 +35,7 @@ jobs:
         env:
           GH_TOKEN: ${{ github.token }}
           GITHUB_REPOSITORY: ${{ github.repository }}
-          GITHUB_WORKFLOW_REF: ${{ github.workflow_ref }} # Still pass for general info, but not for WORKFLOW_ID now
+          GITHUB_WORKFLOW_REF: ${{ github.workflow_ref }}
           GITHUB_REF: ${{ github.ref }}
           GITHUB_EVENT_BEFORE: ${{ github.event.before }}
         run: ./scripts/commit_version.sh
@@ -59,7 +58,7 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v4 
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
@@ -73,8 +72,8 @@ jobs:
           username: ${{ matrix.registry.id == 'ghcr' && github.actor || secrets.DOCKERHUB_USERNAME }}
           password: ${{ matrix.registry.id == 'ghcr' && secrets.GITHUB_TOKEN || secrets.DOCKERHUB_TOKEN }}
 
-      - name: Extract Docker metadata
-        id: meta
+      - name: Extract Docker metadata (Normal Image)
+        id: meta_normal
         uses: docker/metadata-action@v5
         with:
           images: ${{ matrix.registry.id == 'ghcr' && format('ghcr.io/{0}', github.repository) || 'datawhores/of-scraper' }}
@@ -83,23 +82,23 @@ jobs:
             type=raw,value=${{ needs.generate_version.outputs.base_version }}-${{ needs.generate_version.outputs.short_hash }}-${{ needs.generate_version.outputs.commit_timestamp }}
             type=raw,value=${{ needs.generate_version.outputs.base_version }}-${{ needs.generate_version.outputs.short_hash }}
             type=raw,value=latest,enable=${{ needs.generate_version.outputs.is_newer_than_last_successful_run }}
-            
 
-      - name: Build and push Docker image
-        id: build-and-push
+      - name: Build and push Normal Docker image
+        id: build-and-push-normal 
         uses: docker/build-push-action@v5
         with:
           context: .
+          file: ./Dockerfile 
           push: true
-          tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
+          tags: ${{ steps.meta_normal.outputs.tags }}
+          labels: ${{ steps.meta_normal.outputs.labels }}
           cache-from: type=gha
           cache-to: type=gha,mode=max
           platforms: linux/amd64,linux/arm64
           build-args: |
             BUILD_VERSION=${{ needs.generate_version.outputs.version }}
-            INSTALL_FFMPEG=true
 
+      # This part needs adjustment to sign both images.
       - name: Install Cosign
         if: matrix.registry.sign == true
         uses: sigstore/cosign-installer@v3.5.0
@@ -109,9 +108,16 @@ jobs:
         run: echo "${{ secrets.COSIGN_PRIVATE_KEY }}" > cosign.key
         shell: bash
 
-      - name: Sign the published Docker image
+      - name: Sign the published Docker images
         if: matrix.registry.sign == true
         env:
           COSIGN_PASSWORD: ${{ secrets.COSIGN_PASSWORD }}
           COSIGN_EXPERIMENTAL: "true"
-        run: echo "${{ steps.meta.outputs.tags }}" | xargs -I {} cosign sign --yes --key cosign.key {}@${{ steps.build-and-push.outputs.digest }}
+        run: |
+          # Get all tags for the normal image
+          NORMAL_IMAGE_TAGS="${{ steps.meta_normal.outputs.tags }}"
+          # Get the digest of the normal image
+          NORMAL_IMAGE_DIGEST="${{ steps.build-and-push-normal.outputs.digest }}"
+
+          echo "Signing Normal Image: ${NORMAL_IMAGE_TAGS} @ ${NORMAL_IMAGE_DIGEST}"
+          echo "$NORMAL_IMAGE_TAGS" | xargs -I {} cosign sign --yes --key cosign.key {}@$NORMAL_IMAGE_DIGEST

{ofscraper-3.13.dev10 → ofscraper-3.13.dev12}/.github/workflows/package-builder-release.yml

@@ -141,9 +141,8 @@ jobs:
       - name: Build Package with Hatch
         run: |
           source .venv/bin/activate
-          echo "HATCH_VCS_PRETEND_VERSION=$PACKAGE_VERSION" >> "$GITHUB_ENV"
+          echo "HATCH_VCS_PRETEND_VERSION=$PACKAGE_VERSION=${{ needs.determine_tags_and_versions.outputs.package_version }}" >> $GITHUB_ENV
           echo "SETUPTOOLS_SCM_PRETEND_VERSION=${{ needs.determine_tags_and_versions.outputs.package_version }}" >> $GITHUB_ENV
-
           hatch build
         shell: bash
 

{ofscraper-3.13.dev10 → ofscraper-3.13.dev12}/.github/workflows/publish-release-dockers.yml

@@ -90,9 +90,7 @@ jobs:
           context: .
           platforms: linux/amd64,linux/arm64
           build-args: |
-            SETUPTOOLS_SCM_PRETEND_VERSION=${{ inputs.package_version }}
-            HATCH_VCS_PRETEND_VERSION=${{ inputs.package_version }}
-            INSTALL_FFMPEG=true
+            BUILD_VERSION=${{ inputs.package_version }}
           tags: ${{ steps.meta_hub_stable.outputs.tags }}
           labels: ${{ steps.meta_hub_stable.outputs.labels }}
           cache-from: type=gha
@@ -163,8 +161,7 @@ jobs:
           platforms: linux/amd64,linux/arm64
           #pass enviroment variables as input
           build-args: |
-            SETUPTOOLS_SCM_PRETEND_VERSION=${{ inputs.package_version }}
-            HATCH_VCS_PRETEND_VERSION=${{ inputs.package_version }}
+            BUILD_VERSION=${{ inputs.package_version }}
           tags: ${{ steps.meta_hub_dev.outputs.tags }}
           labels: ${{ steps.meta_hub_dev.outputs.labels }}
           cache-from: type=gha
@@ -235,8 +232,7 @@ jobs:
           context: .
           platforms: linux/amd64,linux/arm64
           build-args: |
-            SETUPTOOLS_SCM_PRETEND_VERSION=${{ inputs.package_version }}
-            HATCH_VCS_PRETEND_VERSION=${{ inputs.package_version }}
+            BUILD_VERSION=${{ inputs.package_version }}
           tags: ${{ steps.meta_ghcr_stable.outputs.tags }}
           labels: ${{ steps.meta_ghcr_stable.outputs.labels }}
           cache-from: type=gha
@@ -304,8 +300,7 @@ jobs:
           context: .
           platforms: linux/amd64,linux/arm64
           build-args: |
-            SETUPTOOLS_SCM_PRETEND_VERSION=${{ inputs.package_version }}
-            HATCH_VCS_PRETEND_VERSION=${{ inputs.package_version }}
+            BUILD_VERSION=${{ inputs.package_version }}
           tags: ${{ steps.meta_ghcr_dev.outputs.tags }}
           labels: ${{ steps.meta_ghcr_dev.outputs.labels }}
           cache-from: type=gha

ofscraper-3.13.dev12/.gitignore

@@ -0,0 +1,69 @@
+# --- Standard Python Artifacts ---
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+
+# Distribution / packaging / Build artifacts
+build/
+dist/
+sdist/
+wheels/
+*.egg-info/
+*.egg
+MANIFEST
+*.spec # Keep .spec files if they are manually edited, ignore if auto-generated.
+       # Yours look like configuration, so I've commented this out.
+       # *.spec
+
+# PyInstaller temporary files
+# *.manifest # Already covered by standard rules if needed
+
+# Installer logs & Test reports
+pip-log.txt
+pip-delete-this-directory.txt
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# This is the 'info/' directory in your project
+/info/
+# Also include other common venv names
+.venv/
+venv/
+env/
+ENV/
+
+# Editor & IDE-specific files
+.vscode/
+.idea/
+.spyderproject
+.ropeproject
+
+# mypy cache
+.mypy_cache/
+
+# Custom generated files or logs
+*.log
+# Ignoring a custom compiled/encrypted version file
+ofscraper/__version__.pye
+
+/crypto/
+
+# Ignore specific generated files if they appear
+# dupe.json
+.tempmedia/
+
+#trunk
+.trunk
+#documentation
+wiki/
+docs/

ofscraper-3.13.dev12/Dockerfile

@@ -0,0 +1,70 @@
+# syntax=docker/dockerfile:1.7-labs
+# Stage 1: Build the application artifact
+FROM ghcr.io/astral-sh/uv:0.7.13-python3.11-bookworm-slim AS builder
+
+ARG BUILD_VERSION
+WORKDIR /app
+
+# --- This is the ABSOLUTE ONLY essential change to fix the psutil/gcc error ---
+# It installs the necessary compiler (gcc via build-essential) and Python development headers
+# needed to compile Python packages like psutil, especially for non-AMD64 platforms.
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    build-essential\
+    python3-dev \
+    git \
+    && rm -rf /var/lib/apt/lists/*
+COPY --parents pyproject.toml uv.lock README.md .git ofscraper .
+
+RUN \
+    if [ -n "$BUILD_VERSION" ]; then \
+      VERSION="$BUILD_VERSION"; \
+    else \
+      SHORT_HASH=$(git rev-parse --short HEAD 2>/dev/null || echo "0000000"); \
+      HIGHEST_TAG=$(git tag --sort=-committerdate | grep -E '^v?[0-9]+\.[0-9]+(\.[0-9]+)?([-.][a-zA-Z0-9.]+)?$' | head -n 1); \
+      if [ -z "$HIGHEST_TAG" ]; then BASE_VERSION="0.0.0"; else BASE_VERSION=$(echo "$HIGHEST_TAG" | sed 's/^v//'); fi; \
+      VERSION="${BASE_VERSION}+g${SHORT_HASH}"; \
+    fi && \
+    export HATCH_VCS_PRETEND_VERSION=$VERSION && \
+    export SETUPTOOLS_SCM_PRETEND_VERSION=$VERSION && \
+    echo "Build OF-SCRAPER with ${VERSION}" && \
+
+    python3 -m pip install --no-cache-dir hatch hatch-vcs && \
+    uv sync --locked && \
+    hatch build
+
+
+# Stage 2: Create the final, minimal production image
+FROM ghcr.io/astral-sh/uv:0.7.13-python3.11-bookworm-slim
+WORKDIR /app
+
+# Configure uv's venv and cache location.
+ENV VIRTUAL_ENV="/app/.venv"
+ENV UV_CACHE_DIR="/app/.uv_cache"
+ENV PATH="${VIRTUAL_ENV}/bin:${PATH}"
+
+# STEP 1: Create venv, install pyffmpeg and force static
+# timestamps for venv & cache to avoid unnecessary layer updates.
+RUN uv venv && uv pip install pyffmpeg==2.4.2.20 && \
+    find "${VIRTUAL_ENV}" -print0 | xargs -0 touch -h -d '2025-01-01T00:00:00Z' && \
+    find "${UV_CACHE_DIR}" -print0 | xargs -0 touch -h -d '2025-01-01T00:00:00Z'
+
+# STEP 2: Install all OS-level dependencies in a single layer.
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    gosu \
+    && rm -rf /var/lib/apt/lists/*
+
+# STEP 3: Copy entrypoint scripts and make them executable.
+COPY --chmod=755 ./scripts/entry/. /usr/local/bin/entry/
+
+# STEP 4: Copy and install the ofscraper wheels into the venv.
+COPY --from=builder /app/dist/*.whl .
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    build-essential && \
+    uv pip install *.whl -v && \
+    rm *.whl && \
+    apt-get purge -y --auto-remove build-essential && \
+    rm -rf /var/lib/apt/lists/*
+
+USER root 
+ENTRYPOINT ["/usr/local/bin/entry/entrypoint.sh"]
+CMD ["ofscraper"]

{ofscraper-3.13.dev10 → ofscraper-3.13.dev12}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ofscraper
-Version: 3.13.dev10
+Version: 3.13.dev12
 Summary: automatically scrape onlyfans
 Project-URL: Homepage, https://github.com/datawhores/OF-Scraper
 Author-email: datawhores <datawhores@riseup.net>

{ofscraper-3.13.dev10 → ofscraper-3.13.dev12}/docker-compose.yml

@@ -1,10 +1,10 @@
 version: "3.8" # Recommended to use a recent version for new features
 services:
+  #latest
   of-scraper:
     stdin_open: true # Keeps STDIN open for interactive processes
     tty: true # Allocates a pseudo-TTY for a terminal-like interface
     container_name: ofscraper
-    
     environment: # Optional environment variables
       # These are recommended for correct file permissions on mounted volumes.
       # Default UID/GID is 1000:1000 if not specified via environment variables in your shell.
@@ -37,4 +37,15 @@ services:
     # command: "ofscraper --username ALL --posts all"
 
     # Optional: Configure restart policy.
-    # restart: "unless-stopped" # Ensures the container restarts automatically unless explicitly stopped.
+    # restart: "unless-stopped" # Ensures the container restarts automatically unless explicitly stopped.
+
+  #local 
+  #comment out this and comment out the first of-scraper example to run locally
+  #  of-scraper:
+  #   stdin_open: true # Keeps STDIN open for interactive processes
+  #   tty: true # Allocates a pseudo-TTY for a terminal-like interface
+  #   container_name: ofscraper
+
+  #   build:
+  #     context: .
+  #     dockerfile: Dockerfile

{ofscraper-3.13.dev10 → ofscraper-3.13.dev12}/ofscraper/classes/sessionmanager/sessionmanager.py

@@ -321,7 +321,7 @@ class sessionManager:
         exceptions = exceptions or []
         actions = actions or []
         for _ in Retrying(
-            retry=retry_if_not_exception_type((KeyboardInterrupt)),
+            retry=retry_if_not_exception_type((KeyboardInterrupt,SystemExit)),
             stop=tenacity.stop.stop_after_attempt(retries),
             wait=tenacity.wait.wait_random(min=min, max=max),
             before=lambda x: (
@@ -424,7 +424,7 @@ class sessionManager:
             wait_exponential=tenacity.wait.wait_exponential(
                 multiplier=2, min=wait_min_exponential, max=wait_max_exponential
             ),
-            retry=retry_if_not_exception_type((KeyboardInterrupt)),
+            retry=retry_if_not_exception_type((KeyboardInterrupt,SystemExit)),
             wait_random=tenacity.wait_random(min=wait_min, max=wait_max),
             stop=tenacity.stop.stop_after_attempt(retries),
             before=lambda x: (

{ofscraper-3.13.dev10 → ofscraper-3.13.dev12}/ofscraper/commands/scraper/actions/download/utils/keyhelpers.py

@@ -48,12 +48,8 @@ async def un_encrypt(item, c, ele, input_=None):
             log.debug(f"{get_medialog(ele)} got key from cache")
         elif keymode == "manual":
             key = await key_helper_manual(c, item["pssh"], ele.license, ele.id)
-        elif keymode == "keydb":
-            key = await key_helper_keydb(c, item["pssh"], ele.license, ele.id)
         elif keymode == "cdrm":
             key = await key_helper_cdrm(c, item["pssh"], ele.license, ele.id)
-        elif keymode == "cdrm2":
-            key = await key_helper_cdrm2(c, item["pssh"], ele.license, ele.id)
         if not key:
             raise Exception(f"{get_medialog(ele)} Could not get key")
         key = key.strip()
@@ -118,101 +114,14 @@ async def key_helper_cdrm(c, pssh, licence_url, id):
         headers["cookie"] = auth_requests.get_cookies_str()
         auth_requests.create_sign(licence_url, headers)
         json_data = {
-            "License URL": licence_url,
-            "Headers": json.dumps(headers),
-            "PSSH": pssh,
-            "JSON": "{}",
-            "Cookies": "{}",
-            "Data": "{}",
-            "Proxy": "",
-        }
-        async with c.requests_async(
-            url=constants.getattr("CDRM"),
-            method="post",
-            json=json_data,
-            retries=get_cmd_download_req_retries(),
-            wait_min=constants.getattr("OF_MIN_WAIT_API"),
-            wait_max=constants.getattr("OF_MAX_WAIT_API"),
-            total_timeout=constants.getattr("CDM_TIMEOUT"),
-            skip_expection_check=True,
-        ) as r:
-            data = await r.json_()
-            log.debug(f"keydb json {data}")
-            key = data["Message"]
-        return key
-    except Exception as E:
-        log.traceback_(E)
-        log.traceback_(traceback.format_exc())
-        raise E
-
-
-async def key_helper_cdrm2(c, pssh, licence_url, id):
-    key = None
-    log.debug(f"ID:{id} using cdrm2 auto key helper")
-    try:
-        log.debug(f"ID:{id} pssh: {pssh is not None}")
-        log.debug(f"ID:{id} licence: {licence_url}")
-        headers = auth_requests.make_headers()
-        headers["cookie"] = auth_requests.get_cookies_str()
-        auth_requests.create_sign(licence_url, headers)
-        json_data = {
-            "license": licence_url,
+            "licurl": licence_url,
             "headers": json.dumps(headers),
             "pssh": pssh,
-            "buildInfo": "google/sdk_gphone_x86/generic_x86:8.1.0/OSM1.180201.037/6739391:userdebug/dev-keys",
-            "proxy": "",
-            "cache": True,
         }
         async with c.requests_async(
-            url=constants.getattr("CDRM2"),
-            method="post",
-            json=json_data,
-            retries=get_cmd_download_req_retries(),
-            wait_min=constants.getattr("OF_MIN_WAIT_API"),
-            wait_max=constants.getattr("OF_MAX_WAIT_API"),
-            total_timeout=constants.getattr("CDM_TIMEOUT"),
-            skip_expection_check=True,
-        ) as r:
-            httpcontent = await r.text_()
-            log.debug(f"ID:{id} key_response: {httpcontent}")
-            soup = BeautifulSoup(httpcontent, "html.parser")
-            key = soup.find("li").contents[0]
-        return key
-    except Exception as E:
-        log.traceback_(E)
-        log.traceback_(traceback.format_exc())
-        raise E
-
-
-async def key_helper_keydb(c, pssh, licence_url, id):
-    key = None
-    log.debug(f"ID:{id} using keydb auto key helper")
-    try:
-        log.debug(f"ID:{id} pssh: {pssh is not None}")
-        log.debug(f"ID:{id} licence: {licence_url}")
-        headers = auth_requests.make_headers()
-        headers["cookie"] = auth_requests.get_cookies_str()
-        auth_requests.create_sign(licence_url, headers)
-        json_data = {
-            "license_url": licence_url,
-            "headers": json.dumps(headers),
-            "pssh": pssh,
-            "buildInfo": "",
-            "proxy": "",
-            "cache": True,
-        }
-
-        headers = {
-            "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (Ktesttemp, like Gecko) Chrome/90.0.4430.85 Safari/537.36",
-            "Content-Type": "application/json",
-            "X-API-Key": settings.get_keydb_api(),
-        }
-
-        async with c.requests_async(
-            url=constants.getattr("KEYDB"),
+            url=constants.getattr("CDRM"),
             method="post",
             json=json_data,
-            headers=headers,
             retries=get_cmd_download_req_retries(),
             wait_min=constants.getattr("OF_MIN_WAIT_API"),
             wait_max=constants.getattr("OF_MAX_WAIT_API"),
@@ -220,20 +129,14 @@ async def key_helper_keydb(c, pssh, licence_url, id):
             skip_expection_check=True,
         ) as r:
             data = await r.json_()
-            log.debug(f"keydb json {data}")
-            if isinstance(data, str):
-                key = data
-            elif isinstance(data["keys"][0], str):
-                key = data["keys"][0]
-            elif isinstance(data["keys"][0], object):
-                key = data["keys"][0]["key"]
+            log.debug(f"cdrm json {data}")
+            key = data["message"]
         return key
     except Exception as E:
         log.traceback_(E)
         log.traceback_(traceback.format_exc())
         raise E
 
-
 async def key_helper_manual(c, pssh, licence_url, id):
     key = None
     log.debug(f"ID:{id}  manual key helper")

{ofscraper-3.13.dev10 → ofscraper-3.13.dev12}/ofscraper/const/values/general.py

@@ -1,5 +1,5 @@
 SUPRESS_OUTPUTS = {"CRITICAL", "ERROR", "WARNING", "OFF", "LOW", "PROMPT"}
-KEY_OPTIONS = ["cdrm", "cdrm2", "manual", "keydb"]
+KEY_OPTIONS = ["cdrm",  "manual"]
 refreshScreen = 50
 NUMBER_REGEX = "[0-9]"
 USERNAME_REGEX = "[^/]"

ofscraper-3.13.dev12/ofscraper/const/values/url/other_url.py

@@ -0,0 +1,6 @@
+CDRM = "https://cdrm-project.com/api/decrypt"
+BAD_URL_HOST = {
+    ".*\.upload.onlyfans.com",
+    "of2transcoder.s3-accelerate.amazonaws.com",
+    ".*\.convert.onlyfans.com",
+}

{ofscraper-3.13.dev10 → ofscraper-3.13.dev12}/ofscraper/main/open/load.py

@@ -23,7 +23,6 @@ def main():
         readConfig()
         setLogger()
         make_folder()
-        check()
         run.main()
     except Exception as E:
         print(E)
@@ -64,6 +63,3 @@ def readConfig():
 def make_folder():
     paths_manage.make_folders()
 
-
-def check():
-    checkers.check_config_key_mode()