hkex-filing-scraper 2.1.0__tar.gz

hkex_filing_scraper-2.1.0/.editorconfig

@@ -0,0 +1,16 @@
+root = true
+
+[*]
+charset = utf-8
+end_of_line = lf
+insert_final_newline = true
+trim_trailing_whitespace = true
+indent_style = space
+indent_size = 4
+max_line_length = 100
+
+[*.{yml,yaml,json,toml}]
+indent_size = 2
+
+[*.md]
+trim_trailing_whitespace = false

hkex_filing_scraper-2.1.0/.env.example

@@ -0,0 +1,140 @@
+# =============================================================================
+# HKEx Filing Scraper — Environment Configuration
+# =============================================================================
+# Copy this file to .env and fill in your values:
+#   cp .env.example .env
+#
+# .env is loaded from the CURRENT WORKING DIRECTORY, not the repo root.
+# =============================================================================
+
+# ---------------------------------------------------------------------------
+# Database sink selection (REQUIRED)
+# ---------------------------------------------------------------------------
+# An ordered, comma-separated list of sink ids. Every configured sink is
+# written to; reads come from the first sink that supports them.
+# Valid ids (documented popularity order):
+#   postgres    PostgreSQL 13+    (extra: postgres)
+#   mysql       MySQL 8           (extra: mysql)
+#   sqlite      SQLite file       (no extra)
+#   mongodb     MongoDB 6+        (extra: mongodb)
+#   mariadb     MariaDB 10.5+     (extra: mysql)
+#   neo4j       Neo4j 5           (extra: neo4j)
+#   clickhouse  ClickHouse 24+    (extra: clickhouse)
+#   duckdb      DuckDB file       (extra: duckdb)
+#   surrealdb   SurrealDB         (no extra)
+#
+# Examples:
+#   DATABASE_TARGET=postgres,sqlite
+#   DATABASE_TARGET=mysql,clickhouse
+#   DATABASE_TARGET=sqlite
+DATABASE_TARGET=postgres,sqlite
+
+# ---------------------------------------------------------------------------
+# PostgreSQL (required when DATABASE_TARGET includes postgres)
+# ---------------------------------------------------------------------------
+# Install the driver first:  pip install "hkex-filing-scraper[postgres]"
+#
+# Either provide a full connection string...
+POSTGRES_DSN=
+# ...or the discrete settings below (used when POSTGRES_DSN is empty).
+POSTGRES_HOST=localhost
+POSTGRES_PORT=5432
+POSTGRES_DATABASE=
+POSTGRES_USER=
+POSTGRES_PASSWORD=
+POSTGRES_SCHEMA=public
+# Connection pool sizing (OPTIONAL)
+# POSTGRES_MIN_POOL=1
+# POSTGRES_MAX_POOL=15
+
+# ---------------------------------------------------------------------------
+# MySQL / MariaDB (required when DATABASE_TARGET includes mysql or mariadb)
+# ---------------------------------------------------------------------------
+# Install the driver first:  pip install "hkex-filing-scraper[mysql]"
+#
+# MYSQL_DSN=mysql://user:password@localhost:3306/hkex
+MYSQL_HOST=localhost
+MYSQL_PORT=3306
+MYSQL_DATABASE=
+MYSQL_USER=
+MYSQL_PASSWORD=
+# The mariadb sink reads MARIADB_* and falls back to MYSQL_* when its own are unset.
+# MARIADB_DSN=
+# MARIADB_HOST=localhost
+# MARIADB_PORT=3306
+# MARIADB_DATABASE=
+# MARIADB_USER=
+# MARIADB_PASSWORD=
+
+# ---------------------------------------------------------------------------
+# SQLite / DuckDB (required when DATABASE_TARGET includes sqlite or duckdb)
+# ---------------------------------------------------------------------------
+# SQLite needs no extra dependency.
+# DuckDB:  pip install "hkex-filing-scraper[duckdb]"
+#
+# SQLITE_PATH=hkex.db          # a file path, or :memory:
+# DUCKDB_PATH=hkex.duckdb      # a file path, or :memory:
+
+# ---------------------------------------------------------------------------
+# MongoDB (required when DATABASE_TARGET includes mongodb)
+# ---------------------------------------------------------------------------
+# Install the driver first:  pip install "hkex-filing-scraper[mongodb]"
+#
+# MONGODB_URI=mongodb://localhost:27017
+# MONGODB_DATABASE=hkex
+
+# ---------------------------------------------------------------------------
+# ClickHouse (required when DATABASE_TARGET includes clickhouse)
+# ---------------------------------------------------------------------------
+# Install the driver first:  pip install "hkex-filing-scraper[clickhouse]"
+#
+# CLICKHOUSE_HOST=localhost
+# CLICKHOUSE_PORT=8123
+# CLICKHOUSE_DATABASE=hkex
+# CLICKHOUSE_USER=default
+# CLICKHOUSE_PASSWORD=
+
+# ---------------------------------------------------------------------------
+# Neo4j (required when DATABASE_TARGET includes neo4j)
+# ---------------------------------------------------------------------------
+# Install the driver first:  pip install "hkex-filing-scraper[neo4j]"
+#
+# NEO4J_URI=bolt://localhost:7687
+# NEO4J_USER=neo4j
+# NEO4J_PASSWORD=
+# NEO4J_DATABASE=
+
+# ---------------------------------------------------------------------------
+# SurrealDB (required when DATABASE_TARGET includes surrealdb)
+# ---------------------------------------------------------------------------
+# No extra dependency.
+#
+# SURREAL_ENDPOINT=http://localhost:8000
+# SURREAL_NAMESPACE=default
+# SURREAL_DATABASE=default
+# SURREAL_USERNAME=root
+# SURREAL_PASSWORD=
+
+# ---------------------------------------------------------------------------
+# Graph linking (OPTIONAL)
+# ---------------------------------------------------------------------------
+# Set COMPANY_TABLE to the name of your existing company table to enable
+# graph edges (has_filing, references_filing) between companies and filings.
+# Leave empty to disable graph linking entirely.
+#
+# COMPANY_TABLE=my_company
+#
+# Pattern for converting a ticker like "0451.HK" into a company key.
+# Placeholders: {code} = leading-zero-stripped stock code, {exchange} = exchange suffix.
+# Default: "{code}_{exchange}" produces "451_HK"
+#
+# COMPANY_ID_PATTERN={code}_{exchange}
+
+# ---------------------------------------------------------------------------
+# Performance tuning (OPTIONAL)
+# ---------------------------------------------------------------------------
+# MAX_DOWNLOAD_WORKERS=15
+#
+# Minimum seconds between request starts, process-wide. 0 disables pacing (default).
+# Set a small value (e.g. 0.25) to be deliberately polite to the endpoint.
+# REQUEST_DELAY_SECONDS=0

hkex_filing_scraper-2.1.0/.gitattributes

@@ -0,0 +1,18 @@
+# Auto-detect text files and normalise line endings.
+* text=auto eol=lf
+
+# Windows scripts keep CRLF.
+*.bat text eol=crlf
+*.cmd text eol=crlf
+
+# Binary assets.
+*.png binary
+*.jpg binary
+*.gif binary
+*.pdf binary
+*.whl binary
+*.gz binary
+
+# Keep generated/vendored files out of language stats.
+src/hkex_scraper/_version.py linguist-generated=true
+docs/*.png linguist-documentation

hkex_filing_scraper-2.1.0/.github/CODEOWNERS

@@ -0,0 +1,8 @@
+# Default owner for everything in the repo.
+* @simonplmak-cloud
+
+# Sink architecture and per-engine adapters.
+/src/hkex_scraper/sinks/ @simonplmak-cloud
+
+# CI, release, and GitHub configuration.
+/.github/ @simonplmak-cloud

hkex_filing_scraper-2.1.0/.github/FUNDING.yml

@@ -0,0 +1,5 @@
+# Funding options for this project.
+# The Sponsor button appears only for platforms that are actually enrolled,
+# so leaving a handle here is safe; remove a line if you are not enrolled.
+github: [simonplmak-cloud]
+# custom: ["https://simonplmak-cloud.github.io/hkex-filing-scraper/"]

hkex_filing_scraper-2.1.0/.github/ISSUE_TEMPLATE/bug_report.yml

@@ -0,0 +1,78 @@
+name: Bug report
+description: Report a reproducible problem with the scraper
+title: "[Bug] "
+labels: ["bug"]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thanks for the report. Please do not include credentials or `.env` contents.
+  - type: textarea
+    id: what-happened
+    attributes:
+      label: What happened?
+      description: A clear description of the bug and what you expected instead.
+    validations:
+      required: true
+  - type: textarea
+    id: reproduce
+    attributes:
+      label: Steps to reproduce
+      description: Exact commands and configuration (redact secrets).
+      placeholder: |
+        1. DATABASE_TARGET=...
+        2. hkex-scraper ...
+        3. ...
+    validations:
+      required: true
+  - type: dropdown
+    id: sink
+    attributes:
+      label: Database target
+      options:
+        - postgres
+        - mysql
+        - mariadb
+        - sqlite
+        - duckdb
+        - mongodb
+        - clickhouse
+        - neo4j
+        - surrealdb
+        - multiple / other
+    validations:
+      required: true
+  - type: input
+    id: version
+    attributes:
+      label: Version
+      description: Output of `hkex-scraper --version` or the release tag.
+      placeholder: "1.1.0"
+    validations:
+      required: true
+  - type: input
+    id: python
+    attributes:
+      label: Python version
+      placeholder: "3.11.9"
+    validations:
+      required: true
+  - type: dropdown
+    id: os
+    attributes:
+      label: Operating system
+      options:
+        - Linux
+        - macOS
+        - Windows
+        - Other
+    validations:
+      required: true
+  - type: textarea
+    id: logs
+    attributes:
+      label: Relevant log output
+      description: Paste the relevant lines from `logs/hkex_filings_*.log` (redact secrets).
+      render: shell
+    validations:
+      required: false

hkex_filing_scraper-2.1.0/.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,11 @@
+blank_issues_enabled: false
+contact_links:
+  - name: Question or usage help
+    url: https://github.com/simonplmak-cloud/hkex-filing-scraper/discussions
+    about: Ask a question, share a setup, or discuss an idea in Discussions. Please do not open an issue for usage questions.
+  - name: Documentation
+    url: https://simonplmak-cloud.github.io/hkex-filing-scraper/
+    about: Getting started, configuration, CLI, sink guides, and troubleshooting.
+  - name: Security vulnerability
+    url: https://github.com/simonplmak-cloud/hkex-filing-scraper/security/policy
+    about: Report a vulnerability privately per the security policy. Never open a public issue for security.

hkex_filing_scraper-2.1.0/.github/ISSUE_TEMPLATE/feature_request.yml

@@ -0,0 +1,46 @@
+name: Feature request
+description: Suggest an idea or improvement
+title: "[Feature] "
+labels: ["enhancement"]
+body:
+  - type: textarea
+    id: problem
+    attributes:
+      label: Problem / motivation
+      description: What problem does this solve? Who benefits?
+    validations:
+      required: true
+  - type: textarea
+    id: proposal
+    attributes:
+      label: Proposed solution
+      description: Describe the change you would like. Include CLI/config sketches if relevant.
+    validations:
+      required: true
+  - type: textarea
+    id: alternatives
+    attributes:
+      label: Alternatives considered
+      description: Other approaches or workarounds you have tried.
+    validations:
+      required: false
+  - type: dropdown
+    id: area
+    attributes:
+      label: Area
+      options:
+        - Scraping / API
+        - Document extraction
+        - PostgreSQL sink
+        - MySQL / MariaDB sink
+        - SQLite / DuckDB sink
+        - MongoDB sink
+        - ClickHouse sink
+        - Neo4j sink
+        - SurrealDB sink
+        - Graph linking
+        - CLI / configuration
+        - Documentation
+        - Other
+    validations:
+      required: true

hkex_filing_scraper-2.1.0/.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,35 @@
+<!--
+  Thanks for contributing. Keep the description focused on the why and the impact.
+-->
+
+## Summary
+
+<!-- One or two sentences: what changes, and why. -->
+
+## Type of change
+
+- [ ] `feat` — new feature
+- [ ] `fix` — bug fix
+- [ ] `docs` — documentation only
+- [ ] `refactor` — no behavior change
+- [ ] `chore` — tooling, dependencies, maintenance
+- [ ] `test` — tests only
+
+## Related issues
+
+<!-- e.g. Closes #123 -->
+
+## How this was tested
+
+- [ ] `ruff check && ruff format --check` passes
+- [ ] `pytest -q` passes
+- [ ] Added or updated tests for the change
+- [ ] Verified against a live sink (which one? — PostgreSQL / MySQL / SQLite / DuckDB / MongoDB / ClickHouse / Neo4j / SurrealDB)
+
+## Checklist
+
+- [ ] New or changed sink fields are mirrored across every schema (SurrealDB, PostgreSQL, dialect DDL, MongoDB/Neo4j/ClickHouse)
+- [ ] SQL is parameterised; no values interpolated; credentials are never logged
+- [ ] Docs, templates, and `.env.example` updated (the docs consistency test covers this)
+- [ ] `CHANGELOG.md` updated under `Unreleased`
+- [ ] No secrets, tokens, or real credentials committed

hkex_filing_scraper-2.1.0/.github/dependabot.yml

@@ -0,0 +1,30 @@
+version: 2
+updates:
+  - package-ecosystem: pip
+    directory: "/"
+    schedule:
+      interval: weekly
+    open-pull-requests-limit: 5
+    groups:
+      python-minor:
+        patterns: ["*"]
+        update-types: ["minor", "patch"]
+    labels: ["dependencies"]
+
+  - package-ecosystem: uv
+    directory: "/"
+    schedule:
+      interval: weekly
+    open-pull-requests-limit: 5
+    labels: ["dependencies"]
+
+  - package-ecosystem: github-actions
+    directory: "/"
+    schedule:
+      interval: weekly
+    labels: ["dependencies", "ci"]
+
+# Container images are pinned by digest in .github/workflows/ci.yml and
+# examples/docker-compose.yml. Dependabot's `docker` ecosystem only understands Dockerfiles
+# and Kubernetes YAML (not compose files), so those digests are bumped manually when an
+# engine release is worth adopting.

hkex_filing_scraper-2.1.0/.github/workflows/canary.yml

@@ -0,0 +1,70 @@
+name: Canary
+
+# Early-warning control for risk R1: the undocumented HKEx API changing shape.
+# Runs the real fetch path against a small recent window once a day, and opens an issue
+# (or comments on the existing one) when the response no longer matches what the scraper
+# depends on. Target: detect an API break within 24 hours.
+
+on:
+  schedule:
+    - cron: "17 1 * * *"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  canary:
+    name: HKEx API canary
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      issues: write # create/comment on the tracking issue
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+
+      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+        with:
+          python-version: "3.12"
+
+      - name: Install the package
+        run: pip install --quiet .
+
+      - name: Run the canary
+        id: canary
+        run: |
+          set +e
+          set -o pipefail
+          python scripts/canary.py --days 7 --max 5 --json | tee canary.json
+          code=$?
+          echo "exit_code=$code" >> "$GITHUB_OUTPUT"
+
+      - name: Report a failure on the tracking issue
+        if: steps.canary.outputs.exit_code != '0'
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          set -euo pipefail
+          TITLE="Canary: HKEx API response shape changed"
+          BODY="$(printf 'The scheduled canary failed on %s.\n\n```json\n%s\n```\n\nSee the workflow run: %s/%s/actions/runs/%s' \
+            "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$(cat canary.json)" \
+            "$GITHUB_SERVER_URL" "$GITHUB_REPOSITORY" "$GITHUB_RUN_ID")"
+          EXISTING="$(gh issue list --state open --search "in:title \"$TITLE\"" --json number --jq '.[0].number // empty')"
+          if [ -n "$EXISTING" ]; then
+            gh issue comment "$EXISTING" --body "$BODY"
+          else
+            gh issue create --title "$TITLE" --body "$BODY" --label bug
+          fi
+
+      - name: Keep the report
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: canary-report
+          path: canary.json
+          if-no-files-found: ignore
+          retention-days: 14
+
+      - name: Fail the run when the canary failed
+        if: steps.canary.outputs.exit_code != '0'
+        run: exit 1