scrapping-cli 1.0.0__tar.gz

scrapping_cli-1.0.0/PKG-INFO

@@ -0,0 +1,3 @@
+Metadata-Version: 2.4
+Name: scrapping-cli
+Version: 1.0.0

scrapping_cli-1.0.0/mycli/cli.py

@@ -0,0 +1,19 @@
+import sys
+from mycli.commands import startproject
+
+COMMANDS = {
+    "startproject": startproject.run,
+}
+
+def main():
+    if len(sys.argv) < 3:
+        print("Usage: mycli <command> <project_name>")
+        return
+
+    command = sys.argv[1]
+    args = sys.argv[2:]
+
+    if command in COMMANDS:
+        COMMANDS[command](*args)
+    else:
+        print(f"Unknown command: {command}")

scrapping_cli-1.0.0/mycli/commands/startproject.py

@@ -0,0 +1,19 @@
+import os
+import shutil
+import pkg_resources
+
+def run(project_name):
+    template_dir = pkg_resources.resource_filename("mycli", "template")
+    target_dir = os.path.abspath(project_name)
+
+    if os.path.exists(target_dir):
+        print(f"Folder '{project_name}' already exists!")
+        return
+
+    shutil.copytree(
+        template_dir,
+        target_dir,
+        ignore=shutil.ignore_patterns("__pycache__", "*.pyc", ".DS_Store")
+    )
+
+    print(f"Project '{project_name}' created successfully!")

scrapping_cli-1.0.0/mycli/template/config.py

@@ -0,0 +1,23 @@
+import os
+from dotenv import load_dotenv
+
+load_dotenv()
+
+def get_env(key, default=None, required=False):
+    value = os.getenv(key, default)
+    if required and value is None:
+        raise ValueError(f"Missing required env variable: {key}")
+    return value
+
+DB_CONFIG = {
+    "host": get_env("DB_HOST", required=True),
+    "user": get_env("DB_USER", required=True),
+    "password": get_env("DB_PASSWORD", required=True),
+    "database": get_env("DB_NAME", required=True),
+}
+
+FOLDER_PATH = get_env("INPUT_FOLDER", required=True)
+OUTPUT_FOLDER_PATH = get_env("OUTPUT_FOLDER", "./output")
+
+MAX_WORKERS = int(get_env("MAX_WORKERS", 5))
+BATCH_SIZE = int(get_env("BATCH_SIZE", 500))

scrapping_cli-1.0.0/mycli/template/db.py

@@ -0,0 +1,37 @@
+import mysql.connector
+from config import DB_CONFIG
+from logger import setup_logger
+
+logger = setup_logger()
+
+def get_connection():
+    return mysql.connector.connect(
+        host=DB_CONFIG["host"],
+        user=DB_CONFIG["user"],
+        password=DB_CONFIG["password"],
+    )
+
+def get_connection_thread():
+    return mysql.connector.connect(**DB_CONFIG)
+
+def create_database(cursor):
+    db_name = DB_CONFIG["database"]
+    cursor.execute(f"CREATE DATABASE IF NOT EXISTS {db_name}")
+    cursor.execute(f"USE {db_name}")
+    logger.info(f"Database ready: {db_name}")
+
+def create_table(cursor):
+    cursor.execute("""
+    CREATE TABLE IF NOT EXISTS table_name(
+        id INT AUTO_INCREMENT PRIMARY KEY
+    )
+    """)
+    logger.info("Table created")
+
+def insert_multiple_data(cursor, data):
+    if not data:
+        return
+
+    query = "INSERT INTO table_name() VALUES ()"
+    cursor.executemany(query)
+    logger.info(f"Inserted {len(data)} records")

scrapping_cli-1.0.0/mycli/template/logger.py

@@ -0,0 +1,7 @@
+import logging
+
+def setup_logging():
+    logging.basicConfig(
+        level=logging.INFO,
+        format="%(asctime)s | %(levelname)s | %(name)s | %(message)s",
+    )

scrapping_cli-1.0.0/mycli/template/model.py

@@ -0,0 +1,5 @@
+from pydantic import BaseModel, field_validator
+import re
+from logger import setup_logger
+
+logger = setup_logger()

scrapping_cli-1.0.0/mycli/template/parser.py

@@ -0,0 +1,49 @@
+import json
+import gzip
+import os
+from logger import setup_logger
+
+logger = setup_logger()
+
+def load_file(file_path):
+    name, ext = os.path.splitext(file_path)
+
+    try:
+        if ext == ".gz":
+            with gzip.open(file_path, "rt", encoding="utf-8") as f:
+                return json.load(f)
+
+        elif ext == ".json":
+            with open(file_path, "r", encoding="utf-8") as f:
+                return json.load(f)
+
+        else:
+            logger.warning(f"Unsupported file: {file_path}")
+            return None
+
+    except Exception as e:
+        logger.error(f"File read error: {file_path} | {e}")
+        return None
+
+
+# MAIN PARSER ENTRY
+def parse_file(file_path):
+    raw = load_file(file_path)
+    if not raw:
+        return None
+
+    try:
+        return transform(raw)
+    except Exception as e:
+        logger.error(f"Parse error: {file_path} | {e}")
+        return None
+
+def transform(raw):
+    """
+    User writes extraction logic here
+    Supports:
+    - JSON
+    - lxml
+    - parsel
+    """
+    return raw

scrapping_cli-1.0.0/mycli/template/requirements.txt

@@ -0,0 +1,3 @@
+python-dotenv
+mysql-connector-python
+pydantic

scrapping_cli-1.0.0/mycli/template/simple_main.py

@@ -0,0 +1,31 @@
+import os
+from parser import parse_file
+from db import FOLDER_PATH, get_connection, create_table, insert_data, create_database
+from logger import setup_logger
+
+logger = setup_logger()
+def main():
+    conn = get_connection()
+    cursor = conn.cursor()
+    
+    create_database(cursor)
+    create_table(cursor)
+    
+    for file_name in os.listdir(FOLDER_PATH):
+        file_path = os.path.join(FOLDER_PATH, file_name)
+        print(f"Processing: {file_name}")
+
+        try:
+            data = parse_file(file_path)
+            insert_data(cursor,data)
+            conn.commit()
+
+        except Exception as e:
+            print(f"Error in {file_name}: {e}")
+
+    cursor.close()
+    conn.close()
+    print("Done!")
+
+if __name__ == "__main__":
+    main()

scrapping_cli-1.0.0/mycli/template/thread_main.py

@@ -0,0 +1,60 @@
+import os
+import time
+
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
+from parser import parse_file
+from db import get_connection, get_connection_thread, create_table, create_database, insert_multiple_data
+from config import FOLDER_PATH, BATCH_SIZE, MAX_WORKERS
+from logger import setup_logger
+
+logger = setup_logger()
+
+def insert_batch(batch):
+    conn = get_connection_thread()
+    cursor = conn.cursor()
+
+    insert_multiple_data(cursor, batch)
+    conn.commit()
+
+    cursor.close()
+    conn.close()
+
+def main():
+    start = time.time()
+
+    conn = get_connection()
+    cursor = conn.cursor()
+    create_database(cursor)
+    create_table(cursor)
+    conn.commit()
+    cursor.close()
+    conn.close()
+
+    batch = []
+    futures = []
+
+    with ThreadPoolExecutor(MAX_WORKERS) as parser_pool:
+        tasks = {
+            parser_pool.submit(parse_file, os.path.join(FOLDER_PATH, f)): f
+            for f in os.listdir(FOLDER_PATH)
+        }
+
+        with ThreadPoolExecutor(MAX_WORKERS) as db_pool:
+            for future in as_completed(tasks):
+                result = future.result()
+
+                if result:
+                    batch.append(result)
+
+                if len(batch) >= BATCH_SIZE:
+                    futures.append(db_pool.submit(insert_batch, batch.copy()))
+                    batch.clear()
+
+            if batch:
+                futures.append(db_pool.submit(insert_batch, batch.copy()))
+
+            for f in futures:
+                f.result()
+
+    logger.info(f"Runtime: {time.time() - start}")

scrapping_cli-1.0.0/pyproject.toml

@@ -0,0 +1,3 @@
+[build-system]
+requires = ["setuptools>=61.0"]
+build-backend = "setuptools.build_meta"

scrapping_cli-1.0.0/scrapping_cli.egg-info/PKG-INFO

@@ -0,0 +1,3 @@
+Metadata-Version: 2.4
+Name: scrapping-cli
+Version: 1.0.0

scrapping_cli-1.0.0/scrapping_cli.egg-info/SOURCES.txt

@@ -0,0 +1,19 @@
+pyproject.toml
+setup.py
+mycli/__init__.py
+mycli/cli.py
+mycli/commands/__init__.py
+mycli/commands/startproject.py
+mycli/template/config.py
+mycli/template/db.py
+mycli/template/logger.py
+mycli/template/model.py
+mycli/template/parser.py
+mycli/template/requirements.txt
+mycli/template/simple_main.py
+mycli/template/thread_main.py
+scrapping_cli.egg-info/PKG-INFO
+scrapping_cli.egg-info/SOURCES.txt
+scrapping_cli.egg-info/dependency_links.txt
+scrapping_cli.egg-info/entry_points.txt
+scrapping_cli.egg-info/top_level.txt

scrapping_cli-1.0.0/scrapping_cli.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

scrapping_cli-1.0.0/scrapping_cli.egg-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+scrapping-cli = mycli.cli:main

scrapping_cli-1.0.0/scrapping_cli.egg-info/top_level.txt

@@ -0,0 +1 @@
+mycli

scrapping_cli-1.0.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

scrapping_cli-1.0.0/setup.py

@@ -0,0 +1,16 @@
+from setuptools import setup, find_packages
+
+setup(
+    name="scrapping-cli",
+    version="1.0.0",   
+    packages=find_packages(),
+    include_package_data=True,
+    package_data={
+        "mycli": ["template/**/*"],
+    },
+    entry_points={
+        'console_scripts': [
+            'scrapping-cli=mycli.cli:main',
+        ],
+    },
+)