wo2net_delpher_tools 0.1.2__tar.gz

wo2net_delpher_tools-0.1.2/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Stichting WO2Net
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

wo2net_delpher_tools-0.1.2/PKG-INFO

@@ -0,0 +1,54 @@
+Metadata-Version: 2.3
+Name: wo2net_delpher_tools
+Version: 0.1.2
+Summary: Tools for extracting and processing Delpher data for WO2Net
+Author: Daan Raven
+Author-email: Daan Raven <daan.raven@wo2net.nl>
+License: MIT License
+         
+         Copyright (c) 2026 Stichting WO2Net
+         
+         Permission is hereby granted, free of charge, to any person obtaining a copy
+         of this software and associated documentation files (the "Software"), to deal
+         in the Software without restriction, including without limitation the rights
+         to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+         copies of the Software, and to permit persons to whom the Software is
+         furnished to do so, subject to the following conditions:
+         
+         The above copyright notice and this permission notice shall be included in all
+         copies or substantial portions of the Software.
+         
+         THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+         IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+         FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+         AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+         LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+         OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+         SOFTWARE.
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: Topic :: Text Processing :: Linguistic
+Requires-Dist: lxml>=6.0.2
+Requires-Dist: requests>=2.32.5
+Requires-Dist: sickle>=0.7.0
+Requires-Python: >=3.9.6
+Project-URL: Homepage, https://github.com/Stichting-WO2Net/wo2net-delpher-tools
+Project-URL: Repository, https://github.com/Stichting-WO2Net/wo2net-delpher-tools.git
+Project-URL: Issues, https://github.com/Stichting-WO2Net/wo2net-delpher-tools/issues
+Description-Content-Type: text/markdown
+
+# WO2Net Delpher Tools
+
+Tools for extracting and processing Delpher data for WO2Net.
+
+## Installatie
+
+```bash
+pip install wo2net-delpher-tools

wo2net_delpher_tools-0.1.2/README.md

@@ -0,0 +1,8 @@
+# WO2Net Delpher Tools
+
+Tools for extracting and processing Delpher data for WO2Net.
+
+## Installatie
+
+```bash
+pip install wo2net-delpher-tools

wo2net_delpher_tools-0.1.2/pyproject.toml

@@ -0,0 +1,39 @@
+[build-system]
+requires = ["uv>=0.9.24,<0.10.0"]
+build-backend = "uv_build"
+
+[tool.uv.extra-build-dependencies]
+wo2net_delpher_tools = ["uv_build"]
+
+[project]
+name = "wo2net_delpher_tools"
+version = "0.1.2"
+description = "Tools for extracting and processing Delpher data for WO2Net"
+readme = "README.md"
+requires-python = ">=3.9.6"
+authors = [
+    { name = "Daan Raven", email = "daan.raven@wo2net.nl" }
+]
+license = {file = "LICENSE"}
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+    "Intended Audience :: Developers",
+    "Intended Audience :: Science/Research",
+    "Topic :: Text Processing :: Linguistic",
+]
+dependencies = [
+    "lxml>=6.0.2",
+    "requests>=2.32.5",
+    "sickle>=0.7.0",
+]
+
+[project.urls]
+Homepage = "https://github.com/Stichting-WO2Net/wo2net-delpher-tools"
+Repository = "https://github.com/Stichting-WO2Net/wo2net-delpher-tools.git"
+Issues = "https://github.com/Stichting-WO2Net/wo2net-delpher-tools/issues"

wo2net_delpher_tools-0.1.2/src/wo2net_delpher_tools/__init__.py

@@ -0,0 +1,6 @@
+from .models.issue import Issue
+from .models.page import Page
+from .models.article import Article
+from .pipeline import process_ppn_to_json
+from .utils import save_issues_to_json
+from .ppn import load_ppn_numbers_from_txt

wo2net_delpher_tools-0.1.2/src/wo2net_delpher_tools/models/__init__.py

@@ -0,0 +1,3 @@
+from .issue import Issue
+from .page import Page
+from .article import Article

wo2net_delpher_tools-0.1.2/src/wo2net_delpher_tools/models/article.py

@@ -0,0 +1,28 @@
+from typing import Optional
+
+class Article:
+    def __init__(
+        self,
+        identifier: str,
+        title: str,
+        subject: Optional[str] = None,
+        url: Optional[str] = None,
+        ocr: Optional[str] = None,
+        ocr_text: Optional[str] = None
+    ):
+        self.identifier = identifier
+        self.title = title
+        self.subject = subject
+        self.url = url
+        self.ocr = ocr
+        self.ocr_text = ocr_text
+
+    def to_dict(self):
+        return {
+            'identifier': self.identifier,
+            'title': self.title,
+            'subject': self.subject,
+            'url': self.url,
+            'ocr': self.ocr,
+            'ocr_text': self.ocr_text
+        }

wo2net_delpher_tools-0.1.2/src/wo2net_delpher_tools/models/issue.py

@@ -0,0 +1,156 @@
+from typing import Optional
+from .article import Article
+from .page import Page
+from lxml import etree
+
+class Issue:
+    def __init__(
+        self,
+        identifier: str,
+        url: str,
+        ppn: str,
+        title: Optional[str] = None,
+        date: Optional[str] = None,
+        source: Optional[str] = None,
+        rights: Optional[str] = None,
+        publisher: Optional[str] = None,
+        volume: Optional[str] = None,
+        issue_number: Optional[str] = None,
+        issued: Optional[str] = None,
+        spatial: Optional[str] = None,
+        articles: Optional[list[Article]] = None,
+        pages: Optional[list[Page]] = None
+    ):
+        self.identifier = identifier
+        self.url = url
+        self.ppn = ppn
+        self.title = title
+        self.source = source
+        self.date = date
+        self.rights = rights
+        self.publisher = publisher
+        self.volume = volume
+        self.issue_number = issue_number
+        self.issued = issued
+        self.spatial = spatial
+        self.articles = articles or []
+        self.pages = pages or []
+    
+    @classmethod
+    def from_oai_record(cls, record):
+        root = etree.fromstring(record.raw)
+        ns = {
+            'didl': 'urn:mpeg:mpeg21:2002:02-DIDL-NS',
+            'dc': 'http://purl.org/dc/elements/1.1/',
+            'dcx': 'http://krait.kb.nl/coop/tel/handbook/telterms.html',
+            'oai': 'http://www.openarchives.org/OAI/2.0/',
+            'srw_dc': 'info:srw/schema/1/dc-v1.1',
+            'dcterms': 'http://purl.org/dc/terms/',
+            'dcmitype': 'http://purl.org/dc/dcmitype/',
+            'ddd': 'http://www.kb.nl/namespaces/ddd',
+            'didmodel': 'urn:mpeg:mpeg21:2002:02-DIDMODEL-NS',
+            'xsi': 'http://www.w3.org/2001/XMLSchema-instance',
+        }
+
+        def get_text(element, xpath, namespaces):
+            result = element.xpath(xpath, namespaces=namespaces)
+            return result[0].text if result else None
+        
+        def get_attribute(element, xpath, namespaces):
+            result = element.xpath(xpath, namespaces=namespaces)
+            return result[0] if result else None
+
+        issue_identifier = root.xpath('//oai:header/oai:identifier', namespaces = ns)[0].text.split('DDD:')[-1]
+        url = 'https://resolver.kb.nl/resolve?urn=' + issue_identifier.split(':mpeg21')[0]
+        item = root.xpath(f'//didl:Item[@dc:identifier="{issue_identifier}"]', namespaces = ns)[0]
+        metadata = item.xpath(f'./didl:Component[@dc:identifier="{issue_identifier}:metadata"]/didl:Resource/srw_dc:dcx', namespaces = ns)[0]
+        ppn = get_text(metadata, './dc:identifier[@xsi:type="dcx:PPN"]', ns)
+        title = get_text(metadata, './dc:title', ns)
+        date = get_text(metadata, './dc:date', ns)
+        source = get_text(metadata, './dc:source', ns)
+        rights = get_text(metadata, './dc:rights', ns)
+        publisher = get_text(metadata, './dc:publisher', ns)
+        volume = get_text(metadata, './dcx:volume', ns)
+        issue_number = get_text(metadata, './dcx:issuenumber', ns)
+        issued = get_text(metadata, './dcterms:issued', ns)
+        spatial = get_text(metadata, './dcterms:spatial', ns)
+
+        issue = cls(
+            identifier = issue_identifier,
+            url = url,
+            ppn = ppn,
+            title = title,
+            date = date,
+            source = source,
+            rights = rights,
+            publisher = publisher,
+            volume = volume,
+            issue_number = issue_number,
+            issued = issued,
+            spatial = spatial
+        )
+
+        article_indicator = f"{issue_identifier}:a"
+        article_items = item.xpath(f'./didl:Item[contains(@dc:identifier, "{article_indicator}")]', namespaces=ns)
+
+        for article in article_items:
+            identifier = article.get('{http://purl.org/dc/elements/1.1/}identifier')
+            article_metadata = article.xpath(f'./didl:Component[@dc:identifier="{identifier}:metadata"]/didl:Resource/srw_dc:dcx', namespaces = ns)[0]
+            title = get_text(article_metadata, './dc:title', ns)
+            subject = get_text(article_metadata, './dc:subject', ns)
+            url = get_text(article_metadata, './dc:identifier', ns)
+            ocr = get_attribute(article, f'./didl:Component[@dc:identifier="{identifier}:ocr"]/didl:Resource/@ref', ns)
+            article_object = Article (
+                identifier = identifier,
+                title = title,
+                subject = subject,
+                url = url,
+                ocr = ocr
+            )
+            issue.articles.append(article_object)
+
+        page_indicator = f"{issue_identifier}:p"
+        page_items = item.xpath(f'./didl:Item[contains(@dc:identifier, "{page_indicator}")]', namespaces=ns)
+        for page in page_items:
+            identifier = page.get('{http://purl.org/dc/elements/1.1/}identifier')
+            page_metadata = page.xpath(f'./didl:Component[@dc:identifier="{identifier}:metadata"]/didl:Resource/srw_dc:dcx', namespaces = ns)[0]
+            page_number = get_text(page_metadata, './ddd:nativePageNumber', ns)
+            image_url = get_attribute(page, f'./didl:Component[@dc:identifier="{identifier}:image"]/didl:Resource/@ref', ns)
+            page_url = get_text(page_metadata, './dc:identifier', ns)
+            
+            page_articles = []
+            page_object = Page(
+                identifier = identifier,
+                page_number = page_number,
+                image = image_url,
+                url = page_url,
+                articles = page_articles
+            )
+
+            page_article_items = page.xpath(f'.//didl:Item[@ddd:article_id]', namespaces=ns)
+            for page_article_item in page_article_items:
+                article_id = page_article_item.get('{http://www.kb.nl/namespaces/ddd}article_id')
+                for article in issue.articles:
+                    if article.identifier == article_id:
+                        page_object.articles.append(article)
+
+            issue.pages.append(page_object)
+        return issue
+
+    def to_dict(self):
+        return {
+            'identifier': self.identifier,
+            'url': self.url,
+            'ppn': self.ppn,
+            'title': self.title,
+            'date': self.date,
+            'source': self.source,
+            'rights': self.rights,
+            'publisher': self.publisher,
+            'volume': self.volume,
+            'issue_number': self.issue_number,
+            'issued': self.issued,
+            'spatial': self.spatial,
+            'articles': [article.to_dict() for article in self.articles],
+            'pages': [page.to_dict() for page in self.pages]
+        }

wo2net_delpher_tools-0.1.2/src/wo2net_delpher_tools/models/page.py

@@ -0,0 +1,26 @@
+from typing import Optional
+from .article import Article
+
+class Page:
+    def __init__(
+        self,
+        identifier: str,
+        page_number: Optional[int] = None,
+        image: Optional[str] = None,
+        url: Optional[str] = None,
+        articles: Optional[list[Article]] = None,
+    ):
+        self.identifier = identifier
+        self.page_number = page_number
+        self.image = image
+        self.url = url
+        self.articles = articles
+
+    def to_dict(self):
+        return {
+            'identifier': self.identifier,
+            'page_number': self.page_number,
+            'image': self.image,
+            'url': self.url,
+            'article_identifiers': [article.identifier for article in self.articles]
+        }

wo2net_delpher_tools-0.1.2/src/wo2net_delpher_tools/oai.py

@@ -0,0 +1,7 @@
+from sickle import Sickle
+
+def oai_get_record(identifier: str, prefix: str = 'DDD:'):
+    oai_identifier = prefix + identifier
+    sickle = Sickle('https://services.kb.nl/mdo/oai')
+    record = sickle.GetRecord(identifier=oai_identifier, metadataPrefix='didl')
+    return record

wo2net_delpher_tools-0.1.2/src/wo2net_delpher_tools/pipeline.py

@@ -0,0 +1,22 @@
+from .oai import oai_get_record
+from .sru import get_issue_identifiers
+from .utils import save_issues_to_json
+from .models.issue import Issue
+
+def process_ppn_to_json(ppn: str, output_file: str = "issues.json") -> list[Issue]:
+    print(f"Verwerken gestart voor PPN: {ppn}")
+
+    issue_identifiers = get_issue_identifiers(ppn)
+    print(f"Gevonden {len(issue_identifiers)} issue-identifiers voor PPN {ppn}")
+
+    issues = []
+    for identifier in issue_identifiers:
+        record = oai_get_record(identifier)
+        issue = Issue.from_oai_record(record)
+        issues.append(issue)
+        print(f"Issue verwerkt: {issue.identifier}")
+
+    save_issues_to_json(issues, output_file)
+    print(f"Verwerken voltooid. Resultaten opgeslagen in {output_file}")
+
+    return issues

wo2net_delpher_tools-0.1.2/src/wo2net_delpher_tools/ppn.py

@@ -0,0 +1,7 @@
+def load_ppn_numbers_from_txt(file_path: str) -> list[str]:
+    ppn_numbers = []
+    with open(file_path, encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            ppn_numbers.append(line)
+    return ppn_numbers    

wo2net_delpher_tools-0.1.2/src/wo2net_delpher_tools/sru.py

@@ -0,0 +1,41 @@
+import requests
+from urllib.parse import quote
+from lxml import etree
+
+def _get_delpher_item_identifiers_from_ppn(ppn_number: int, constraints: str = "(date >= 1940) AND (date <= 1945)", collection: str = 'DDD_artikel', max_records: int = 25):
+    identifiers = []
+    query = quote(f"{constraints} AND ppn any ({ppn_number})")
+    start_record = 1
+
+    while True:
+        url = f"https://jsru.kb.nl/sru/sru?operation=searchRetrieve&version=1.2&recordSchema=dc&query={query}&x-collection={collection}&maximumRecords={max_records}&startRecord={start_record}"
+    
+        response = requests.get(url)
+        
+        if response.status_code != 200:
+                raise Exception(f"SRU request failed: {response.status_code}")
+        
+        content = response.content
+        root = etree.fromstring(content)
+        ns = {
+            "srw": "http://www.loc.gov/zing/srw/",
+            "dc": "http://purl.org/dc/elements/1.1/"
+        }
+
+        nr_of_records = int(root.xpath("srw:numberOfRecords/text()", namespaces=ns)[0])
+        batch_identifiers = root.xpath("//srw:recordData/dc:identifier/text()", namespaces=ns)
+        identifiers.extend(batch_identifiers)
+
+
+        if start_record + max_records > nr_of_records:
+            break
+        start_record += max_records
+    return identifiers
+
+def get_issue_identifiers(ppn_number: int, constraints: str = "(date >= 1940) AND (date <= 1945)", collection: str = 'DDD_artikel', max_records: int = 25):
+    delpher_item_identifiers = _get_delpher_item_identifiers_from_ppn(ppn_number, constraints, collection, max_records)
+    issue_identifiers = set()
+    for item in delpher_item_identifiers:
+        issue_id = item.split("urn=")[1].split(':a0')[0]
+        issue_identifiers.add(issue_id)
+    return list(issue_identifiers)

wo2net_delpher_tools-0.1.2/src/wo2net_delpher_tools/utils.py

@@ -0,0 +1,8 @@
+import json
+from .models.issue import Issue
+
+def save_issues_to_json(issues: list[Issue], file_path: str) -> None:
+    issues_data = [issue.to_dict() for issue in issues]
+
+    with open(file_path, 'w') as f:
+        json.dump(issues_data, f, indent=4)