tulit 0.0.1__py3-none-any.whl

tulit/__init__.py

@@ -0,0 +1,4 @@
+"""
+This package provides functions to send a SPARQL query to the SPARQL endpoint of the Publications Office of the European Union (CELLAR),
+retrieve the results, and store them in a JSON file.
+"""

tulit/download/__init__.py

@@ -0,0 +1,23 @@
+"""
+This module provides functionality for [describe the main purpose of the module].
+
+Classes:
+    [ClassName]: [Brief description of the class]
+
+Functions:
+    [function_name_1]: [Brief description of the function]
+    [function_name_2]: [Brief description of the function]
+    ...
+
+Usage:
+    [Provide a brief example of how to use the module]
+
+Notes:
+    [Any additional notes or important information about the module]
+
+Author:
+    [Your Name]
+
+Date:
+    [Date of creation or last modification]
+"""

tulit/download/cellar.py

@@ -0,0 +1,163 @@
+import logging
+
+import requests
+import json
+from tulit.download.download import DocumentDownloader
+
+class CellarDownloader(DocumentDownloader):
+    
+    def __init__(self, download_dir, log_dir):
+        super().__init__(download_dir, log_dir)
+        self.endpoint = 'http://publications.europa.eu/resource/cellar/'
+   
+
+    def fetch_content(self, url) -> requests.Response:
+        """
+        Send a GET request to download a file
+
+        Parameters
+        ----------
+        url : str
+            The URL to send the request to.
+
+        Returns
+        -------
+        requests.Response
+            The response from the server.
+
+        Notes
+        -----
+        The request is sent with the following headers:
+        - Accept: application/zip;mtype=fmx4, application/xml;mtype=fmx4, application/xhtml+xml, text/html, text/html;type=simplified, application/msword, text/plain, application/xml;notice=object
+        - Accept-Language: eng
+        - Content-Type: application/x-www-form-urlencoded
+        - Host: publications.europa.eu
+
+        Raises
+        ------
+        requests.RequestException
+            If there is an error sending the request.
+
+        See Also
+        --------
+        requests : The underlying library used for making HTTP requests.
+
+        """
+        try:
+            headers = {
+                'Accept': "*, application/zip, application/zip;mtype=fmx4, application/xml;mtype=fmx4, application/xhtml+xml, text/html, text/html;type=simplified, application/msword, text/plain, application/xml, application/xml;notice=object",
+                'Accept-Language': "eng",
+                'Content-Type': "application/x-www-form-urlencoded",
+                'Host': "publications.europa.eu"
+            }
+            response = requests.request("GET", url, headers=headers)
+            response.raise_for_status()
+            return response
+        except requests.RequestException as e:
+            logging.error(f"Error sending GET request: {e}")
+            return None
+             
+    def build_request_url(self, params):
+        """
+        Build the request URL based on the source and parameters.
+        """
+        url = f"{self.endpoint}{params['cellar']}"
+        
+        return url
+    
+    def get_cellar_ids_from_json_results(self, results, format):
+        """
+        Extract CELLAR ids from a JSON dictionary.
+
+        Parameters
+        ----------
+        cellar_results : dict
+            A dictionary containing the response of the CELLAR SPARQL query
+
+        Returns
+        -------
+        list
+            A list of CELLAR ids.
+
+        Notes
+        -----
+        The function assumes that the JSON dictionary has the following structure:
+        - The dictionary contains a key "results" that maps to another dictionary.
+        - The inner dictionary contains a key "bindings" that maps to a list of dictionaries.
+        - Each dictionary in the list contains a key "cellarURIs" that maps to a dictionary.
+        - The innermost dictionary contains a key "value" that maps to a string representing the CELLAR URI.
+
+        The function extracts the CELLAR id by splitting the CELLAR URI at "cellar/" and taking the second part.
+
+        Examples
+        --------
+        >>> cellar_results = {
+        ...     "results": {
+        ...         "bindings": [
+        ...             {"cellarURIs": {"value": "https://example.com/cellar/some_id"}},
+        ...             {"cellarURIs": {"value": "https://example.com/cellar/another_id"}}
+        ...         ]
+        ...     }
+        ... }
+        >>> cellar_ids = get_cellar_ids_from_json_results(cellar_results)
+        >>> print(cellar_ids)
+        ['some_id', 'another_id']
+        """
+        cellar_ids = []
+        results_list = results["results"]["bindings"]
+        for i, file in enumerate(results_list):
+            if file['format']['value'] == format:
+                cellar_ids.append(file['cellarURIs']["value"].split("cellar/")[1])
+
+        return cellar_ids
+
+    def download(self, results, format=None):
+        """
+        Sends a REST query to the specified source APIs and downloads the documents
+        corresponding to the given results.
+
+        Parameters
+        ----------
+        results : dict
+            A dictionary containing the JSON results from the APIs.
+        format : str, optional
+            The format of the documents to download.        
+
+        Returns
+        -------
+        list
+            A list of paths to the downloaded documents.
+        """
+        
+        cellar_ids = self.get_cellar_ids_from_json_results(results, format=format)
+        
+        try:
+            document_paths = []
+            
+            for id in cellar_ids:
+                # Build the request URL
+                url = self.build_request_url(params={'cellar': id})
+                
+                # Send the GET request
+                response = self.fetch_content(url)
+                # Handle the response
+                file_path = self.handle_response(response=response, filename=id)
+                # Append the file path to the list
+                document_paths.append(file_path)
+                
+            return document_paths
+
+        except Exception as e:
+            logging.error(f"Error processing range: {e}")
+        
+        return document_paths
+      
+    
+# Example usage
+if __name__ == "__main__":
+    downloader = CellarDownloader(download_dir='./tests/data/formex', log_dir='./tests/logs')
+    with open('./tests/metadata/query_results/query_results.json', 'r') as f:
+        results = json.loads(f.read())
+    documents = downloader.download(results, format='fmx4')
+    print(documents)
+    

tulit/download/download.py

@@ -0,0 +1,85 @@
+import os
+import io
+import logging
+import zipfile
+import requests
+
+class DocumentDownloader:
+    """	
+    A generic document downloader class.
+    """	
+    def __init__(self, download_dir, log_dir):
+        self.download_dir = download_dir
+        self.log_dir = log_dir
+        self._ensure_directories()
+
+    def _ensure_directories(self):
+        if not os.path.exists(self.download_dir):
+            os.makedirs(self.download_dir)
+        if not os.path.exists(self.log_dir):
+            os.makedirs(self.log_dir)
+    
+    def handle_response(self, response, filename):
+        """
+        Handle a server response by saving or extracting its content.
+
+        Parameters
+        ----------
+        response : requests.Response
+            The HTTP response object.
+        folder_path : str
+            Directory where the file will be saved.
+        cid : str
+            CELLAR ID of the document.
+
+        Returns
+        -------
+        str or None
+            Path to the saved file or None if the response couldn't be processed.
+        """
+        content_type = response.headers.get('Content-Type', '')
+        
+        # The return file is usually either a zip file, or a file with the name DOC_* inside a folder named as the cellar_id
+        target_path = os.path.join(self.download_dir, filename)
+        os.makedirs(os.path.dirname(target_path), exist_ok=True)
+
+        if 'zip' in content_type:
+            self.extract_zip(response, target_path)
+            return target_path
+        else:
+            extension = self.get_extension_from_content_type(content_type)
+            if not extension:
+                logging.warning(f"Unknown content type for ID {filename}: {content_type}")
+                return None
+
+            file_path = f"{target_path}.{extension}"
+            file_path = os.path.normpath(file_path)
+            
+            with open(file_path, mode='wb+') as f:
+                f.write(response.content)            
+                
+            return file_path
+        
+    def get_extension_from_content_type(self, content_type):
+        """Map Content-Type to a file extension."""
+        content_type_mapping = {
+            'text/html': 'html',
+            'application/json': 'json',
+            'application/xml': 'xml',
+            'text/plain': 'txt',
+            'application/zip': 'zip',
+            'text/xml': 'xml',
+            'application/xhtml+xml': 'xhtml',
+        }
+        for ext, mapped_ext in content_type_mapping.items():
+            if ext in content_type:
+                return mapped_ext
+
+    # Function to download a zip file and extract it
+    def extract_zip(self, response: requests.Response, folder_path: str):
+        try:
+            z = zipfile.ZipFile(io.BytesIO(response.content))
+            z.extractall(folder_path)
+        except Exception as e:
+            logging.error(f"Error downloading zip: {e}")
+

tulit/download/legilux.py

@@ -0,0 +1,39 @@
+import requests
+from tulit.download.download import DocumentDownloader
+
+class LegiluxDownloader(DocumentDownloader):
+    def __init__(self, download_dir, log_dir):
+        super().__init__(download_dir, log_dir)
+        #self.endpoint = "https://legilux.public.lu/eli/etat/leg/loi"
+
+    def build_request_url(self, eli) -> str:
+        """
+        Build the request URL based on the source and parameters.
+        """
+        url = eli
+        return url
+    
+    def fetch_content(self, url):
+        """
+        Fetch the content of the document.
+        """
+        headers = {"Accept": "application/xml"}
+        response = requests.get(url, headers=headers)
+        return response
+
+    def download(self, eli):
+        file_paths = []
+        url = self.build_request_url(eli)
+        response = self.fetch_content(url)        
+        filename = eli.split('loi/')[1].replace('/', '_')
+        if response.status_code == 200:
+            file_paths.append(self.handle_response(response, filename=filename))
+            print(f"Document downloaded successfully and saved to {file_paths}")
+            return file_paths
+        else:
+            print(f"Failed to download document. Status code: {response.status_code}")
+            return None
+
+if __name__ == "__main__":
+    downloader = LegiluxDownloader(download_dir='./tests/data/legilux', log_dir='./tests/metadata/logs')
+    downloader.download(eli='http://data.legilux.public.lu/eli/etat/leg/loi/2006/07/31/n2/jo')

tulit/download/normattiva.py

@@ -0,0 +1,97 @@
+from tulit.download.download import DocumentDownloader
+import requests
+import logging
+from datetime import datetime
+
+
+class NormattivaDownloader(DocumentDownloader):
+    def __init__(self, download_dir, log_dir):
+        super().__init__(download_dir, log_dir)
+        self.endpoint = "https://www.normattiva.it/do/atto/caricaAKN"
+    
+    def build_request_url(self, params=None) -> str:
+        """
+        Build the request URL based on the source and parameters.
+        """
+        uri = f"https://www.normattiva.it/eli/id/{params['date']}//{params['codiceRedaz']}/CONSOLIDATED"
+        # In case we want to use the NIR:URI instead of ELI
+        #uri = f"https://www.normattiva.it/uri-res/N2Ls?urn:nir:stato:decreto.legislativo:{params['date']};{params['number']}"
+        url = f"{self.endpoint}?dataGU={params['dataGU']}&codiceRedaz={params['codiceRedaz']}&dataVigenza={params['dataVigenza']}"
+        
+        return uri, url
+                    
+    def fetch_content(self, uri, url) -> requests.Response:
+        """
+        Send a GET request to download a file
+
+        Parameters
+        ----------
+        url : str
+            The URL to send the request to.
+
+        Returns
+        -------
+        requests.Response
+            The response from the server.
+
+        Raises
+        ------
+        requests.RequestException
+            If there is an error sending the request.
+        """
+        try:
+            
+            # Make a GET request to the URI to get the cookies        
+            cookies_response = requests.get(uri)
+            cookies_response.raise_for_status()
+            cookies = cookies_response.cookies
+
+            headers = {
+                'Accept': "text/xml",
+                'Accept-Encoding': "gzip, deflate, br, zstd",
+                'Accept-Language': "en-US,en;q=0.9",
+                
+            }                     
+            response = requests.get(url, headers=headers, cookies=cookies)
+            response.raise_for_status()
+            return response
+        except requests.RequestException as e:
+            logging.error(f"Error sending GET request: {e}")
+            return None    
+        
+    def download(self, dataGU, codiceRedaz, dataVigenza = datetime.today().strftime('%Y%m%d')):     
+        document_paths = []
+        
+        # Convert the dataGU to a datetime object
+        dataGU = datetime.strptime(dataGU, '%Y%m%d')
+            
+        params = {
+            # dataGU as a string in the format YYYYMMDD
+            'dataGU': dataGU.strftime('%Y%m%d'),
+            'codiceRedaz': codiceRedaz,
+            'dataVigenza': dataVigenza,
+            # dataGU as a string in the format YYYY/MM/DD
+            'date': dataGU.strftime('%Y/%m/%d')
+        }
+        
+        uri, url = self.build_request_url(params)
+        response = self.fetch_content(uri, url)
+        
+        # If the response in HTML, raise an error saying that the date or codiceRedaz is wrong
+        if 'text/html' in response.headers.get('Content-Type', ''):
+            logging.error(f"Error downloading document: there is not an XML file with the following parameters: {params}")
+            return None
+        
+        file_path = self.handle_response(response=response, filename=f"{params['dataGU']}_{params['codiceRedaz']}_VIGENZA_{params['dataVigenza']}")
+        document_paths.append(file_path)
+        return document_paths
+
+# Example usage
+if __name__ == "__main__":
+    
+    downloader = NormattivaDownloader(download_dir='./tests/data/akn/italy', log_dir='./tests/logs')
+    #documents = downloader.download(dataGU='19410716', codiceRedaz='041U0633')
+    documents = downloader.download(dataGU='19410716', codiceRedaz='041U0633', dataVigenza='20211231')
+
+    
+    print(documents)

tulit/main.py

@@ -0,0 +1,64 @@
+import json
+import logging
+import os
+from tulit.download.download import download_documents
+from sparql import send_sparql_query
+from parsers.html import HTMLParser
+from parsers.formex import Formex4Parser
+
+def main():
+    """
+    Main function to execute SPARQL query and download documents
+    """
+    # Configure logging
+    logging.basicConfig(level=logging.INFO)
+    logger = logging.getLogger(__name__)
+
+    try:
+
+        # Send SPARQL query
+        logger.info("Executing SPARQL query")
+        results = send_sparql_query('./tests/metadata/queries/formex_query.rq', celex='32008R1137')
+        
+
+        # Save query results to JSON
+        results_file = './tests/metadata/query_results/query_results.json'
+        with open(results_file, "w") as f:
+            json.dump(results, f, indent=4)
+            logger.info(f"Results dumped in {results_file}")
+
+        # Load query results
+        with open('./tests/metadata/query_results/query_results.json', 'r') as f:
+            results = json.loads(f.read())
+
+        # Download documents
+        logger.info("Downloading documents")
+        downloaded_document_paths = download_documents(
+            results, 
+            './tests/data/formex', 
+            log_dir='./tests/logs', 
+            format='fmx4'
+        )
+        logger.info(f'{len(downloaded_document_paths)} documents downloaded in {downloaded_document_paths}')
+                
+        # Extract the directory path (removing what's after the last '/')
+
+        # List the contents of the first directory
+        first_path = downloaded_document_paths[0]
+        first_item = os.listdir(first_path)[0]
+        file_path = os.path.join(*first_path.split('/'), first_item)
+    
+        print(f'Parsing {file_path}')
+        # Sort the contents alphabetically and get the first item
+        
+        parser = Formex4Parser()
+        parser.parse(file_path)
+        print(parser.articles)
+        #print(document_tree)
+
+    except Exception as e:
+        logger.error(f"An error occurred: {e}")
+        raise
+
+if __name__ == "__main__":
+    main()

tulit/parsers/__init__.py

@@ -0,0 +1,3 @@
+"""
+This subpackage provides classes and functions to parse a given file and extract the text within it.
+"""