datamarket 0.9.26__py3-none-any.whl → 0.9.28__py3-none-any.whl

datamarket/interfaces/nominatim.py

@@ -1,7 +1,10 @@
 ########################################################################################################################
 # IMPORTS
 
+import gettext
 import logging
+import pycountry
+from geopy.distance import geodesic
 
 import requests
 
@@ -11,6 +14,8 @@ from ..params.nominatim import POSTCODES
 # CLASSES
 
 logger = logging.getLogger(__name__)
+spanish = gettext.translation("iso3166-1", pycountry.LOCALES_DIR, languages=["es"])
+spanish.install()
 
 
 class GeoNames:
@@ -63,28 +68,102 @@ class Nominatim:
         return requests.get(f"{self.endpoint}/reverse?lat={lat}&lon={lon}&format=json").json()
 
     def reverse_parsed(self, lat, lon):
-        raw_json = self.reverse(lat, lon).get("address", {})
-        geoname = self.geonames.reverse(lat, lon)
-
-        postcode = self.geonames.validate_postcode(
-            str(geoname.get("postal_code", ""))
-        ) or self.geonames.validate_postcode(str(raw_json.get("postcode")))
-
-        city = self.get_attribute(raw_json, ["city", "town", "village"]) or geoname.get("place_name")
-
-        district, quarter = self.get_district_quarter(raw_json)
-        return {
-            "country": raw_json.get("country"),
-            "country_code": (raw_json.get("country_code") or geoname.get("country_code") or "").lower(),
-            "state": raw_json.get("state") or geoname.get("community"),
-            "province": self.geonames.get_province_from_postcode(postcode),
-            "city": city,
-            "postcode": postcode,
-            "district": district,
-            "quarter": quarter,
-            "street": raw_json.get("road"),
-            "number": raw_json.get("house_number"),
-        }
+        nominatim_raw_json = self.reverse(lat, lon)
+        geonames_raw_json = self.geonames.reverse(lat, lon)
+
+        nominatim_res_lat_str = nominatim_raw_json.get("lat")
+        nominatim_res_lon_str = nominatim_raw_json.get("lon")
+        geonames_res_lat_str = geonames_raw_json.get("lat")
+        geonames_res_lon_str = geonames_raw_json.get("lon")
+
+        dist_nominatim = float("inf")
+        dist_geonames = float("inf")
+
+        try:
+            input_coords = (float(lat), float(lon))
+        except (ValueError, TypeError):
+            logger.error(f"Invalid input coordinates for distance calculation: lat={lat}, lon={lon}")
+        else:
+            if nominatim_res_lat_str and nominatim_res_lon_str:
+                try:
+                    nominatim_coords = (float(nominatim_res_lat_str), float(nominatim_res_lon_str))
+                    dist_nominatim = geodesic(input_coords, nominatim_coords).km
+                except (ValueError, TypeError):
+                    logger.warning("Invalid Nominatim coordinates for distance calculation.")
+
+            if geonames_res_lat_str and geonames_res_lon_str:
+                try:
+                    geonames_coords = (float(geonames_res_lat_str), float(geonames_res_lon_str))
+                    dist_geonames = geodesic(input_coords, geonames_coords).km
+                except (ValueError, TypeError):
+                    logger.warning("Invalid GeoNames coordinates for distance calculation.")
+
+        if dist_nominatim <= dist_geonames and nominatim_res_lat_str is not None and nominatim_res_lon_str is not None:
+            # Use Nominatim data
+            raw_address = nominatim_raw_json.get("address", {})
+            postcode_str = str(raw_address.get("postcode", ""))
+            postcode = self.geonames.validate_postcode(postcode_str)
+            province = self.geonames.get_province_from_postcode(postcode) if postcode else None
+            city = self.get_attribute(raw_address, ["city", "town", "village"])
+            district, quarter = self.get_district_quarter(raw_address)
+
+            return {
+                "country": raw_address.get("country"),
+                "country_code": (raw_address.get("country_code") or "").lower(),
+                "state": raw_address.get("state"),
+                "province": province,
+                "city": city,
+                "postcode": postcode,
+                "district": district,
+                "quarter": quarter,
+                "street": raw_address.get("road"),
+                "number": raw_address.get("house_number"),
+            }
+
+        elif dist_geonames < dist_nominatim and geonames_res_lat_str is not None and geonames_res_lon_str is not None:
+            # Use GeoNames data
+            geonames_country_code_str = geonames_raw_json.get("country_code")
+            country_name = None
+            if geonames_country_code_str:
+                try:
+                    country_obj = pycountry.countries.get(alpha_2=geonames_country_code_str.upper())
+                    if country_obj:
+                        country_name = spanish.gettext(country_obj.name)
+                except LookupError:
+                    logger.warning(f"Country name not found for code: {geonames_country_code_str} using pycountry.")
+
+            postcode_str = str(geonames_raw_json.get("postal_code", ""))
+            postcode = self.geonames.validate_postcode(postcode_str)
+            province = self.geonames.get_province_from_postcode(postcode) if postcode else None
+            city = geonames_raw_json.get("place_name")
+
+            return {
+                "country": country_name,
+                "country_code": (geonames_country_code_str or "").lower(),
+                "state": geonames_raw_json.get("community"),
+                "province": province,
+                "city": city,
+                "postcode": postcode,
+                "district": None,
+                "quarter": None,
+                "street": None,
+                "number": None,
+            }
+
+        else:
+            # Neither source provided valid coordinates
+            return {
+                "country": None,
+                "country_code": None,
+                "state": None,
+                "province": None,
+                "city": None,
+                "postcode": None,
+                "district": None,
+                "quarter": None,
+                "street": None,
+                "number": None,
+            }
 
     def get_district_quarter(self, raw_json):
         district = self.get_attribute(raw_json, ["city_district", "suburb", "borough"])

datamarket/utils/strings.py

@@ -37,8 +37,14 @@ class NamingConvention(Enum):
 
 def transliterate_symbols(s: str) -> str:
     """
-    Translates symbols (category S*) to lowercase Unicode names,
-    with spaces→underscores. The rest of the text remains the same.
+    Translates Unicode symbols (category S*) in the input string to their lowercase Unicode names,
+    with spaces replaced by underscores. Other characters remain unchanged.
+
+    Args:
+        s: The input string.
+
+    Returns:
+        The string with symbols transliterated.
     """
     out: list[str] = []
     for c in s:
@@ -55,16 +61,32 @@ def normalize(
     s: Any, mode: NormalizationMode = NormalizationMode.BASIC, naming: NamingConvention = NamingConvention.NONE
 ) -> str:
     """
-    1. Normalizes the string according to `mode`:
-       - NONE: returns the original input as an unprocessed string.
-       - BASIC: removes accents, converts punctuation to spaces, preserves alphanumeric characters.
-       - SYMBOLS: translates only symbols to Unicode name.
-       - FULL: combines BASIC + SYMBOLS.
-    2. Applies naming convention according to `naming`:
-       - NONE: returns the normalized text.
-       - PARAM: parameterize (hyphens).
-       - SNAKE: snake_case (underscore, lowercase).
-       - CONSTANT: CONSTANT_CASE (underscore, uppercase).
+    Normalizes and applies a naming convention to the input.
+
+    Handles None and NaN values by returning an empty string. Converts non-string inputs to strings.
+
+    Normalization is applied according to `mode`:
+    - NONE: Returns the input as a string without any normalization.
+    - BASIC: Removes accents, converts punctuation and spaces to single spaces, and preserves alphanumeric characters.
+    - SYMBOLS: Translates only Unicode symbols (category S*) to their lowercase Unicode names with underscores.
+    - FULL: Applies both BASIC and SYMBOLS normalization.
+
+    After normalization, a naming convention is applied according to `naming`:
+    - NONE: Returns the normalized text.
+    - CONSTANT: Converts to CONSTANT_CASE (uppercase with underscores).
+    - SNAKE: Converts to snake_case (lowercase with underscores).
+    - CAMEL: Converts to camelCase (lowercase first word, capitalize subsequent words, no spaces).
+    - PASCAL: Converts to PascalCase (capitalize all words, no spaces).
+    - PARAM: Converts to parameterize (lowercase with hyphens).
+    - TITLE: Converts to Title Case (capitalize each word).
+
+    Args:
+        s: The input value to normalize and format. Can be any type.
+        mode: The normalization mode to apply. Defaults to NormalizationMode.BASIC.
+        naming: The naming convention to apply. Defaults to NamingConvention.NONE.
+
+    Returns:
+        The normalized and formatted string.
     """
     # Parameter mapping
     if isinstance(mode, str):

{datamarket-0.9.26.dist-info → datamarket-0.9.28.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: datamarket
-Version: 0.9.26
+Version: 0.9.28
 Summary: Utilities that integrate advanced scraping knowledge into just one library.
 License: GPL-3.0-or-later
 Author: DataMarket
@@ -28,7 +28,6 @@ Provides-Extra: duckduckgo-search
 Provides-Extra: fake-useragent
 Provides-Extra: geoalchemy2
 Provides-Extra: geopandas
-Provides-Extra: geopy
 Provides-Extra: google-api-python-client
 Provides-Extra: google-auth-httplib2
 Provides-Extra: google-auth-oauthlib
@@ -77,7 +76,7 @@ Requires-Dist: dynaconf (>=3.0.0,<4.0.0)
 Requires-Dist: fake-useragent (>=2.0.0,<3.0.0) ; extra == "fake-useragent"
 Requires-Dist: geoalchemy2 (>=0.17.0,<0.18.0) ; extra == "geoalchemy2"
 Requires-Dist: geopandas (>=1.0.0,<2.0.0) ; extra == "geopandas"
-Requires-Dist: geopy (>=2.0.0,<3.0.0) ; extra == "geopy"
+Requires-Dist: geopy (>=2.0.0,<3.0.0)
 Requires-Dist: google-api-python-client (>=2.0.0,<3.0.0) ; extra == "google-api-python-client"
 Requires-Dist: google-auth-httplib2 (>=0.2.0,<0.3.0) ; extra == "google-auth-httplib2"
 Requires-Dist: google-auth-oauthlib (>=1.0.0,<2.0.0) ; extra == "google-auth-oauthlib"
@@ -98,6 +97,7 @@ Requires-Dist: playwright (==1.47.0) ; extra == "playwright"
 Requires-Dist: pre-commit (>=4.0.0,<5.0.0)
 Requires-Dist: psycopg2-binary (>=2.0.0,<3.0.0)
 Requires-Dist: pyarrow (>=19.0.0,<20.0.0) ; extra == "pyarrow"
+Requires-Dist: pycountry (>=24.0.0,<25.0.0)
 Requires-Dist: pydrive2 (>=1.0.0,<2.0.0) ; extra == "pydrive2" or extra == "drive"
 Requires-Dist: pymupdf (>=1.0.0,<2.0.0) ; extra == "pymupdf"
 Requires-Dist: pysocks (>=1.0.0,<2.0.0) ; extra == "pysocks"

{datamarket-0.9.26.dist-info → datamarket-0.9.28.dist-info}/RECORD

@@ -4,7 +4,7 @@ datamarket/interfaces/alchemy.py,sha256=4q_gLKCKPK437VKOpdBKSrCyy42P_yWxIhE7KuvH
 datamarket/interfaces/aws.py,sha256=7KLUeBxmPN7avEMPsu5HC_KHB1N7W6Anp2X8fo43mlw,2383
 datamarket/interfaces/drive.py,sha256=shbV5jpQVe_KPE-8Idx6Z9te5Zu1SmVfrvSAyd9ZIgE,2915
 datamarket/interfaces/ftp.py,sha256=o0KlJxtksbop9OjCiQRzyAa2IeG_ExVXagS6apwrAQo,1881
-datamarket/interfaces/nominatim.py,sha256=ysIA2J1GhsZ0TJxD6B8N1_a7dkMEqtZQV6mT4Hayecg,3672
+datamarket/interfaces/nominatim.py,sha256=TD4OhWdIwn53Va41BS1ugogHEZw0ANKxWIfc9G2JWLU,7280
 datamarket/interfaces/peerdb.py,sha256=cwYwvO740GyaPo9zLAwJsf3UeJDGDiYzjQVM9Q6s-_g,23652
 datamarket/interfaces/proxy.py,sha256=updoOStKd8-nQBbxWbnD9eOt6HksnYi-5dQ0rEySf5M,3152
 datamarket/interfaces/tinybird.py,sha256=AYrcRGNOCoCt7ojilkWa27POROee9sTCwZ61GGHEPeM,2698
@@ -16,10 +16,10 @@ datamarket/utils/alchemy.py,sha256=SRq6kgh1aANXVShBPgAuglmNhZssPWwWEY503gKSia8,6
 datamarket/utils/main.py,sha256=j8wnAxeLvijdRU9M4V6HunWH7vgWWHP4u4xamzkWcUU,7009
 datamarket/utils/selenium.py,sha256=IMKlbLzXABFhACnWzhHmB0l2hhVzNwHGZwbo14nEewQ,2499
 datamarket/utils/soda.py,sha256=eZTXFbI1P3WoMd1MM-YjoVTpdjTcDSWuvBb7ViBMhSQ,941
-datamarket/utils/strings.py,sha256=3zLZxoPvE4E4np-q-Z6tfEhAvrN22W7DfMmsroBxVV8,4648
+datamarket/utils/strings.py,sha256=rEX9NeBG4C7RECgT0EQebgoFoxgZMy9-7EcBSxgBANU,5654
 datamarket/utils/typer.py,sha256=FDF3l6gh3UlAFPsHCtesnekvct2rKz0oFn3uKARBQvE,814
 datamarket/utils/types.py,sha256=vxdQZdwdXrfPR4Es52gBgol-tMRIOD6oK9cBo3rB0JQ,74
-datamarket-0.9.26.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-datamarket-0.9.26.dist-info/METADATA,sha256=dM-E47x3IuZiQRKlOI-vHsHFCRqc0ZmLB-0jyntnAeU,6543
-datamarket-0.9.26.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-datamarket-0.9.26.dist-info/RECORD,,
+datamarket-0.9.28.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+datamarket-0.9.28.dist-info/METADATA,sha256=5V61eLQ9HR_8Tl7J0sRKwR_mzKbJctIjvEebNtPtqug,6546
+datamarket-0.9.28.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+datamarket-0.9.28.dist-info/RECORD,,