tibiawikisql 6.1.0__tar.gz → 6.2.1__tar.gz

{tibiawikisql-6.1.0 → tibiawikisql-6.2.1}/CHANGELOG.rst

@@ -2,6 +2,22 @@
 Changelog
 =========
 
+.. 6.2.1
+
+6.2.1 (2024-06-27)
+==================
+- Fix database generation failing due to invalid date format.
+
+
+.. 6.2.0
+
+6.2.0 (2022-07-23)
+==================
+- Added `bosstiary_class` to creatures.
+- NPC Offers are read from the ItemPrices data list.
+- NPC Spell offers are read from the ItemPrices spell list.
+- Performance improvements.
+
 .. 6.1.0
 
 6.1.0 (2022-01-04)

{tibiawikisql-6.1.0/tibiawikisql.egg-info → tibiawikisql-6.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tibiawikisql
-Version: 6.1.0
+Version: 6.2.1
 Summary: Python script that generates a SQLite database from TibiaWiki articles
 Home-page: https://github.com/Galarzaa90/tibiawiki-sql
 Author: Galarzaa90

{tibiawikisql-6.1.0 → tibiawikisql-6.2.1}/requirements.txt

@@ -1,4 +1,5 @@
 requests
 click
 colorama
-mwparserfromhell
+mwparserfromhell
+lupa

{tibiawikisql-6.1.0 → tibiawikisql-6.2.1}/tibiawikisql/__init__.py

@@ -15,10 +15,10 @@
 """API that reads and parses information from `TibiaWiki <https://tibiawiki.fandom.com>`_."""
 
 __author__ = "Allan Galarza"
-__copyright__ = "Copyright 2021 Allan Galarza"
+__copyright__ = "Copyright 2024 Allan Galarza"
 
 __license__ = "Apache 2.0"
-__version__ = "6.1.0"
+__version__ = "6.2.1"
 
 from tibiawikisql import models
 from tibiawikisql.api import Article, Image, WikiClient, WikiEntry

{tibiawikisql-6.1.0 → tibiawikisql-6.2.1}/tibiawikisql/__main__.py

@@ -11,11 +11,12 @@
 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
-
+import collections
 import datetime
 import json
 import os
 import platform
+import re
 import sqlite3
 import time
 from typing import Optional, Type
@@ -24,6 +25,7 @@ import click
 import colorama
 import requests
 from colorama import Fore, Style
+from lupa import LuaRuntime
 
 from tibiawikisql import Image, WikiClient, __version__, models, schema
 from tibiawikisql.models import abc
@@ -34,6 +36,8 @@ DATABASE_FILE = "tibiawiki.db"
 
 colorama.init()
 
+lua = LuaRuntime()
+
 
 def progress_bar(iterable, label, length, **kwargs):
     return click.progressbar(iterable=iterable, length=length, label=label, fill_char="█", empty_char="░", width=10,
@@ -53,23 +57,24 @@ class Category:
 
     Class for internal use only, for easier autocompletion and maintenance."""
     def __init__(self, name: Optional[str], model: Type[abc.Row] = None, *, no_images=False, extension=".gif",
-                 include_deprecated=False, no_title=False):
+                 include_deprecated=False, no_title=False, generate_map=False):
         self.name = name
         self.model = model
         self.no_images = no_images
         self.extension = extension
         self.include_deprecated = include_deprecated
         self.no_title = no_title
+        self.generate_map = generate_map
 
 
 categories = {
     "achievements": Category("Achievements", models.Achievement, no_images=True),
-    "spells": Category("Spells", models.Spell),
-    "items": Category("Objects", models.Item),
-    "creatures": Category("Creatures", models.Creature),
+    "spells": Category("Spells", models.Spell, generate_map=True),
+    "items": Category("Objects", models.Item, generate_map=True),
+    "creatures": Category("Creatures", models.Creature, generate_map=True),
     "books": Category("Book Texts", models.Book, no_images=True),
     "keys": Category("Keys", models.Key, no_images=True),
-    "npcs": Category("NPCs", models.Npc),
+    "npcs": Category("NPCs", models.Npc, generate_map=True),
     "imbuements": Category("Imbuements", models.Imbuement, extension=".png"),
     "quests": Category("Quest Overview Pages", models.Quest, no_images=True),
     "house": Category("Player-Ownable Buildings", models.House, no_images=True),
@@ -111,6 +116,9 @@ def generate(skip_images, db_name, skip_deprecated):
         if not issubclass(model, abc.Parseable):
             continue
         titles = [a.title for a in data_store[key]]
+
+        if value.generate_map:
+            data_store[f"{key}_map"] = {}
         unparsed = []
         exec_time = time.perf_counter()
         generator = WikiClient.get_articles(titles)
@@ -120,6 +128,8 @@ def generate(skip_images, db_name, skip_deprecated):
                     entry = model.from_article(article)
                     if entry is not None:
                         entry.insert(conn)
+                        if value.generate_map:
+                            data_store[f"{key}_map"][entry.title.lower()] = entry.article_id
                     else:
                         unparsed.append(article.title)
             if unparsed:
@@ -131,7 +141,9 @@ def generate(skip_images, db_name, skip_deprecated):
     for position in rashid_positions:
         position.insert(conn)
 
-    generate_loot_statistics(conn)
+    generate_item_offers(conn, data_store)
+    generate_spell_offers(conn, data_store)
+    generate_loot_statistics(conn, data_store)
 
     if not skip_images:
         with conn:
@@ -154,45 +166,165 @@ def generate(skip_images, db_name, skip_deprecated):
     click.echo(f"Command finished in {dt:.2f} seconds.")
 
 
-def generate_loot_statistics(conn: sqlite3.Connection):
+link_pattern = re.compile(r"(?:(?P<price>\d+))?\s?\[\[([^\]|]+)")
+
+
+def generate_spell_offers(conn: sqlite3.Connection, data_store):
+    if "npcs_map" not in data_store or "spells_map" not in data_store:
+        return
+    start_time = time.perf_counter()
+    article = WikiClient.get_article("Module:ItemPrices/spelldata")
+    data = lua.execute(article.content)
+
+    spell_offers = []
+    not_found_store = collections.defaultdict(set)
+    spells = list(data.items())
+    with progress_bar(spells, "Fetching spell offers", len(spells)) as bar:
+        for name, table in bar:
+            spell_id = data_store["spells_map"].get(name.lower())
+            if spell_id is None:
+                not_found_store["spell"].add(name)
+                continue
+            spell_vocations = list((table["vocation"].values()))
+            for npc, vocation in table["sellers"].items():
+                npc_id = data_store["npcs_map"].get(npc.lower())
+                if npc_id is None:
+                    not_found_store["item"].add(npc)
+                    continue
+                if isinstance(vocation, bool):
+                    npc_vocations = spell_vocations
+                elif isinstance(vocation, str):
+                    npc_vocations = [vocation]
+                else:
+                    npc_vocations = list(vocation.values())
+                spell_offers.append((
+                    npc_id,
+                    spell_id,
+                    "Knight" in npc_vocations,
+                    "Sorcerer" in npc_vocations,
+                    "Druid" in npc_vocations,
+                    "Paladin" in npc_vocations,
+                ))
+        with conn:
+            conn.execute("DELETE FROM npc_spell")
+            conn.executemany("INSERT INTO npc_spell(npc_id, spell_id, knight, sorcerer, paladin, druid) VALUES(?, ?, ?, ?, ?, ?)", spell_offers)
+        if not_found_store["spell"]:
+            unknonw_spells = not_found_store["spell"]
+            click.echo(f"{Fore.RED}Could not parse offers for {len(unknonw_spells):,} spell.{Style.RESET_ALL}")
+            click.echo(f"\t-> {Fore.RED}{f'{Style.RESET_ALL},{Fore.RED}'.join(unknonw_spells)}{Style.RESET_ALL}")
+        if not_found_store["npc"]:
+            unknown_npcs = not_found_store["npc"]
+            click.echo(f"{Fore.RED}Could not parse offers of {len(unknown_npcs):,} npcs.{Style.RESET_ALL}")
+            click.echo(f"\t-> {Fore.RED}{f'{Style.RESET_ALL},{Fore.RED}'.join(unknown_npcs)}{Style.RESET_ALL}")
+
+
+def generate_item_offers(conn: sqlite3.Connection, data_store):
+    if "npcs_map" not in data_store or "items_map" not in data_store:
+        return
+    start_time = time.perf_counter()
+    article = WikiClient.get_article("Module:ItemPrices/data")
+    data = lua.execute(article.content)
+
+    sell_offers = []
+    buy_offers = []
+    not_found_store = collections.defaultdict(set)
+    npc_offers = list(data.items())
+    with progress_bar(npc_offers, "Fetching NPC offers", len(npc_offers)) as bar:
+        for name, table in bar:
+            npc_id = data_store["npcs_map"].get(name.lower())
+            if npc_id is None:
+                not_found_store["npc"].add(name)
+                continue
+            if "sells" in table:
+                process_offer_list(npc_id, table["sells"], sell_offers, data_store, not_found_store)
+            if "buys" in table:
+                process_offer_list(npc_id, table["buys"], buy_offers, data_store, not_found_store)
+        with conn:
+            conn.execute("DELETE FROM npc_offer_sell")
+            conn.execute("DELETE FROM npc_offer_buy")
+            conn.executemany("INSERT INTO npc_offer_sell(npc_id, value, item_id, currency_id) VALUES(?, ?, ?, ?)", sell_offers)
+            conn.executemany("INSERT INTO npc_offer_buy(npc_id, value, item_id, currency_id) VALUES(?, ?, ?, ?)", buy_offers)
+            dt = (time.perf_counter() - start_time)
+    total_offers = len(sell_offers) + len(buy_offers)
+    if not_found_store["npc"]:
+        unknonw_npcs = not_found_store["npc"]
+        click.echo(f"{Fore.RED}Could not parse offers for {len(unknonw_npcs):,} npcs.{Style.RESET_ALL}")
+        click.echo(f"\t-> {Fore.RED}{f'{Style.RESET_ALL},{Fore.RED}'.join(unknonw_npcs)}{Style.RESET_ALL}")
+    if not_found_store["item"]:
+        unknonw_items = not_found_store["item"]
+        click.echo(f"{Fore.RED}Could not parse offers for {len(unknonw_items):,} items.{Style.RESET_ALL}")
+        click.echo(f"\t-> {Fore.RED}{f'{Style.RESET_ALL},{Fore.RED}'.join(unknonw_items)}{Style.RESET_ALL}")
+    click.echo(f"{Fore.GREEN}\tSaved {total_offers:,} NPC offers in {dt:.2f} seconds.")
+
+
+def process_offer_list(npc_id, array, list_store, data_store, not_found):
+    for lua_item in array.values():
+        data = dict(lua_item.items())
+        price = data["price"]
+        currency = data.get("currency", "gold coin")
+        if not isinstance(price, int):
+            m = link_pattern.search(price)
+            price = int(m.group("price") or 1)
+            currency = m.group(2)
+        item_name = data["item"]
+        currency_name = currency
+        item_id = data_store["items_map"].get(item_name.lower())
+        currency_id = data_store["items_map"].get(currency_name.lower())
+        if currency_id is None:
+            not_found["item"].add(currency_name)
+            continue
+        if item_id is None:
+            not_found["item"].add(item_name)
+            continue
+        list_store.append((
+            npc_id,
+            price,
+            item_id,
+            currency_id
+        ))
+
+
+def generate_loot_statistics(conn: sqlite3.Connection, data_store):
     c = conn.cursor()
     try:
         results = conn.execute("SELECT title FROM creature")
         titles = [f"Loot Statistics:{t[0]}" for t in results]
         start_time = time.perf_counter()
+        unknown_items = set()
         with progress_bar(WikiClient.get_articles(titles), "Fetching loot statistics", len(titles),
                           item_show_func=article_show) as bar:
             for article in bar:
                 if article is None:
                     continue
                 creature_title = article.title.replace("Loot Statistics:", "")
-                c.execute("SELECT article_id from creature WHERE title = ?", (creature_title,))
-                result = c.fetchone()
-                if result is None:
+                creature_id = data_store["creatures_map"].get(creature_title.lower())
+                if creature_id is None:
                     # This could happen if a creature's article was deleted but its Loot Statistics weren't
                     continue
-                creature_id = result[0]
-                # Most loot statistics contain stats for older versions too, we onl care about the latest version.
+                # Most loot statistics contain stats for older versions too, we only care about the latest version.
                 kills, loot_stats = parse_loot_statistics(article.content)
                 loot_items = []
                 for entry in loot_stats:
-                    item = entry["item"]
-                    times = entry["times"]
-                    amount = entry.get("amount", 1)
-                    c.execute("SELECT article_id FROM item WHERE title LIKE ?", (item,))
-                    result = c.fetchone()
-                    if result is None:
-                        continue
-                    item_id = result[0]
-                    percentage = min(int(times) / kills * 100, 100)
-                    _min, _max = parse_min_max(amount)
-                    loot_items.append((creature_id, item_id, percentage, _min, _max))
-                    # We delete any duplicate record that was added from the creature's article's loot if it exists
-                    c.execute("DELETE FROM creature_drop WHERE creature_id = ? AND item_id = ?",
-                              (creature_id, item_id))
+                    if entry:
+                        item = entry["item"]
+                        times = entry["times"]
+                        amount = entry.get("amount", 1)
+                        item_id = data_store["items_map"].get(item.lower())
+                        if item_id is None:
+                            unknown_items.add(item)
+                            continue
+                        percentage = min(int(times) / kills * 100, 100)
+                        _min, _max = parse_min_max(amount)
+                        loot_items.append((creature_id, item_id, percentage, _min, _max))
+                        # We delete any duplicate record that was added from the creature's article's loot if it exists
+                        c.execute("DELETE FROM creature_drop WHERE creature_id = ? AND item_id = ?",
+                                (creature_id, item_id))
                 c.executemany("INSERT INTO creature_drop(creature_id, item_id, chance, min, max) VALUES(?,?,?,?,?)",
                               loot_items)
         dt = (time.perf_counter() - start_time)
+        if unknown_items:
+            click.echo(f"{Fore.RED}Could not find {len(unknown_items):,} items.{Style.RESET_ALL}")
+            click.echo(f"\t-> {Fore.RED}{f'{Style.RESET_ALL},{Fore.RED}'.join(unknown_items)}{Style.RESET_ALL}")
         click.echo(f"{Fore.GREEN}\tParsed loot statistics in {dt:.2f} seconds.{Style.RESET_ALL}")
     finally:
         conn.commit()

{tibiawikisql-6.1.0 → tibiawikisql-6.2.1}/tibiawikisql/models/creature.py

@@ -261,6 +261,8 @@ class Creature(abc.Row, abc.Parseable, table=schema.Creature):
         The creature's bestiary level, from 'Trivial' to 'Hard'
     bestiary_occurrence: :class:`str`
         The creature's bestiary occurrence, from 'Common'  to 'Very Rare'.
+    bosstiary_class: :class:`str`
+        The creature's bosstiary class, if applicable.
     hitpoints: :class:`int`
         The creature's hitpoints, may be `None` if unknown.
     experience: :class:`int`
@@ -336,6 +338,7 @@ class Creature(abc.Row, abc.Parseable, table=schema.Creature):
         "creatureclass": ("creature_class", str.strip),
         "bestiaryclass": ("bestiary_class", str.strip),
         "bestiarylevel": ("bestiary_level", str.strip),
+        "bosstiaryclass": ("bosstiary_class", str.strip),
         "occurrence": ("bestiary_occurrence", str.strip),
         "primarytype": ("creature_type", str.strip),
         "secondarytype": ("type_secondary", str.strip),
@@ -384,6 +387,7 @@ class Creature(abc.Row, abc.Parseable, table=schema.Creature):
         "bestiary_level",
         "bestiary_class",
         "bestiary_occurrence",
+        "bosstiary_class",
         "hitpoints",
         "experience",
         "armor",

{tibiawikisql-6.1.0 → tibiawikisql-6.2.1}/tibiawikisql/models/imbuement.py

@@ -123,6 +123,8 @@ class Imbuement(abc.Row, abc.Parseable, table=schema.Imbuement):
         The tier of the imbuement.
     type: :class:`str`
         The imbuement's type.
+    category: :class:`str`
+        The imbuement's category.
     effect: :class:`str`
         The effect given by the imbuement.
     slots: :class:`str`
@@ -141,6 +143,7 @@ class Imbuement(abc.Row, abc.Parseable, table=schema.Imbuement):
         "name": ("name", str.strip),
         "prefix": ("tier", str.strip),
         "type": ("type", str.strip),
+        "category": ("category", str.strip),
         "effect": ("effect", parse_effect),
         "implemented": ("version", str.strip),
         "slots": ("slots", parse_slots),
@@ -155,6 +158,7 @@ class Imbuement(abc.Row, abc.Parseable, table=schema.Imbuement):
         "name",
         "tier",
         "type",
+        "category",
         "effect",
         "slots",
         "version",

{tibiawikisql-6.1.0 → tibiawikisql-6.2.1}/tibiawikisql/models/item.py

@@ -282,15 +282,18 @@ class Item(abc.Row, abc.Parseable, table=schema.Item):
             if attribute in item._raw_attributes and item._raw_attributes[attribute]:
                 item.attributes.append(ItemAttribute(item_id=item.article_id, name=name,
                                                      value=item._raw_attributes[attribute]))
-        item._parse_attributes()
-        item._parse_resists()
-        vocations = item._raw_attributes.get('vocrequired')
-        if vocations and "none" not in vocations.lower():
-            vocation = vocations.replace('and', '+').replace(',', '+').replace(' ', '')
-            item.attributes.append(ItemAttribute(item_id=item.article_id, name="vocation", value=vocation))
-        item._parse_sounds()
-        item._parse_store_value()
-        return item
+        try:
+            item._parse_attributes()
+            item._parse_resists()
+            vocations = item._raw_attributes.get('vocrequired')
+            if vocations and "none" not in vocations.lower():
+                vocation = vocations.replace('and', '+').replace(',', '+').replace(' ', '')
+                item.attributes.append(ItemAttribute(item_id=item.article_id, name="vocation", value=vocation))
+            item._parse_sounds()
+            item._parse_store_value()
+            return item
+        except Exception:
+            return None
 
     def _parse_attributes(self):
         if "attrib" not in self._raw_attributes:
@@ -600,7 +603,7 @@ class ItemAttribute(abc.Row, table=schema.ItemAttribute):
         "unshootable": "unshootable",
         "blocks_path": "blockspath",
         "walkable": "walkable",
-        "walking_speed": "walkingspeed",
+        "tile_friction": "walkingspeed",
         "map_color": "mapcolor",
         "upgrade_classification": "upgradeclass"
     }

{tibiawikisql-6.1.0 → tibiawikisql-6.2.1}/tibiawikisql/models/npc.py

@@ -164,6 +164,8 @@ class Npc(abc.Row, abc.Parseable, table=schema.Npc):
         The jobs of the NPC.
     location: :class:`str`
         The location of the NPC.
+    subarea: :class:`str`
+        A finer location of the NPC.
     city: :class:`str`
         The nearest city to where the NPC is located.
     x: :class:`int`
@@ -198,6 +200,7 @@ class Npc(abc.Row, abc.Parseable, table=schema.Npc):
         "jobs",
         "location",
         "city",
+        "subarea",
         "x",
         "y",
         "z",
@@ -219,6 +222,7 @@ class Npc(abc.Row, abc.Parseable, table=schema.Npc):
         "location": ("location", clean_links),
         "gender": ("gender", str.strip),
         "city": ("city", str.strip),
+        "subarea": ("subarea", str.strip),
         "posx": ("x", convert_tibiawiki_position),
         "posy": ("y", convert_tibiawiki_position),
         "posz": ("z", int),
@@ -359,10 +363,10 @@ class Npc(abc.Row, abc.Parseable, table=schema.Npc):
 
     def insert(self, c):
         super().insert(c)
-        for offer in getattr(self, "buy_offers", []):
-            offer.insert(c)
-        for offer in getattr(self, "sell_offers", []):
-            offer.insert(c)
+        # for offer in getattr(self, "buy_offers", []):
+        #     offer.insert(c)
+        # for offer in getattr(self, "sell_offers", []):
+        #     offer.insert(c)
         for spell in getattr(self, "teaches", []):
             spell.insert(c)
         for destination in getattr(self, "destinations", []):

{tibiawikisql-6.1.0 → tibiawikisql-6.2.1}/tibiawikisql/schema.py

@@ -63,6 +63,7 @@ class Creature(Table):
     bestiary_class = Column(Text, index=True)
     bestiary_level = Column(Text, index=True)
     bestiary_occurrence = Column(Text, index=True)
+    bosstiary_class = Column(Text, index=True)
     runs_at = Column(Integer)
     summon_cost = Column(Integer)
     convince_cost = Column(Integer)
@@ -220,6 +221,7 @@ class Imbuement(Table):
     name = Column(Text, unique=True, index=True)
     tier = Column(Text, nullable=False)
     type = Column(Text, nullable=False)
+    category = Column(Text, nullable=False)
     effect = Column(Text)
     slots = Column(Text)
     version = Column(Text, index=True)
@@ -290,6 +292,7 @@ class Npc(Table):
     name = Column(Text, no_case=True, index=True)
     gender = Column(Text, index=True)
     city = Column(Text, index=True)
+    subarea = Column(Text)
     location = Column(Text)
     version = Column(Text, index=True)
     x = Column(Integer)

{tibiawikisql-6.1.0 → tibiawikisql-6.2.1}/tibiawikisql/utils.py

@@ -188,6 +188,7 @@ def parse_date(value):
 
     - June 28, 2019
     - Aug 21, 2014
+    - May 14, 2024 17:45
 
     Parameters
     ----------
@@ -200,11 +201,22 @@ def parse_date(value):
         The date represented by the string.
     """
     value = value.strip()
-    try:
-        dt = datetime.datetime.strptime(value, "%B %d, %Y")
-    except ValueError:
-        dt = datetime.datetime.strptime(value, "%b %d, %Y")
-    return dt.date().isoformat()
+    date_formats = [
+        "%B %d, %Y",
+        "%b %d, %Y",
+        "%Y",
+        "%B %d, %Y %H:%M",
+        "%b %d, %Y %H:%M",
+        "%Y %H:%M",
+    ]
+    for date_format in date_formats:
+        try:
+            dt = datetime.datetime.strptime(value, date_format)
+            return dt.date().isoformat()
+        except ValueError:
+            continue
+
+    raise ValueError(f"Date format for value '{value}' not recognized")
 
 
 def parse_float(value, default=0.0):
@@ -297,7 +309,10 @@ def _parse_loot_entry(entry):
             key = subarg[0].strip()
             value = subarg[1].strip()
         entry[key] = value
-    return entry
+    if "item" in entry:
+        return entry
+    else:
+        return None
 
 
 def parse_min_max(value):

{tibiawikisql-6.1.0 → tibiawikisql-6.2.1/tibiawikisql.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tibiawikisql
-Version: 6.1.0
+Version: 6.2.1
 Summary: Python script that generates a SQLite database from TibiaWiki articles
 Home-page: https://github.com/Galarzaa90/tibiawiki-sql
 Author: Galarzaa90

{tibiawikisql-6.1.0 → tibiawikisql-6.2.1}/tibiawikisql.egg-info/requires.txt

@@ -2,6 +2,7 @@ requests
 click
 colorama
 mwparserfromhell
+lupa
 
 [docs]
 Sphinx