wt-resource-tool 0.1.0__tar.gz

wt_resource_tool-0.1.0/PKG-INFO

@@ -0,0 +1,68 @@
+Metadata-Version: 2.3
+Name: wt-resource-tool
+Version: 0.1.0
+Summary: 
+Author: axiangcoding
+Author-email: axiangcoding@gmail.com
+Requires-Python: >=3.12,<4.0
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Dist: aiofiles (>=24.1.0,<25.0.0)
+Requires-Dist: httpx (>=0.28.1,<0.29.0)
+Requires-Dist: loguru (>=0.7.3,<0.8.0)
+Requires-Dist: pydantic (>=2.10.6,<3.0.0)
+Requires-Dist: pydantic-settings (>=2.8.1,<3.0.0)
+Requires-Dist: python-dotenv (>=1.1.0,<2.0.0)
+Description-Content-Type: text/markdown
+
+# WT Resource Tool
+
+> [!warning]
+> This project is still in development, PRs are welcome!
+
+As a War Thunder community developer, interested in unpacking data but turned off by the complexity of the results? Try the WT Resource Tool!
+
+We did the hard work for you, parsing the data and making it human-readable, so you can focus on what you do best: creating awesome projects for the War Thunder community.
+
+## Not official
+
+This project is not affiliated with Gaijin Entertainment.
+
+> However, we did get in touch with Gaijin, maybe some interesting things will happen in the future. WHO KNOWS?
+
+## Features
+
+- **Easy to use**: Install `wt-resource-tool` via pip and start using it right away.
+- **Human-readable**: All data is parsed and presented in a human-readable format.
+- **Accessible**: Don't need to download entire data to your local environment everytime.
+- **Up-to-date**: We keep the data up-to-date with the [War-Thunder-Datamine](https://github.com/gszabi99/War-Thunder-Datamine) repository. Thanks to the contributors!
+
+## Data
+
+- **Player titles**: All player titles in the game, including their localization.
+- **Player medals**: All player medals in the game, including their localization and image at different sizes.
+- **Vehicle data**: All vehicle data in the game
+
+> [!note]
+> More data will be added soon!
+
+## Usage
+
+### Use python package `wt-resource-tool`
+
+Install the package using pip:
+
+> [!warning]
+> This package is not available on PyPi yet. You can install it from source.
+
+Then you can use it in your python code. We provide some examples to get you started, take a look at the [playground](playground) folder.
+
+### Use static data in json format
+
+You can find data we parsed ahead in json format under `/static` folder.
+
+## Credits
+
+Special thanks to [War-Thunder-Datamine](https://github.com/gszabi99/War-Thunder-Datamine) for providing the original data!
+

wt_resource_tool-0.1.0/README.md

@@ -0,0 +1,49 @@
+# WT Resource Tool
+
+> [!warning]
+> This project is still in development, PRs are welcome!
+
+As a War Thunder community developer, interested in unpacking data but turned off by the complexity of the results? Try the WT Resource Tool!
+
+We did the hard work for you, parsing the data and making it human-readable, so you can focus on what you do best: creating awesome projects for the War Thunder community.
+
+## Not official
+
+This project is not affiliated with Gaijin Entertainment.
+
+> However, we did get in touch with Gaijin, maybe some interesting things will happen in the future. WHO KNOWS?
+
+## Features
+
+- **Easy to use**: Install `wt-resource-tool` via pip and start using it right away.
+- **Human-readable**: All data is parsed and presented in a human-readable format.
+- **Accessible**: Don't need to download entire data to your local environment everytime.
+- **Up-to-date**: We keep the data up-to-date with the [War-Thunder-Datamine](https://github.com/gszabi99/War-Thunder-Datamine) repository. Thanks to the contributors!
+
+## Data
+
+- **Player titles**: All player titles in the game, including their localization.
+- **Player medals**: All player medals in the game, including their localization and image at different sizes.
+- **Vehicle data**: All vehicle data in the game
+
+> [!note]
+> More data will be added soon!
+
+## Usage
+
+### Use python package `wt-resource-tool`
+
+Install the package using pip:
+
+> [!warning]
+> This package is not available on PyPi yet. You can install it from source.
+
+Then you can use it in your python code. We provide some examples to get you started, take a look at the [playground](playground) folder.
+
+### Use static data in json format
+
+You can find data we parsed ahead in json format under `/static` folder.
+
+## Credits
+
+Special thanks to [War-Thunder-Datamine](https://github.com/gszabi99/War-Thunder-Datamine) for providing the original data!

wt_resource_tool-0.1.0/pyproject.toml

@@ -0,0 +1,34 @@
+[project]
+name = "wt-resource-tool"
+version = "0.1.0"
+description = ""
+authors = [{ name = "axiangcoding", email = "axiangcoding@gmail.com" }]
+readme = "README.md"
+requires-python = ">=3.12,<4.0"
+dependencies = [
+    "pydantic (>=2.10.6,<3.0.0)",
+    "pydantic-settings (>=2.8.1,<3.0.0)",
+    "httpx (>=0.28.1,<0.29.0)",
+    "python-dotenv (>=1.1.0,<2.0.0)",
+    "aiofiles (>=24.1.0,<25.0.0)",
+    "loguru (>=0.7.3,<0.8.0)",
+]
+
+[tool.poetry]
+packages = [{ include = "wt_resource_tool", from = "src" }]
+
+
+[tool.poetry.group.dev.dependencies]
+ruff = "^0.9.10"
+pytest = "^8.3.5"
+ipykernel = "^6.29.5"
+mypy = "^1.15.0"
+
+[build-system]
+requires = ["poetry-core>=2.0.0,<3.0.0"]
+build-backend = "poetry.core.masonry.api"
+
+
+[tool.ruff]
+line-length = 120
+lint.select = ["I", "N", "UP", "B", "A", "ASYNC"]

wt_resource_tool-0.1.0/src/wt_resource_tool/__init__.py

@@ -0,0 +1,3 @@
+from wt_resource_tool._client import WTResourceTool
+
+__all__ = ["WTResourceTool"]

wt_resource_tool-0.1.0/src/wt_resource_tool/_client.py

@@ -0,0 +1,165 @@
+import asyncio
+import time
+from tracemalloc import start
+from typing import Literal
+
+import httpx
+from loguru import logger
+from pydantic import BaseModel, Field
+
+from wt_resource_tool.parser import player_medal_parser, player_title_parser, vehicle_data_parser
+from wt_resource_tool.schema._wt_schema import (
+    PlayerMedalDesc,
+    PlayerMedalStorage,
+    PlayerTitleDesc,
+    PlayerTitleStorage,
+    Vehicle,
+    VehicleStorage,
+)
+
+type DataType = Literal["player_title", "player_medal", "vehicle"]
+type DataSource = Literal["github", "github-jsdelivr"] | str
+
+
+class WTResourceTool(BaseModel):
+    """
+    A tool to parse and get data about War Thunder.
+
+    """
+
+    title_storage: PlayerTitleStorage | None = Field(default=None)
+    medal_storage: PlayerMedalStorage | None = Field(default=None)
+    vehicle_storage: VehicleStorage | None = Field(default=None)
+
+    async def load_parsed_data(
+        self,
+        data_types: list[DataType],
+        game_version: str = "latest",
+        source: DataSource = "github-jsdelivr",
+    ):
+        """
+        Load pre-parsed data from remote.
+        The data is stored in the static folder of the repository.
+
+        Args:
+            data_types (list[DataType]): The data types to load.
+            game_version (str): The game version to load. Default is "latest".
+            source (DataSource): The source of the data. Default is "github-jsdelivr". It can be "github", "github-jsdelivr" or a custom url.
+        """
+        start_time = time.time()
+        if source == "github":
+            resource_url_prefix = (
+                "https://raw.githubusercontent.com/axiangcoding/wt-resource-tool/refs/heads/main/static"
+            )
+        elif source == "github-jsdelivr":
+            resource_url_prefix = "https://cdn.jsdelivr.net/gh/axiangcoding/wt-resource-tool/static"
+        else:
+            if not source.startswith("https:// "):
+                raise ValueError("Custom source must be a valid safe url")
+            resource_url_prefix = source
+        game_version_folder_str = game_version.replace(".", "_")
+
+        if "player_title" in data_types:
+            resource_url = f"{resource_url_prefix}/{game_version_folder_str}/player_title.json"
+            logger.debug("Loading player title data from {}", resource_url)
+            title_data = await self.__get_data_from_remote(resource_url)
+            self.title_storage = PlayerTitleStorage.model_validate_json(title_data)
+
+        if "player_medal" in data_types:
+            resource_url = f"{resource_url_prefix}/{game_version_folder_str}/player_medal.json"
+            logger.debug("Loading player medal data from {}", resource_url)
+            medal_data = await self.__get_data_from_remote(resource_url)
+
+            self.medal_storage = PlayerMedalStorage.model_validate_json(medal_data)
+
+        if "vehicle" in data_types:
+            resource_url = f"{resource_url_prefix}/{game_version_folder_str}/vehicle.json"
+            logger.debug("Loading vehicle data from {}", resource_url)
+            vehicle_data = await self.__get_data_from_remote(resource_url)
+            self.vehicle_storage = VehicleStorage.model_validate_json(vehicle_data)
+
+        end_time = time.time()
+        logger.debug(
+            "Loaded data in {} seconds",
+            round(end_time - start_time, 2),
+        )
+
+    async def parse_and_load_data(
+        self,
+        data_types: list[DataType],
+        local_repo_path: str,
+        git_pull_when_empty: bool = False,
+    ):
+        """
+        Parse and load data from local repo.
+
+        This action may take a long time if repo not exist. Because it needs to clone the repo first.
+
+        Args:
+            data_types (list[DataType]): The data types to load.
+            local_repo_path (str): The local repo path.
+            git_pull_when_empty (bool): Whether to pull the repo when it is empty. Default is False.
+        """
+        # TODO check if the repo is empty and pull it if it is empty
+        start_time = time.time()
+        if "player_title" in data_types:
+            logger.debug("Parsing player title data from {}", local_repo_path)
+            ms = await asyncio.to_thread(lambda: player_medal_parser.parse_player_medal(local_repo_path))
+            self.medal_storage = ms
+        if "player_medal" in data_types:
+            logger.debug("Parsing player medal data from {}", local_repo_path)
+            ts = await asyncio.to_thread(lambda: player_title_parser.parse_player_title(local_repo_path))
+            self.title_storage = ts
+        if "vehicle" in data_types:
+            logger.debug("Parsing vehicle data from {}", local_repo_path)
+            vs = await asyncio.to_thread(lambda: vehicle_data_parser.parse_vehicle_data(local_repo_path))
+            self.vehicle_storage = vs
+        end_time = time.time()
+        logger.debug(
+            "Parsed data in {} seconds",
+            round(end_time - start_time, 2),
+        )
+
+    async def get_loaded_data_version(self) -> dict[DataType, str | None]:
+        return {
+            "player_title": self.title_storage.game_version if self.title_storage else None,
+            "player_medal": self.medal_storage.game_version if self.medal_storage else None,
+            "vehicle": self.vehicle_storage.game_version if self.vehicle_storage else None,
+        }
+
+    async def get_title(self, title_id: str) -> PlayerTitleDesc:
+        """
+        Get title data by id.
+
+        """
+        if self.title_storage is None:
+            raise ValueError("No data loaded")
+        return self.title_storage.titles_map[title_id]
+
+    async def get_medal(self, medal_id: str) -> PlayerMedalDesc:
+        """
+        Get medal data by id.
+
+        """
+        if self.medal_storage is None:
+            raise ValueError("No data loaded")
+        return self.medal_storage.medals_map[medal_id]
+
+    async def get_vehicle(self, vehicle_id: str) -> Vehicle:
+        """
+        Get vehicle data by id.
+
+        """
+        if self.vehicle_storage is None:
+            raise ValueError("No data loaded")
+        return self.vehicle_storage.vehicles_map[vehicle_id]
+
+    async def __get_data_from_remote(
+        self,
+        resource_url: str,
+    ) -> str:
+        async with httpx.AsyncClient(timeout=60) as client:
+            resp = await client.get(resource_url)
+            resp.raise_for_status()
+            storage_text = resp.text
+        return storage_text

wt_resource_tool-0.1.0/src/wt_resource_tool/parser/player_medal_parser.py

@@ -0,0 +1,44 @@
+import csv
+from os import path
+
+from wt_resource_tool.schema._wt_schema import PlayerMedalDesc, PlayerMedalStorage
+
+
+def _get_dt_from_csv(data: csv.DictReader) -> list[PlayerMedalDesc]:
+    titles: list[PlayerMedalDesc] = []
+    for row in data:
+        row1 = row["<ID|readonly|noverify>"]
+
+        if row1.endswith("/name"):
+            mid = row["<ID|readonly|noverify>"].replace("/name", "")
+
+            td = PlayerMedalDesc(
+                id=mid,
+                english=row["<English>"],
+                french=row["<French>"],
+                italian=row["<Italian>"],
+                german=row["<German>"],
+                spanish=row["<Spanish>"],
+                japanese=row["<Japanese>"].replace("\\t", ""),
+                chinese=row["<Chinese>"].replace("\\t", ""),
+                russian=row["<Russian>"],
+                comments=row["<Comments>"],
+                max_chars=row["<max_chars>"],
+            )
+            titles.append(td)
+    return titles
+
+
+def parse_player_medal(repo_path: str) -> PlayerMedalStorage:
+    all_medals: list[PlayerMedalDesc] = []
+
+    with open(path.join(repo_path, "lang.vromfs.bin_u/lang/unlocks_medals.csv"), encoding="utf-8") as f:
+        data = csv.DictReader(f, delimiter=";")
+        all_medals.extend(_get_dt_from_csv(data))
+
+    medals_map = {}
+    for title in all_medals:
+        medals_map[title.id] = title
+
+    game_version = open(path.join(repo_path, "version"), encoding="utf-8").read()
+    return PlayerMedalStorage(medals_map=medals_map, game_version=game_version.strip())

wt_resource_tool-0.1.0/src/wt_resource_tool/parser/player_title_parser.py

@@ -0,0 +1,51 @@
+import csv
+from os import path
+
+from wt_resource_tool.schema._wt_schema import PlayerTitleDesc, PlayerTitleStorage
+
+
+def _get_dt_from_csv(data: csv.DictReader) -> list[PlayerTitleDesc]:
+    titles: list[PlayerTitleDesc] = []
+    for row in data:
+        row1 = row["<ID|readonly|noverify>"]
+
+        if row1.startswith("title/") and (not row1.endswith("/desc")):
+            mid = row["<ID|readonly|noverify>"].replace("title/", "")
+
+            td = PlayerTitleDesc(
+                id=mid,
+                english=row["<English>"],
+                french=row["<French>"],
+                italian=row["<Italian>"],
+                german=row["<German>"],
+                spanish=row["<Spanish>"],
+                japanese=row["<Japanese>"].replace("\\t", ""),
+                chinese=row["<Chinese>"].replace("\\t", ""),
+                russian=row["<Russian>"],
+                comments=row["<Comments>"],
+                max_chars=row["<max_chars>"],
+            )
+            titles.append(td)
+    return titles
+
+
+def parse_player_title(repo_path: str) -> PlayerTitleStorage:
+    all_titles: list[PlayerTitleDesc] = []
+    with open(path.join(repo_path, "regional.vromfs.bin_u/lang/regional_titles.csv"), encoding="utf-8") as f:
+        data = csv.DictReader(f, delimiter=";")
+        all_titles.extend(_get_dt_from_csv(data))
+
+    with open(path.join(repo_path, "lang.vromfs.bin_u/lang/unlocks_achievements.csv"), encoding="utf-8") as f:
+        data = csv.DictReader(f, delimiter=";")
+        all_titles.extend(_get_dt_from_csv(data))
+
+    with open(path.join(repo_path, "regional.vromfs.bin_u/lang/tournaments.csv"), encoding="utf-8") as f:
+        data = csv.DictReader(f, delimiter=";")
+        all_titles.extend(_get_dt_from_csv(data))
+
+    title_map = {}
+    for title in all_titles:
+        title_map[title.id] = title
+
+    game_version = open(path.join(repo_path, "version"), encoding="utf-8").read()
+    return PlayerTitleStorage(titles_map=title_map, game_version=game_version.strip())

wt_resource_tool-0.1.0/src/wt_resource_tool/parser/vehicle_data_parser.py

@@ -0,0 +1,77 @@
+import json
+from os import path
+
+from loguru import logger
+
+from wt_resource_tool.schema._wt_schema import Vehicle, VehicleStorage
+
+
+def parse_vehicle_data(repo_path: str) -> VehicleStorage:
+    game_version = open(path.join(repo_path, "version"), encoding="utf-8").read()
+
+    wpcost_path = path.join(repo_path, "char.vromfs.bin_u", "config", "wpcost.blkx")
+    with open(wpcost_path, encoding="utf-8") as f:
+        wpcost = f.read()
+
+    data: dict = json.loads(wpcost)
+    convert_key_map = {
+        "rank": "rank",
+        "economicRankArcade": "economic_rank_arcade",
+        "economicRankHistorical": "economic_rank_historical",
+        "economicRankSimulation": "economic_rank_simulation",
+        "country": "country",
+        "unitClass": "unit_class",
+        "spawnType": "spawn_type",
+        "value": "value",
+        "reqExp": "req_exp",
+        "trainCost": "train_cost",
+        "train2Cost": "train2_cost",
+        "train3Cost_gold": "train3_cost_gold",
+        "train3Cost_exp": "train3_cost_exp",
+        "repairCostArcade": "repair_cost_arcade",
+        "repairCostHistorical": "repair_cost_historical",
+        "repairCostSimulation": "repair_cost_simulation",
+        "repairCostPerMinArcade": "repair_cost_per_min_arcade",
+        "repairCostPerMinHistorical": "repair_cost_per_min_historical",
+        "repairCostPerMinSimulation": "repair_cost_per_min_simulation",
+        "repairCostFullUpgradedArcade": "repair_cost_full_upgraded_arcade",
+        "repairCostFullUpgradedHistorical": "repair_cost_full_upgraded_historical",
+        "repairCostFullUpgradedSimulation": "repair_cost_full_upgraded_simulation",
+        "rewardMulArcade": "reward_mul_arcade",
+        "rewardMulHistorical": "reward_mul_historical",
+        "rewardMulSimulation": "reward_mul_simulation",
+        "expMul": "exp_mul",
+        "reqAir": "req_air",
+        "reloadTime_cannon": "reload_time_cannon",
+        "crewTotalCount": "crew_total_count",
+        "primaryWeaponAutoLoader": "primary_weapon_auto_loader",
+        "costGold": "cost_gold",
+        "turretSpeed": "turret_speed",
+        "gift": "gift",
+        "researchType": "research_type",
+        "economicRankTankHistorical": "economic_rank_tank_historical",
+        "economicRankTankSimulation": "economic_rank_tank_simulation",
+    }
+
+    vehicles = {}
+    for key in data.keys():
+        if not isinstance(data[key], dict):
+            logger.warning("key {} is not a dict, skip.", key)
+            continue
+        try:
+            v_data: dict = data[key]
+            n_data = {
+                "id": key,
+            }
+
+            for k, v in v_data.items():
+                if k in convert_key_map:
+                    n_data[convert_key_map[k]] = v
+                else:
+                    n_data[k] = v
+
+            vehicles[key] = Vehicle.model_validate(n_data)
+        except Exception as e:
+            logger.warning("error when parsing vehicle id: {}, skip", key)
+            raise e
+    return VehicleStorage(vehicles_map=vehicles, game_version=game_version.strip())

wt_resource_tool-0.1.0/src/wt_resource_tool/schema/_wt_schema.py

@@ -0,0 +1,168 @@
+from typing import Literal
+
+from pydantic import BaseModel, Field
+
+
+class PlayerTitleDesc(BaseModel):
+    id: str
+    english: str
+    french: str
+    italian: str
+    german: str
+    spanish: str
+    russian: str
+    # polish: str
+    # czech: str
+    # turkish: str
+    chinese: str
+    japanese: str
+    # portuguese: str
+    # ukrainian: str
+    # serbian: str
+    # hungarian: str
+    # korean: str
+    # belarusian: str
+    # romanian: str
+    # vietnamese: str
+    # t_chinese: str
+    # h_chinese: str
+    comments: str
+    max_chars: str
+
+
+class PlayerTitleStorage(BaseModel):
+    titles_map: dict[str, PlayerTitleDesc]
+    game_version: str
+
+
+class PlayerMedalDesc(BaseModel):
+    id: str
+    english: str
+    french: str
+    italian: str
+    german: str
+    spanish: str
+    russian: str
+    # polish: str
+    # czech: str
+    # turkish: str
+    chinese: str
+    japanese: str
+    # portuguese: str
+    # ukrainian: str
+    # serbian: str
+    # hungarian: str
+    # korean: str
+    # belarusian: str
+    # romanian: str
+    # vietnamese: str
+    # t_chinese: str
+    # h_chinese: str
+    comments: str
+    max_chars: str
+
+    def get_image_url(
+        self,
+        mode: Literal["normal", "big", "ribbon"] = "normal",
+    ) -> str:
+        prefix = (
+            "https://cdn.jsdelivr.net/gh/gszabi99/War-Thunder-Datamine@refs/heads/master/atlases.vromfs.bin_u/medals"
+        )
+        if mode == "normal":
+            return f"{prefix}/{self.id}.png"
+        elif mode == "big":
+            return f"{prefix}/{self.id}_big.png"
+        elif mode == "ribbon":
+            return f"{prefix}/{self.id}_ribbon.png"
+
+
+class PlayerMedalStorage(BaseModel):
+    medals_map: dict[str, PlayerMedalDesc]
+    game_version: str
+
+
+type Country = Literal[
+    "country_usa",
+    "country_germany",
+    "country_ussr",
+    "country_britain",
+    "country_japan",
+    "country_france",
+    "country_italy",
+    "country_china",
+    "country_sweden",
+    "country_israel",
+]
+
+
+class Vehicle(BaseModel):
+    id: str
+    rank: int
+    economic_rank_arcade: int
+    economic_rank_historical: int
+    economic_rank_simulation: int
+
+    economic_rank_tank_historical: int | None = Field(default=None)
+    economic_rank_tank_simulation: int | None = Field(default=None)
+
+    country: Country
+    unit_class: str
+    spawn_type: str | None = Field(
+        default=None,
+    )
+    value: int
+    req_exp: int | None = Field(
+        default=None,
+    )
+    train_cost: int
+    train2_cost: int
+    train3_cost_gold: int
+    train3_cost_exp: int
+    repair_cost_arcade: int
+    repair_cost_historical: int
+    repair_cost_simulation: int
+    repair_cost_per_min_arcade: int
+    repair_cost_per_min_historical: int
+    repair_cost_per_min_simulation: int
+    repair_cost_full_upgraded_arcade: int | None = Field(
+        default=None,
+    )
+    repair_cost_full_upgraded_historical: int | None = Field(
+        default=None,
+    )
+    repair_cost_full_upgraded_simulation: int | None = Field(
+        default=None,
+    )
+    reward_mul_arcade: float = Field(
+        default=0,
+    )
+    reward_mul_historical: float
+    reward_mul_simulation: float
+    exp_mul: float
+    req_air: str | None = Field(
+        default=None,
+    )
+    reload_time_cannon: float | None = Field(
+        default=None,
+    )
+    crew_total_count: int | None = Field(default=None)
+
+    primary_weapon_auto_loader: bool | None = Field(default=None)
+
+    cost_gold: int | None = Field(default=None)
+
+    turret_speed: list[float] | None = Field(default=None)
+
+    gift: str | None = Field(default=None)
+
+    research_type: str | None = Field(default=None)
+
+    def get_icon_url(
+        self,
+    ) -> str:
+        return f"https://cdn.jsdelivr.net/gh/gszabi99/War-Thunder-Datamine@refs/heads/master/atlases.vromfs.bin_u/units/{self.id}.png"
+
+
+class VehicleStorage(BaseModel):
+    vehicles_map: dict[str, Vehicle]
+    game_version: str