ZedScripts 1.0.1__tar.gz → 1.2.0__tar.gz

{zedscripts-1.0.1/src/ZedScripts.egg-info → zedscripts-1.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ZedScripts
-Version: 1.0.1
+Version: 1.2.0
 Author: SimKDT
 License: MIT License
         
@@ -36,6 +36,10 @@ Provides-Extra: build
 Requires-Dist: pyinstaller; extra == "build"
 Provides-Extra: debug
 Requires-Dist: debugpy; extra == "debug"
+Provides-Extra: test
+Requires-Dist: pytest; extra == "test"
+Provides-Extra: pygments
+Requires-Dist: pygments; extra == "pygments"
 Dynamic: license-file
 
 # ZedScripts LSP
@@ -211,6 +215,10 @@ Each diagnostic by default have a specific set of modifiers that can be overridd
 }
 ```
 
+### Server
+A list of server-related configurations for the LSP. The following parameters are available:
+- `logging_level` (optional, default: `INFO`): specifies the logging level for the server. This option is only available in the global configuration and accepts values such as `NOTSET`, `DEBUG`, `INFO`, `WARNING`, `ERROR`, and `CRITICAL`.
+
 ## IDE configuration
 Based on the IDE you use, you may need to install the ZedScripts LSP application for your OS in the [releases](https://github.com/PZ-Wiki-Modding/ZedScripts-LSP/releases) (Linux, Windows).
 
@@ -238,5 +246,24 @@ autocmd User LspSetup call LspAddServer(lspServers)
 > [!NOTE]
 > You can see [this video](https://www.youtube.com/watch?v=-5lb_jLQmKc) which explains how to set up VIM, and notably setup a LSP server.
 
+## Pygments
+
+The ZedScripts LSP provides a Pygments lexer for syntax highlighting of ZedScripts code. It is available in the `ZedScripts.structure.pygments_lexer` module. You can notably use the command-line wrapper to highlight any file's content as ZedScripts:
+
+```bash
+uv pip install ZedScripts[pygments]
+highlight-zedscripts path/to/file.txt
+```
+
+![Screenshot of the output of highlight-zedscripts command on an example script file](assets/pygments_implementation.png)
+
+To use the lexer in Sphinx docs, you can add the following to your `conf.py`:
+
+```python
+from ZedScripts.structure.pygments_lexer import ZedScriptsLexer
+from sphinx.highlighting import lexers
+lexers['zedscripts'] = ZedScriptsLexer()
+```
+
 ## License and contribution
 This project is licensed under a [MIT license](LICENSE). Before contributing to the project, please see [CONTRIBUTING](CONTRIBUTING.md).

{zedscripts-1.0.1 → zedscripts-1.2.0}/README.md

@@ -171,6 +171,10 @@ Each diagnostic by default have a specific set of modifiers that can be overridd
 }
 ```
 
+### Server
+A list of server-related configurations for the LSP. The following parameters are available:
+- `logging_level` (optional, default: `INFO`): specifies the logging level for the server. This option is only available in the global configuration and accepts values such as `NOTSET`, `DEBUG`, `INFO`, `WARNING`, `ERROR`, and `CRITICAL`.
+
 ## IDE configuration
 Based on the IDE you use, you may need to install the ZedScripts LSP application for your OS in the [releases](https://github.com/PZ-Wiki-Modding/ZedScripts-LSP/releases) (Linux, Windows).
 
@@ -198,5 +202,24 @@ autocmd User LspSetup call LspAddServer(lspServers)
 > [!NOTE]
 > You can see [this video](https://www.youtube.com/watch?v=-5lb_jLQmKc) which explains how to set up VIM, and notably setup a LSP server.
 
+## Pygments
+
+The ZedScripts LSP provides a Pygments lexer for syntax highlighting of ZedScripts code. It is available in the `ZedScripts.structure.pygments_lexer` module. You can notably use the command-line wrapper to highlight any file's content as ZedScripts:
+
+```bash
+uv pip install ZedScripts[pygments]
+highlight-zedscripts path/to/file.txt
+```
+
+![Screenshot of the output of highlight-zedscripts command on an example script file](assets/pygments_implementation.png)
+
+To use the lexer in Sphinx docs, you can add the following to your `conf.py`:
+
+```python
+from ZedScripts.structure.pygments_lexer import ZedScriptsLexer
+from sphinx.highlighting import lexers
+lexers['zedscripts'] = ZedScriptsLexer()
+```
+
 ## License and contribution
 This project is licensed under a [MIT license](LICENSE). Before contributing to the project, please see [CONTRIBUTING](CONTRIBUTING.md).

{zedscripts-1.0.1 → zedscripts-1.2.0}/pyproject.toml

@@ -3,7 +3,7 @@ name = "ZedScripts"
 description = ""
 readme = { file = "README.md", content-type = "text/markdown" }
 requires-python = ">=3.12.3"
-version = "1.0.1"
+version = "1.2.0"
 license = {file = "LICENSE"}
 authors = [
   { name = "SimKDT" },
@@ -23,12 +23,21 @@ build = [
 debug = [
     "debugpy"
 ]
+test = [
+    "pytest"
+]
+pygments = [
+    "pygments"
+]
 
 # used to load locale JSON files
 [build-system]
 requires = ["setuptools>=61"]
 build-backend = "setuptools.build_meta"
 
+[project.scripts]
+highlight-zedscripts = "ZedScripts.structure.pygments_lexer:main"
+
 [tool.setuptools.package-data]
 ZedScripts = [
     "locale/*.json"

zedscripts-1.2.0/src/ZedScripts/__about__.py

@@ -0,0 +1,7 @@
+__version__ = "1.2.0"
+
+def main():
+    print(__version__)
+
+if __name__ == "__main__":
+    main()

zedscripts-1.2.0/src/ZedScripts/__init__.py

@@ -0,0 +1,75 @@
+import os
+import logging
+from datetime import datetime, timezone
+from pathlib import Path
+
+IDENTIFIER = "ZedScripts"
+SOURCE = IDENTIFIER
+
+IS_DEBUG = os.getenv("ZEDSCRIPTS_DEBUG_MODE") == "1"
+
+# zedscript's cache directory
+ZEDSCRIPT_CACHE_DIR = Path.home() / ".zedscripts"
+"""
+Cache directory to hold various information related to ZedScripts.
+"""
+ZEDSCRIPT_CACHE_DIR.mkdir(parents=True, exist_ok=True)
+
+# config files naming
+CONFIGURATION_FILE_NAME = ".zedscripts.json"
+GLOBAL_CONFIGURATION_FILE = (ZEDSCRIPT_CACHE_DIR / CONFIGURATION_FILE_NAME).resolve()
+
+# setup the logger
+
+log_dir = Path.home() / ".zedscripts" / "logs"
+log_dir.mkdir(parents=True, exist_ok=True)
+
+# define logger
+logger = logging.getLogger()
+
+# set early level, this is changed later by the user configuration
+# in the server initialization
+logger.setLevel(logging.DEBUG if IS_DEBUG else logging.INFO)
+
+formater = logging.Formatter("%(asctime)s %(levelname)s %(message)s")
+
+# setup the time-based log handler
+timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%d-%H-%M-%S-%f")
+time_log_file = log_dir / f"server-{timestamp}.log"
+time_handler = logging.FileHandler(time_log_file, encoding="utf-8")
+time_handler.setFormatter(
+    formater
+)
+logger.addHandler(time_handler)
+
+# setup the debug log handler which stays in the same file to make it easier to view logs
+# when restarting the server for example
+if IS_DEBUG:
+    global_log_file = log_dir / f"server-debug.log"
+    global_handler = logging.FileHandler(
+        global_log_file, encoding="utf-8", mode="w"
+    )
+    global_handler.setFormatter(
+        formater
+    )
+    logger.addHandler(global_handler)
+
+# filenames sort chronologically because the timestamp comes first
+keep_logs = 5
+log_files = sorted(log_dir.glob("server-*.log"), reverse=True)
+
+# delete the oldest log file
+for old_file in log_files[keep_logs:]:
+    try:
+        old_file.unlink()
+    except OSError:
+        logger.exception("Failed to delete old log file: %s", old_file)
+
+logging.info("ZedScripts logger initialized. Debug mode: %s", IS_DEBUG)
+
+# datasets
+SCRIPTS_DATA_MANIFEST = "https://raw.githubusercontent.com/PZ-Wiki-Modding/pz-scripts-data/refs/heads/main/manifest.json"
+SCRIPTS_BLOCKS_DATA_LINK = "https://raw.githubusercontent.com/PZ-Wiki-Modding/pz-scripts-data/refs/tags/{best_tag}/out/scriptsBlocks.json"
+ROOTS_DATA_LINK = "https://raw.githubusercontent.com/PZ-Wiki-Modding/pz-scripts-data/refs/tags/{best_tag}/out/roots.json"
+
+SCRIPTSDOCS_LINK = "https://pz-wiki-modding.github.io/PZ-API-Docs/scripts/"

{zedscripts-1.0.1 → zedscripts-1.2.0}/src/ZedScripts/enums/Diagnostic.py

@@ -52,6 +52,9 @@ class DiagnosticType(enum.StrEnum):
     VALUE_UNMATCHED_BLOCK_REF =              enum.auto()
     VALUE_MULTIPLE_BLOCK_REFS =              enum.auto()
 
+    # debug
+    DEBUG_MISSING_PARAMETER_DESCRIPTION =    enum.auto()
+
 
 # syntax diagnostics
 DiagnosticDefinition(DiagnosticType.PARSER_TOO_MANY_CLOSING_BRACKETS,
@@ -155,6 +158,10 @@ DiagnosticDefinition(DiagnosticType.VALUE_MULTIPLE_BLOCK_REFS,
                      DiagnosticSeverity.Error,
                      args={"value": str, "parameter": str, "expected_block": str})
 
+DiagnosticDefinition(DiagnosticType.DEBUG_MISSING_PARAMETER_DESCRIPTION,
+                     DiagnosticSeverity.Information,
+                     args={"type": str, "key": str})
+
 
 # ensure that all diagnostic types have a corresponding definition
 for diagnostic_type in DiagnosticType:

{zedscripts-1.0.1 → zedscripts-1.2.0}/src/ZedScripts/environment/__init__.py

@@ -1,4 +1,5 @@
 import enum
+import logging
 
 
 class WorkspaceType(enum.StrEnum):
@@ -31,3 +32,10 @@ class VersionType(enum.StrEnum):
     BASE_GAME = enum.auto()
     """Base game files (e.g., `ProjectZomboid/media`)."""
 
+class LoggerLevel(enum.IntEnum):
+    NOTSET = logging.NOTSET
+    DEBUG = logging.DEBUG
+    INFO = logging.INFO
+    WARNING = logging.WARNING
+    ERROR = logging.ERROR
+    CRITICAL = logging.CRITICAL

{zedscripts-1.0.1 → zedscripts-1.2.0}/src/ZedScripts/environment/config.py

@@ -1,11 +1,19 @@
+"""
+Provides the configuration models and schema generation for the ZedScripts environment.
+
+Usage:
+    python -m ZedScripts.environment.config schema -o schema/zedscripts.json
+    python -m ZedScripts.environment.config default -o default_config.json
+"""
 import logging
 import enum
 from pathlib import Path
 from typing import Annotated, Optional
-from pydantic import BaseModel, BeforeValidator, Field, WithJsonSchema
+from pydantic import BaseModel, BeforeValidator, Field, WithJsonSchema, ConfigDict
 
 from lsprotocol.types import DiagnosticSeverity, DiagnosticTag
 
+from ..environment import LoggerLevel
 from ..scripts.version import DataVersion
 from ..enums.Diagnostic import DiagnosticType
 
@@ -68,6 +76,24 @@ DiagnosticTagName = Annotated[
     }),
 ]
 
+def _parse_logger_level_name(value: object) -> object:
+    if isinstance(value, LoggerLevel):
+        return value
+    if isinstance(value, str):
+        try:
+            return LoggerLevel[value]
+        except KeyError:
+            raise ValueError("level must be a LoggerLevel member name") from None
+    return value
+LoggerLevelName = Annotated[
+    LoggerLevel,
+    BeforeValidator(_parse_logger_level_name),
+    WithJsonSchema({
+        "type": "string",
+        "enum": [member.name for member in LoggerLevel],
+    }),
+]
+
 
 
 # define the models
@@ -77,8 +103,13 @@ class DatasetTag(enum.StrEnum):
     STABLE = "stable"
 
 
+class ZedBaseModel(BaseModel):
+    """Base model for all ZedScripts configuration models.
+    Notably used to set additionalProperties to false for everything."""
+    model_config = ConfigDict(extra='forbid')
+
 
-class ReleaseModel(BaseModel):
+class ReleaseModel(ZedBaseModel):
     major: int = Field(
         description="Build game version number.",
         ge=42,
@@ -97,7 +128,7 @@ class ReleaseModel(BaseModel):
         ge=0,
     )
 
-class DatasetModel(BaseModel):
+class DatasetModel(ZedBaseModel):
     release: Optional[ReleaseModel] = Field(
         description="Provides a configuration for the dataset version to use for validation. This overrides the `latest` and `stable` fields if set.",
         default=None,
@@ -126,7 +157,7 @@ class DatasetModel(BaseModel):
             logging.error(f"Failed to get data version from release model: {e}. Defaulting to 'stable'.")
             return DatasetTag.STABLE
 
-class DiagnosticsModel(BaseModel):
+class DiagnosticsModel(ZedBaseModel):
     """Represents the diagnostics configuration for the workspace environment."""
     type: DiagnosticTypeName = Field(
         description="Specifies the diagnostic member name to enable.",
@@ -144,7 +175,15 @@ class DiagnosticsModel(BaseModel):
         default=None,
     )
 
-class ConfigurationModel(BaseModel):
+class ServerModel(ZedBaseModel):
+    """Represents the server configuration for the workspace environment."""
+    logging_level: Optional[LoggerLevelName] = Field(
+        description="Specifies the logging level for the server. This option only works in the global configuration.",
+        default=LoggerLevel.INFO,
+        exclude=True,
+    )
+
+class ConfigurationModel(ZedBaseModel):
     """Represents the content of the configuration files for workspace environments."""
     dataset: DatasetModel = Field(
         description="Provides configuration for the dataset version to use for diagnostics.",
@@ -166,6 +205,10 @@ class ConfigurationModel(BaseModel):
         description="List of diagnostics configurations for the workspace environment.",
         default_factory=list,
     )
+    server: ServerModel = Field(
+        description="Provides configuration for the server in the workspace environment.",
+        default_factory=ServerModel,
+    )
 
     def find_diagnostic(self, type: DiagnosticType) -> DiagnosticsModel | None:
         for diagnostic in self.diagnostics:
@@ -174,6 +217,46 @@ class ConfigurationModel(BaseModel):
         return None
 
 if __name__ == "__main__":
-    import json
-    config = ConfigurationModel()
-    print(json.dumps(config.model_json_schema(), indent=4))
+    import argparse
+
+    def print_schema(args: argparse.Namespace):
+        import json
+        config = ConfigurationModel()
+        out = json.dumps(config.model_json_schema(), indent=4)
+        print(out)
+        if args.output_path:
+            with open(args.output_path, "w") as f:
+                f.write(out)
+
+    def print_default_config(args: argparse.Namespace):
+        config = ConfigurationModel()
+        out = config.model_dump_json(indent=4)
+        print(out)
+        if args.output_path:
+            with open(args.output_path, "w") as f:
+                f.write(out)
+
+    main_parser = argparse.ArgumentParser(description="ZedScripts configuration tool")
+
+    # schema generation command
+    subparser = main_parser.add_subparsers(
+        dest="command",
+        required=True,
+        help="Available commands"
+    )
+    schema_parser = subparser.add_parser("schema", 
+                                         help="Print the JSON schema of the configuration model.")
+    schema_parser.add_argument("-o", "--output-path", 
+                               help="Path to output the JSON schema.")
+    schema_parser.set_defaults(func=print_schema)
+
+    # default configuration command
+    default_config_subparser = subparser.add_parser("default", 
+                                                    help="Print the default configuration of the workspace environment.")
+    default_config_subparser.add_argument("-o", "--output-path", 
+                                          help="Path to output the default configuration.")
+    default_config_subparser.set_defaults(func=print_default_config)
+
+    args = main_parser.parse_args()
+
+    args.func(args)

{zedscripts-1.0.1 → zedscripts-1.2.0}/src/ZedScripts/environment/document.py

@@ -131,12 +131,14 @@ class Document:
     def make(dataset: 'Dataset', path: Path, workspace: 'Workspace') -> 'Document | None':
         # find the rootType of the document
         rootType = dataset.test_for_root(path)
+        logging.debug(f"Tested {path} for root, got: {rootType}")
         if rootType is None:
             return None
 
         # find the version and skip if pre 42
         # since we don't validate that
         version = Version.find_or_make_version(path)
+        logging.debug(f"Tested {path} for version, got: {version}")
         if version.type == VersionType.PRE_42:
             return None
 

{zedscripts-1.0.1 → zedscripts-1.2.0}/src/ZedScripts/environment/workspace.py

@@ -260,7 +260,20 @@ class Workspace:
     @staticmethod
     def update_global_configuration(text: str | None = None) -> None:
         logging.info("Updating global configuration from %s", GLOBAL_CONFIGURATION_FILE)
-        Workspace.global_configuration = Workspace.load_configuration(GLOBAL_CONFIGURATION_FILE, text=text)
+        # init file if it doesn't exist to default values
+        if not GLOBAL_CONFIGURATION_FILE.exists():
+            with open(GLOBAL_CONFIGURATION_FILE, "w") as f:
+                config = ConfigurationModel()
+                f.write(config.model_dump_json(
+                    indent=4
+                ))
+        
+        # load existing global
+        else:
+            config = Workspace.load_configuration(GLOBAL_CONFIGURATION_FILE, text=text)
+
+        # cache current snapshot of the global configuration
+        Workspace.global_configuration = config
         logging.debug(Workspace.global_configuration)
 
     @staticmethod

{zedscripts-1.0.1 → zedscripts-1.2.0}/src/ZedScripts/locale/en.json

@@ -33,6 +33,8 @@
         "VALUE_CANNOT_PROVIDE_MODULE": "Value '{value}' cannot contain a module '{module}' reference, only direct block references are allowed.",
         "VALUE_BLOCK_REF_CANNOT_BE_EMPTY": "Value '{value}' has an empty block reference, which is not allowed.",
         "VALUE_UNMATCHED_BLOCK_REF": "Value '{value}' for parameter '{parameter}' does not match any existing block (expects '{expected_block}' block). Make sure the referenced block exists.",
-        "VALUE_MULTIPLE_BLOCK_REFS": "Multiple block references found for '{value}' for parameter '{parameter}' (expects '{expected_block}' block). Make sure duplicate block references are not present."
+        "VALUE_MULTIPLE_BLOCK_REFS": "Multiple block references found for '{value}' for parameter '{parameter}' (expects '{expected_block}' block). Make sure duplicate block references are not present.",
+
+        "DEBUG_MISSING_PARAMETER_DESCRIPTION": "DEBUG: Parameter '{key}' in block '{type}' is missing a description."
     }
 }

{zedscripts-1.0.1 → zedscripts-1.2.0}/src/ZedScripts/scripts/value.py

@@ -240,7 +240,7 @@ class Value(Element["ValueNode"]):
         if not dataset.can_block_have_parameter(parent_type, key):
             return ''
         param_data = dataset.get_parameter_data(parent_type, key)
-        return param_data.get('description', '') if param_data else ''
+        return param_data.get('description', '')
 
     def get_scriptsdocs_url(self, tree: list[str]) -> str | None:
         return self.parent.get_scriptsdocs_url(tree) + f"#scripts-{self.parent.type}-{self.key()}".replace(' ', '-').lower()
@@ -413,6 +413,14 @@ class Value(Element["ValueNode"]):
 
         param_data = dataset.get_parameter_data(parent_type, key)
 
+        # DEBUG: add information diagnostic if there is not description for the parameter
+        if not param_data.get('description', ''):
+            self.parent.add_diagnostic(
+                type=DiagnosticType.DEBUG_MISSING_PARAMETER_DESCRIPTION,
+                location=key_node.to_range(),
+                args={"type": parent_type, "key": key}
+            )
+
         # do the type highlight before anything else
         # so we highlight by type even if the value is otherwise invalid
         type_data = param_data.get('type')
@@ -652,8 +660,6 @@ class Value(Element["ValueNode"]):
             )
             return False
 
-        # TODO: look for the source block
-
         root = self.parent.get_root()
 
         # retrieve imports

{zedscripts-1.0.1 → zedscripts-1.2.0}/src/ZedScripts/server.py

@@ -47,7 +47,7 @@ class ZedServer(LanguageServer):
         try:
             import debugpy
         except ImportError:
-            logging.error("debugpy is not installed. Debugging will not be available.")
+            logging.error("debugpy is not installed. Debugging will not be available. Install the `debug` group dependencies (`pip install ZedScripts[debug]`).")
             return
 
         debug_port = 5678
@@ -148,7 +148,7 @@ zedserver = ZedServer()
 
 
 
-## EVENT HANDLERS
+## INITIALIZERS
 
 @zedserver.feature(types.INITIALIZE)
 def initialize(server: ZedServer, params: types.InitializeParams):
@@ -165,6 +165,11 @@ def initialize(server: ZedServer, params: types.InitializeParams):
     # this will store what the server can currently do
     capabilities.register_client_capabilities(params)
 
+    global_config = Workspace.get_global_configuration()
+    if global_config.server.logging_level is not None:
+        logging.info("Setting logging level to %s", global_config.server.logging_level)
+        logging.getLogger().setLevel(global_config.server.logging_level)
+
     # from the above, we determine what the server capabilities should be
     return types.InitializeResult(
         capabilities=capabilities.get_server_capabilities(),
@@ -217,6 +222,9 @@ def initialized(server: ZedServer, params: types.InitializedParams) -> None:
     notifications.send_notification(notifications.ZedNotification.LOADING_DOCUMENTS_DONE)
 
 
+
+## EVENT HANDLERS
+
 @zedserver.feature(types.TEXT_DOCUMENT_DID_OPEN)
 def did_open(server: ZedServer, params: types.DidOpenTextDocumentParams) -> None:
     server.on_document_changed(uri_to_path(params.text_document.uri), params.text_document.text)

{zedscripts-1.0.1 → zedscripts-1.2.0}/src/ZedScripts/structure/lexer.py

@@ -194,25 +194,21 @@ WHITESPACE_NO_NEWLINE = string.whitespace.replace("\n", "")
 
 
 
-class ZedscriptSource:
-    def __init__(self, text: str) -> None:
-        self.string: str = text
-        self.tokens: TokenCollection = TokenCollection()
-
-
 class Token:
     def __init__(
             self, type: TokenType, pos: TextPosition, end: TextPosition,
-            source: ZedscriptSource, text: str
+            text: str
     ) -> None:
         self.type: TokenType = type
         self.pos: TextPosition = pos
         """Start position of the token."""
         self.end: TextPosition = end
         """End position of the token."""
-        self.source: ZedscriptSource = source
         self.text = text
 
+    def __repr__(self) -> str:
+        return f"Token(type={self.type}, text={self.text!r}, pos={self.pos}, end={self.end})"
+
     def pos_of(self, i: int) -> TextPosition:
         """
         Returns the TextPosition of the character at index i in the string.
@@ -233,6 +229,16 @@ class Token:
     def __str__(self) -> str:
         return self.text
 
+    def __eq__(self, value: object) -> bool:
+        if not isinstance(value, Token):
+            return NotImplemented
+        return (
+            self.type == value.type and
+            self.pos == value.pos and
+            self.end == value.end and
+            self.text == value.text
+        )
+
 
 class TokenCollection(UserList[Token]):
     def at(self, pos: TextPosition) -> Token | None:
@@ -314,23 +320,22 @@ class TokenCollection(UserList[Token]):
 
 # TODO: this class is kinda redundant after all
 class TokenBuilder:
-    def __init__(self, pos: TextPosition, type: TokenType, source: ZedscriptSource) -> None:
+    def __init__(self, pos: TextPosition, type: TokenType) -> None:
         self.type: TokenType = type
         self.pos: TextPosition = pos
         self.text: str = ""
-        self.source: ZedscriptSource = source
 
     def add(self, char: str) -> None:
         self.text += char
 
     def build(self, end: TextPosition) -> Token:
-        return Token(self.type, self.pos, end, self.source, self.text)
+        return Token(self.type, self.pos, end, self.text)
 
 
 class Lexer:
     def __init__(self, text: str) -> None:
         self.text: str = text
-        self.source = ZedscriptSource(text)
+        self.tokens = TokenCollection()
         self.pos: int = -1
         self.line: int = 0
         self.offset: int = 0
@@ -364,7 +369,7 @@ class Lexer:
         return self.text[self.pos + num_characters]
 
     def start_token(self, type: TokenType) -> TokenBuilder:
-        return TokenBuilder(TextPosition(self.line, self.offset), type, self.source)
+        return TokenBuilder(TextPosition(self.line, self.offset), type)
 
     def tokenize_comment(self) -> None:
         assert self.check_comment()
@@ -381,7 +386,7 @@ class Lexer:
                 token.add(self.next())
                 break
 
-        self.source.tokens.append(
+        self.tokens.append(
             token.build(
                 TextPosition(self.line, self.offset)
             )
@@ -396,7 +401,7 @@ class Lexer:
     def tokenize_punctuator(self) -> None:
         token = self.start_token(TokenType.PUNCTUATOR)
         token.add(self.next())
-        self.source.tokens.append(
+        self.tokens.append(
             token.build(
                 TextPosition(self.line, self.offset)
             )
@@ -414,7 +419,7 @@ class Lexer:
                 break
             token.add(self.next())
 
-        self.source.tokens.append(
+        self.tokens.append(
             token.build(
                 TextPosition(self.line, self.offset)
             )
@@ -429,7 +434,7 @@ class Lexer:
                 break
             token.add(self.next())
 
-        self.source.tokens.append(
+        self.tokens.append(
             token.build(
                 TextPosition(self.line, self.offset)
             )
@@ -449,7 +454,7 @@ class Lexer:
             elif char in ELEMENTS_DELIMITERS:
                 token = lexer.start_token(TokenType.ELEMENT_DELIMITER)
                 token.add(lexer.next())
-                lexer.source.tokens.append(
+                lexer.tokens.append(
                     token.build(
                         TextPosition(lexer.line, lexer.offset)
                     )
@@ -460,7 +465,7 @@ class Lexer:
                 continue
             lexer.tokenize_text()
 
-        return TokenCollection(lexer.source.tokens)
+        return TokenCollection(lexer.tokens)
 
 
 if __name__ == "__main__":