janus-llm 2.0.2__py3-none-any.whl → 3.0.0__py3-none-any.whl

{janus_llm-2.0.2.dist-info → janus_llm-3.0.0.dist-info}/RECORD

@@ -1,14 +1,21 @@
-janus/__init__.py,sha256=GWW38p6MV8wH_SpAjgMKsoncBmX5J-7qBjCjgyoO8TY,341
-janus/__main__.py,sha256=Qd-f8z2Q2vpiEP2x6PBFsJrpACWDVxFKQk820MhFmHo,59
+janus/__init__.py,sha256=LVDmiK9hI9u2qwCVVKQtBo3fR1FomBig_DphrdUtr3E,351
+janus/__main__.py,sha256=lEkpNtLVPtFo8ySDZeXJ_NXDHb0GVdZFPWB4gD4RPS8,64
 janus/_tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 janus/_tests/conftest.py,sha256=V7uW-oq3YbFiRPvrq15YoVVrA1n_83pjgiyTZ-IUGW8,963
-janus/_tests/test_cli.py,sha256=oP-WOM-ai4jZLDGqjLzI6kCtfXWpoeUR8TynP5p6cVg,4254
-janus/_tests/test_translate.py,sha256=71oRTTfdSVFOBvUhuOavgl3TuimTz1K6eG-04pUGpfE,16439
-janus/cli.py,sha256=X_7HXQGeQEYUkIF3AHbbc04lUEUjKegZ7tUFwmKjf10,29549
-janus/converter.py,sha256=ge7nJA1DlOrHW_uB9P0dguc48Au8g7bqCERNqMIqfxs,5941
+janus/_tests/test_cli.py,sha256=mi7wAWV07ZFli5nQdExRGIGA3AMFD9s39-HcmDV4B6Y,4232
+janus/cli.py,sha256=-aeg8R6CobK2EG_BPoZgBy_x1d6G9gp-KKKhnLMepo4,29541
+janus/converter/__init__.py,sha256=kzVmWOPXRDayqqBZ8ZDaFQzA_q8PEdv407dc-DefPxY,255
+janus/converter/_tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+janus/converter/_tests/test_translate.py,sha256=eiLbmouokZrAeAYmdoJgnlx5-k4QiO6i0N6e6ZvZsvM,15885
+janus/converter/converter.py,sha256=Bq07_9N_3Dv9NBqVACvb7LC2HxdQmfVZ1b0BlWrxjgo,23521
+janus/converter/diagram.py,sha256=v-3ZZ4t1q74lDOjF2N6NRPkC3IK-sjLDn5_VChZTEGA,4608
+janus/converter/document.py,sha256=hsW512veNjFWbdl5WriuUdNmMEqZy8ktRvqn9rRmA6E,4566
+janus/converter/evaluate.py,sha256=APWQUY3gjAXqkJkPzvj0UA4wPK3Cv9QSJLM-YK9t-ng,476
+janus/converter/requirements.py,sha256=orOVFymFSqSVyyII1DeDqeo3pTZLxw6dJ-JKBPigR0A,1751
+janus/converter/translate.py,sha256=kMlGUiBYGQBXSxwX5in3CUyUifPM95wynCaRMxSDxMw,4238
 janus/embedding/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 janus/embedding/_tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-janus/embedding/_tests/test_collections.py,sha256=llg-JSuRRFhKkHFiWWSHEWV3iaT6Lwue0lp2tEml9io,2668
+janus/embedding/_tests/test_collections.py,sha256=eT0cYv-qmPrHJRjDZqWPFTkqVzFDRoPrRKR__FPiz58,2651
 janus/embedding/_tests/test_database.py,sha256=uqI2Jgj8DEIlciqiwiZx_n0osjcspIPrHOSSN1NRZSk,1019
 janus/embedding/_tests/test_vectorize.py,sha256=NnJLHBwgMVycAProRJxuLVSByxrpJ35eaZCFca52gNY,1964
 janus/embedding/collections.py,sha256=ZE8QGYQ82DCLqhV0m1y7PiqpuHjEfxHPcS5SCKU0LAw,5411
@@ -18,35 +25,39 @@ janus/embedding/vectorize.py,sha256=ap3e6ZMai8U3M5vdpLc_st4Sw31xyqoaqEno0IJlVOU,
 janus/language/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 janus/language/_tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 janus/language/_tests/test_combine.py,sha256=ydCYNbTxvaxT-5axiEBzPQLn6s4arSyZ5Tx2SYKLpJY,1830
-janus/language/_tests/test_splitter.py,sha256=Hqexa39LLEXlK3ZUw7Zot4PUIACvye2vkq0Jaox0T10,373
+janus/language/_tests/test_splitter.py,sha256=VK48eqp5PYJfjdhD_x7IkeAjbF1KC3AyNnICfK8XnUQ,360
+janus/language/alc/__init__.py,sha256=j7vOMGhT1Vri6p8dsjSaY-fkO5uFn0sJ0nrNGGvcizM,42
+janus/language/alc/_tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+janus/language/alc/_tests/test_alc.py,sha256=DttXpouP9Vkdlf23_a0qFalKdGtadGv6oXTsmN1pk8Q,994
+janus/language/alc/alc.py,sha256=n8KVHTb6FFILw50N8UM3gfT60gLVvkTjk37easwluWs,3061
 janus/language/binary/__init__.py,sha256=AlNAe12ZA366kcGSrQ1FJyOdbwxFqGBFkYR2K6yL818,51
 janus/language/binary/_tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 janus/language/binary/_tests/test_binary.py,sha256=a-8RSfKA23UrJC9c1xPQK792XZCz8npCHI7isN2dAP8,1727
 janus/language/binary/binary.py,sha256=CS1RAieN8klSsCeXQEFYKUWioatUX-sOPXKQr5S6NzE,6534
 janus/language/binary/reveng/decompile_script.py,sha256=veW51oJzuO-4UD3Er062jXZ_FYtTFo9OCkl82Z2xr6A,2182
-janus/language/block.py,sha256=4f8e3YYSS2p-0fXjjl2erbbXDOHcBxiLzDHALKlPTg4,9188
-janus/language/combine.py,sha256=hSEc1dHLcOELks-ZGsRKHOgYBXAronKus6BTmb1u42k,2940
+janus/language/block.py,sha256=57hfOY-KSVMioKhkCvfDtovQt4h8lCg9cJbRF7ddV1s,9280
+janus/language/combine.py,sha256=e7j8zQO_D3_LElaVCsGgtnzia7aFFK56m-mhArQBlR0,2908
 janus/language/file.py,sha256=X2MYcAMlCABK77uhMdI_J2foXLrqEdinapYRfLPyKB8,563
 janus/language/mumps/__init__.py,sha256=-Ou_wJ-JgHezfp1dub2_qCYNiK9wO-zo2MlqxM9qiwE,48
 janus/language/mumps/_tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 janus/language/mumps/_tests/test_mumps.py,sha256=6l7q14lPnKf231iWwMdRbf-dg9QuHa26YMS7-K7yC4A,1001
-janus/language/mumps/mumps.py,sha256=J8ewuLDh7y9GeyCcGNYqCGJ9HOwrEWypc2HNxClZ8is,7382
+janus/language/mumps/mumps.py,sha256=MkF_TZB1SOIj3JQfGKYow1Hh2Bja0EglUlpd4aAY5Iw,7351
 janus/language/mumps/patterns.py,sha256=FW5T6Nt5kBO2UKgSL1KLVDbYRgMaJAzDvEmvBkxHppA,2310
-janus/language/naive/__init__.py,sha256=6P5rDAZtoHTObkFbZkiEdd-PVRA_9VTQogUjwvRMKK8,198
-janus/language/naive/basic_splitter.py,sha256=RM9pJK2YkHfb6_EFEV-dh_rLqkjS6v0cn3ASPf8A6Fg,459
-janus/language/naive/chunk_splitter.py,sha256=ebRSbaJhDW-Hyr5__ukbdmAl6kQ1WWFqrq_SfCgHo6k,772
-janus/language/naive/registry.py,sha256=8YQX1q0IdAm7t69-oC_00I-vfkdRnHuX-OD3KEjEIuU,294
-janus/language/naive/simple_ast.py,sha256=gix_fh864sHZ5KeXoOZIVdKdQeCN_4Qwq4Ox-haZ6sY,593
-janus/language/naive/tag_splitter.py,sha256=IXWMn9tBVUGAtzvQi89GhoZ6g7fPXk5MzO0kMCr2mb0,2045
+janus/language/naive/__init__.py,sha256=gsdC543qsIX8y_RxblCBIgyW0tfucljFms6v2WTrEz0,178
+janus/language/naive/basic_splitter.py,sha256=NFW3TvMFQwEmcj5r4jvQXBJCzgNcSZI-3Arjb191gAo,407
+janus/language/naive/chunk_splitter.py,sha256=g1nqbhvaOZ31SjO-smIwAg6lHGTy2rPOOnQ-m6fIKAA,713
+janus/language/naive/registry.py,sha256=CDUkMIgscdPBV_qu49u9TGnOIgr9mRasinPRwViTWz8,281
+janus/language/naive/simple_ast.py,sha256=boX_pJ8x52_MxiM6hJ-0oa6MR75Fu4pyebBEtYJJZUc,907
+janus/language/naive/tag_splitter.py,sha256=6DHBJdM3IllcVV-MrAyj8KPg5zXTiRdeD42CYrmEFHk,1986
 janus/language/node.py,sha256=-ymv--oILEYLVO2KSOrzOlzL2cZHNQpQJYwE1cKA-pY,200
-janus/language/splitter.py,sha256=Ep8RxWrnuih3MAcdkkbtAsSLrPmyQcjnk0IzbRC-460,16741
+janus/language/splitter.py,sha256=4XAe0hXka7njS30UHGCngJzDgHxn3lygUjikSHuV7Xo,16924
 janus/language/treesitter/__init__.py,sha256=mUliw7ZJLZ8NkJKyUQMSoUV82hYXE0HvLHrEdGPJF4Q,43
 janus/language/treesitter/_tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 janus/language/treesitter/_tests/test_treesitter.py,sha256=nsavUV0aI6cpT9FkQve58eTTehLyQG6qJJBGlNa_bIw,2170
-janus/language/treesitter/treesitter.py,sha256=9hbP7eBuSEKSZm6OD4C9q2tbjzrEidaCAKw74aO4lEM,6855
+janus/language/treesitter/treesitter.py,sha256=UiV4OuWTt6IwMohHSw4FHsVNA_zxr9lNk4_Du09APdo,7509
 janus/llm/__init__.py,sha256=8Pzn3Jdx867PzDc4xmwm8wvJDGzWSIhpN0NCEYFe0LQ,36
-janus/llm/model_callbacks.py,sha256=zMCbMgniKrzKf-sU9SxOcfoOvc3xz7y0VxIxfdlS5tA,6766
-janus/llm/models_info.py,sha256=jNTp7mg7MVSS-Anp9Z-wMTz8odiE-1xXeyi8ngpJi1E,7151
+janus/llm/model_callbacks.py,sha256=h_xlBAHRx-gxQBBjVKRpGXxdxYf6d9L6kBoXjbEAEdI,7106
+janus/llm/models_info.py,sha256=B9Dn5mHc43OeZe5mHFj5wuhO194XHCTwShAa2ybnPyY,7688
 janus/metrics/__init__.py,sha256=AsxtZJUzZiXJPr2ehPPltuYP-ddechjg6X85WZUO7mA,241
 janus/metrics/_tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 janus/metrics/_tests/reference.py,sha256=hiaJPP9CXkvFBV_wL-gOe_BzELTw0nvB6uCxhxtIiE8,13
@@ -54,19 +65,19 @@ janus/metrics/_tests/target.py,sha256=hiaJPP9CXkvFBV_wL-gOe_BzELTw0nvB6uCxhxtIiE
 janus/metrics/_tests/test_bleu.py,sha256=TcSnNGpMh00Nkkk1zq5wDfdCANMUq9eXscU_hcBRU8A,1640
 janus/metrics/_tests/test_chrf.py,sha256=O4v1Cj513H8NYffJILpSI7CuR_dnm7F8CeB3C7sZYr0,2202
 janus/metrics/_tests/test_file_pairing.py,sha256=A4Qy6JIesFXUcaig45Ze6LiViuHQS7MFSQzDHQP3j9w,1880
-janus/metrics/_tests/test_llm.py,sha256=IYsLwX5zC2WcaaPeSlHuQVmU2sB55-dsOXnmQPhcKps,3007
+janus/metrics/_tests/test_llm.py,sha256=dGXrdd79v-ix_560t6Q8RJEx-6mgZ-pkzJgm-O2ZBwA,2998
 janus/metrics/_tests/test_reading.py,sha256=NDLFyjmOpM5gWf1LLTjGIw3aUR8Qf22zTt9hwe7NABs,840
-janus/metrics/_tests/test_rouge_score.py,sha256=dnP99nry-U5wyE-CiC0eQwm78IyScnmGQH3BeIEdmLY,2032
-janus/metrics/_tests/test_similarity_score.py,sha256=tdzH_8hYb2h7fKxpd_a75di-GPnU_frZ0zn2aeYzkso,811
+janus/metrics/_tests/test_rouge_score.py,sha256=rcHmrpy55cW507PnTnGQnp9Tsn5rk7JEyXmusY7la3Q,2020
+janus/metrics/_tests/test_similarity_score.py,sha256=jc3r0lWW5Iqm6AMKc36ewz5rboKwVw29fliBHClkzIg,799
 janus/metrics/_tests/test_treesitter_metrics.py,sha256=tqpAg9LY811gfQ3n2ypRqBJesAFQodMf6Gz7dvOsqp4,4337
 janus/metrics/bleu.py,sha256=eRoHIQulPp5mezJzHCNkwUB_89tAj4PqV2pF9eV9HfI,1746
 janus/metrics/chrf.py,sha256=zNGWZ40CPMgj8rctnmwkbf25_PvSOLPbOjv-iN2cGXM,1472
 janus/metrics/cli.py,sha256=Duuw2RF47Z-t1pal0cg3L_-N_91rx29krirqtIwjYLY,157
-janus/metrics/complexity_metrics.py,sha256=kJh_TyZttMP716MXGfl-WbaS4beR_DaQWGYVg6MurSU,6573
+janus/metrics/complexity_metrics.py,sha256=1Z9n0o_CrILqayk40wRkjR1f7yvHIsJG38DxAbqj614,6560
 janus/metrics/file_pairing.py,sha256=WNHRV1D8GOJMq8Pla5SPkTDAT7yVaS4-UU0XIGKvEVs,3729
 janus/metrics/llm_metrics.py,sha256=3677S6GYcoVcokpmAN-fwvNu-lYWAKd7M5mebiE6RZc,5687
-janus/metrics/metric.py,sha256=QRzLCkHY2g4pWDIiVLNaxNiEMF4gXw_eUrjecH5CdwA,16987
-janus/metrics/reading.py,sha256=KYuWjKnk0ALRU5S7mCNNZtaNgK02l0fdIGsaNvxLMO4,1690
+janus/metrics/metric.py,sha256=Lgdtq87oJ-kWC_6jdPQ6-d1MqoeTnhkRszo6IZJV6c0,16974
+janus/metrics/reading.py,sha256=srLb2MO-vZL5ccRjaHz-dA4MwAvXVNyIKnOrvJXg77E,2244
 janus/metrics/rouge_score.py,sha256=HfUJwUWI-yq5pOjML2ee4QTOMl0NQahnqEY2Mt8Dtnw,2865
 janus/metrics/similarity.py,sha256=9pjWWpLKCsk0QfFfSgQNdPXiisqi7WJYOOHaiT8S0iY,1613
 janus/metrics/splitting.py,sha256=610ScHRvALwdkqA6YyGI-tr3a18_cUofldBxGYX0SwE,968
@@ -78,17 +89,16 @@ janus/parsers/doc_parser.py,sha256=X8eCb1QXbL6sVWLEFGjsPyxrpJ9XnOPg7G4KZSo9A9E,5
 janus/parsers/eval_parser.py,sha256=HB5-zY_Jpmkj6FDbuNCCVCRxwmzhViSAjPKbyyC0Ebc,2723
 janus/parsers/reqs_parser.py,sha256=MFBvtR3otpyPZlkZxu0dVH1YeEJhvhNzhaGKGHaQVHA,2359
 janus/prompts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-janus/prompts/prompt.py,sha256=GMQ9EqwtIhB_x8MgfoeTeb4OkLaySYdGd3wVbpDMZXA,8911
-janus/translate.py,sha256=bsQ1YvjCPrVrL3y-rAA2PrCv2-x3ObCF7a6LuPWQPuE,38747
+janus/prompts/prompt.py,sha256=vd7UbitF0VFCi21RsggDebD51xcuyls_lQLGKkphfI8,10578
 janus/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 janus/utils/_tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 janus/utils/_tests/test_logger.py,sha256=4jZFm8LX828Dt9lOjiFHZIPbxYy_hHaswyrMPkscgdM,2199
 janus/utils/_tests/test_progress.py,sha256=Yh5NDNq-24n2nhHHbJm39pENAH70PYnh9ymwdcn0_UU,481
-janus/utils/enums.py,sha256=SlZKHojLPYOSjuekQGirSHem5Etcgy57txCtVCej2Ag,27533
+janus/utils/enums.py,sha256=AoilbdiYyMvY2Mp0AM4xlbLSELfut2XMwhIM1S_msP4,27610
 janus/utils/logger.py,sha256=KZeuaMAnlSZCsj4yL0P6N-JzZwpxXygzACWfdZFeuek,2337
 janus/utils/progress.py,sha256=pKcCzO9JOU9fSD7qTmLWcqY5smc8mujqQMXoPgqNysE,1458
-janus_llm-2.0.2.dist-info/LICENSE,sha256=_j0st0a-HB6MRbP3_BW3PUqpS16v54luyy-1zVyl8NU,10789
-janus_llm-2.0.2.dist-info/METADATA,sha256=sUtUMkjBZEmov6apazRphlqnBKG02dLkzWmq6-qIrqc,4184
-janus_llm-2.0.2.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-janus_llm-2.0.2.dist-info/entry_points.txt,sha256=OGhQwzj6pvXp79B0SaBD5apGekCu7Dwe9fZZT_TZ544,39
-janus_llm-2.0.2.dist-info/RECORD,,
+janus_llm-3.0.0.dist-info/LICENSE,sha256=_j0st0a-HB6MRbP3_BW3PUqpS16v54luyy-1zVyl8NU,10789
+janus_llm-3.0.0.dist-info/METADATA,sha256=arXVqb1tVW76Bpjj1b1hq-rWsvu25DHg9rNv3ZaFZW0,4184
+janus_llm-3.0.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+janus_llm-3.0.0.dist-info/entry_points.txt,sha256=OGhQwzj6pvXp79B0SaBD5apGekCu7Dwe9fZZT_TZ544,39
+janus_llm-3.0.0.dist-info/RECORD,,

janus/converter.py

@@ -1,158 +0,0 @@
-import functools
-from typing import Any
-
-from langchain.schema.language_model import BaseLanguageModel
-
-from .language.binary import BinarySplitter
-from .language.mumps import MumpsSplitter
-from .language.splitter import Splitter
-from .language.treesitter import TreeSitterSplitter
-from .utils.enums import CUSTOM_SPLITTERS, LANGUAGES
-from .utils.logger import create_logger
-
-log = create_logger(__name__)
-
-
-def run_if_changed(*tracked_vars):
-    """Wrapper to skip function calls if the given instance attributes haven't
-    been updated. Requires the _changed_attrs set to exist, and the __setattr__
-    method to be overridden to track parameter updates in _changed_attrs.
-    """
-
-    def wrapper(func):
-        @functools.wraps(func)
-        def wrapped(self, *args, **kwargs):
-            # If there is overlap between the tracked variables and the changed
-            #  ones, then call the function as normal
-            if self._changed_attrs.intersection(tracked_vars):
-                func(self, *args, **kwargs)
-
-        return wrapped
-
-    return wrapper
-
-
-class Converter:
-    """Parent class that converts code into something else.
-
-    Children will determine what the code gets converted into. Whether that's translated
-    into another language, into pseudocode, requirements, documentation, etc., or
-    converted into embeddings
-    """
-
-    def __init__(
-        self,
-        source_language: str = "fortran",
-        max_tokens: None | int = None,
-        protected_node_types: set[str] | list[str] | tuple[str] = (),
-        prune_node_types: set[str] | list[str] | tuple[str] = (),
-    ) -> None:
-        """Initialize a Converter instance.
-
-        Arguments:
-            source_language: The source programming language.
-            parser_type: The type of parser to use for parsing the LLM output. Valid
-                values are `"code"`, `"text"`, `"eval"`, and `None` (default). If `None`,
-                the `Converter` assumes you won't be parsing an output (i.e., adding to an
-                embedding DB).
-        """
-        self._changed_attrs: set = set()
-
-        self._source_language: None | str
-        self._source_glob: None | str
-        self._protected_node_types: tuple[str] = ()
-        self._prune_node_types: tuple[str] = ()
-        self._splitter: None | Splitter
-        self._llm: None | BaseLanguageModel = None
-        self._max_tokens: None | int = max_tokens
-
-        self.set_source_language(source_language)
-        self.set_protected_node_types(protected_node_types)
-        self.set_prune_node_types(prune_node_types)
-
-        # Child class must call this. Should we enforce somehow?
-        # self._load_parameters()
-
-    def __setattr__(self, key: Any, value: Any) -> None:
-        if hasattr(self, "_changed_attrs"):
-            if not hasattr(self, key) or getattr(self, key) != value:
-                self._changed_attrs.add(key)
-        # Avoid infinite recursion
-        elif key != "_changed_attrs":
-            self._changed_attrs = set()
-        super().__setattr__(key, value)
-
-    def _load_parameters(self) -> None:
-        self._load_splitter()
-        self._changed_attrs.clear()
-
-    def set_source_language(self, source_language: str) -> None:
-        """Validate and set the source language.
-
-        The affected objects will not be updated until _load_parameters() is called.
-
-        Arguments:
-            source_language: The source programming language.
-        """
-        source_language = source_language.lower()
-        if source_language not in LANGUAGES:
-            raise ValueError(
-                f"Invalid source language: {source_language}. "
-                "Valid source languages are found in `janus.utils.enums.LANGUAGES`."
-            )
-
-        self._source_glob = f"**/*.{LANGUAGES[source_language]['suffix']}"
-        self._source_language = source_language
-
-    def set_protected_node_types(
-        self, protected_node_types: set[str] | list[str] | tuple[str]
-    ) -> None:
-        """Set the protected (non-mergeable) node types. This will often be structures
-        like functions, classes, or modules which you might want to keep separate
-
-        The affected objects will not be updated until _load_parameters() is called.
-
-        Arguments:
-            protected_node_types: A set of node types that aren't to be merged
-        """
-        self._protected_node_types = tuple(set(protected_node_types or []))
-
-    def set_prune_node_types(
-        self, prune_node_types: set[str] | list[str] | tuple[str]
-    ) -> None:
-        """Set the node types to prune. This will often be structures
-        like comments or whitespace which you might want to keep out of the LLM
-
-        The affected objects will not be updated until _load_parameters() is called.
-
-        Arguments:
-            prune_node_types: A set of node types which should be pruned
-        """
-        self._prune_node_types = tuple(set(prune_node_types or []))
-
-    @run_if_changed(
-        "_source_language",
-        "_max_tokens",
-        "_llm",
-        "_protected_node_types",
-        "_prune_node_types",
-    )
-    def _load_splitter(self) -> None:
-        """Load the splitter according to this instance's attributes.
-
-        If the relevant fields have not been changed since the last time this method was
-        called, nothing happens.
-        """
-        kwargs = dict(
-            max_tokens=self._max_tokens,
-            model=self._llm,
-            protected_node_types=self._protected_node_types,
-            prune_node_types=self._prune_node_types,
-        )
-        if self._source_language in CUSTOM_SPLITTERS:
-            if self._source_language == "mumps":
-                self._splitter = MumpsSplitter(**kwargs)
-            elif self._source_language == "binary":
-                self._splitter = BinarySplitter(**kwargs)
-        else:
-            self._splitter = TreeSitterSplitter(language=self._source_language, **kwargs)