chunk-metadata-adapter 2.2.0__tar.gz → 2.2.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/PKG-INFO +1 -1
- {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/chunk_metadata_adapter/__init__.py +1 -1
- {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/chunk_metadata_adapter/chunk_query.py +2 -3
- {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/chunk_metadata_adapter/data_types.py +55 -7
- {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/chunk_metadata_adapter/examples.py +0 -37
- {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/chunk_metadata_adapter/metadata_builder.py +1 -16
- {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/chunk_metadata_adapter/semantic_chunk.py +4 -9
- {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/chunk_metadata_adapter.egg-info/PKG-INFO +1 -1
- {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/pyproject.toml +1 -1
- {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/tests/test_chunk_query.py +0 -2
- {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/tests/test_data_types.py +28 -1
- {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/tests/test_examples.py +2 -23
- {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/tests/test_language_enum_integration.py +31 -1
- {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/tests/test_metadata_builder.py +1 -422
- {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/tests/test_round_trip_redis.py +9 -13
- {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/README.md +0 -0
- {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/chunk_metadata_adapter/query_business_examples.py +0 -0
- {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/chunk_metadata_adapter/query_examples.py +0 -0
- {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/chunk_metadata_adapter/query_serialization_examples.py +0 -0
- {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/chunk_metadata_adapter/utils.py +0 -0
- {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/chunk_metadata_adapter.egg-info/SOURCES.txt +0 -0
- {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/chunk_metadata_adapter.egg-info/dependency_links.txt +0 -0
- {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/chunk_metadata_adapter.egg-info/requires.txt +0 -0
- {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/chunk_metadata_adapter.egg-info/top_level.txt +0 -0
- {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/setup.cfg +0 -0
- {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/tests/test_array_validation.py +0 -0
- {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/tests/test_coverage_improvement.py +0 -0
- {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/tests/test_examples_part1.py +0 -0
- {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/tests/test_examples_part2.py +0 -0
- {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/tests/test_examples_part3.py +0 -0
- {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/tests/test_full_cycle_all_fields.py +0 -0
- {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/tests/test_full_cycle_integration.py +0 -0
- {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/tests/test_language_integration.py +0 -0
- {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/tests/test_lifecycle.py +0 -0
- {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/tests/test_metadata_builder_part1.py +0 -0
- {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/tests/test_metadata_builder_part2.py +0 -0
- {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/tests/test_metrics_extension.py +0 -0
- {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/tests/test_models_part1.py +0 -0
- {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/tests/test_query_business_examples.py +0 -0
- {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/tests/test_query_examples.py +0 -0
- {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/tests/test_query_serialization_examples.py +0 -0
- {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/tests/test_semantic_chunk_advanced.py +0 -0
- {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/tests/test_semantic_chunk_basic.py +0 -0
- {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/tests/test_semantic_chunk_coverage_part1.py +0 -0
- {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/tests/test_semantic_chunk_coverage_part2.py +0 -0
- {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/tests/test_semantic_chunk_edge.py +0 -0
- {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/tests/test_semantic_chunk_is_code.py +0 -0
- {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/tests/test_semantic_chunk_serialization.py +0 -0
- {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/tests/test_utils.py +0 -0
{chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/chunk_metadata_adapter/chunk_query.py
RENAMED
|
@@ -64,7 +64,6 @@ class ChunkQuery(BaseModel):
|
|
|
64
64
|
source_lines_end: Optional[Union[str, int, float, bool, dict, list]] = Field(default=None, description="End line in the source file")
|
|
65
65
|
tags: Optional[List[str]] = Field(default=None, description="Categorical tags for the chunk.")
|
|
66
66
|
links: Optional[List[str]] = Field(default=None, description="References to other chunks in the format 'relation:uuid'.")
|
|
67
|
-
chunks: Optional[List[str]] = Field(default=None, description="List of text chunks returned after vectorization (search by chunk content).")
|
|
68
67
|
block_meta: Optional[dict] = Field(default=None, description="Additional metadata about the block.")
|
|
69
68
|
tags_flat: Optional[str] = Field(default=None, description="Comma-separated tags for flat storage.")
|
|
70
69
|
link_related: Optional[Union[str, int, float, bool, dict, list]] = Field(default=None, description="Related chunk UUID")
|
|
@@ -143,8 +142,8 @@ class ChunkQuery(BaseModel):
|
|
|
143
142
|
if field in uuid_fields or field in enum_fields:
|
|
144
143
|
continue
|
|
145
144
|
if isinstance(value, (list, dict)):
|
|
146
|
-
# Для фильтра допускается list только для tags, links
|
|
147
|
-
if field not in ["tags", "links", "
|
|
145
|
+
# Для фильтра допускается list только для tags, links и dict для block_meta
|
|
146
|
+
if field not in ["tags", "links", "block_meta"]:
|
|
148
147
|
errors.setdefault(field, []).append(f"{field} must be str, int, float, bool or dict (got {type(value).__name__})")
|
|
149
148
|
elif not isinstance(value, allowed_types):
|
|
150
149
|
errors.setdefault(field, []).append(f"{field} must be str/int/float/bool/None, got {type(value).__name__}")
|
{chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/chunk_metadata_adapter/data_types.py
RENAMED
|
@@ -219,9 +219,26 @@ class BlockType(str, ComparableEnum):
|
|
|
219
219
|
|
|
220
220
|
|
|
221
221
|
class LanguageEnum(str, ComparableEnum):
|
|
222
|
-
"""Programming and
|
|
222
|
+
"""Programming, natural, and formula languages supported by the system.
|
|
223
223
|
|
|
224
|
-
Includes all 54 programming languages supported by guesslang
|
|
224
|
+
Includes all 54 programming languages supported by guesslang, natural languages, and formula languages (e.g., LaTeX, MathML, AsciiMath, MathJax, KaTeX, SymPy).
|
|
225
|
+
|
|
226
|
+
Special values:
|
|
227
|
+
- UNKNOWN: Unknown or unspecified language
|
|
228
|
+
|
|
229
|
+
Natural languages:
|
|
230
|
+
- EN, RU, UK, DE, FR, ES, ZH, JA
|
|
231
|
+
|
|
232
|
+
Programming languages:
|
|
233
|
+
- Assembly, Batchfile, C, C#, C++, Clojure, CMake, COBOL, CoffeeScript, CSS, CSV, Dart, DM, Dockerfile, Elixir, Erlang, Fortran, Go, Groovy, Haskell, HTML, INI, Java, JavaScript, JSON, Julia, Kotlin, Lisp, Lua, Makefile, Markdown, Matlab, Objective-C, OCaml, Pascal, Perl, PHP, PowerShell, Prolog, Python, R, Ruby, Rust, Scala, Shell, SQL, Swift, TeX, TOML, TypeScript, Verilog, Visual Basic, XML, YAML, 1C
|
|
234
|
+
|
|
235
|
+
Formula languages (for mathematical and scientific notation):
|
|
236
|
+
- LATEX: LaTeX markup language for scientific documents
|
|
237
|
+
- MATHML: MathML (Mathematical Markup Language)
|
|
238
|
+
- ASCIIMATH: AsciiMath notation
|
|
239
|
+
- MATHJAX: MathJax JavaScript display engine
|
|
240
|
+
- KATEX: KaTeX JavaScript display engine
|
|
241
|
+
- SYMPY: SymPy symbolic mathematics language
|
|
225
242
|
"""
|
|
226
243
|
# Special values
|
|
227
244
|
UNKNOWN = "UNKNOWN"
|
|
@@ -295,12 +312,43 @@ class LanguageEnum(str, ComparableEnum):
|
|
|
295
312
|
# Additional languages
|
|
296
313
|
ONEC = "1C" # 1С programming language
|
|
297
314
|
|
|
315
|
+
# Formula languages
|
|
316
|
+
LATEX = "LaTeX" # LaTeX markup language
|
|
317
|
+
MATHML = "MathML" # Mathematical Markup Language
|
|
318
|
+
ASCIIMATH = "AsciiMath" # AsciiMath notation
|
|
319
|
+
MATHJAX = "MathJax" # MathJax JavaScript engine
|
|
320
|
+
KATEX = "KaTeX" # KaTeX JavaScript engine
|
|
321
|
+
SYMPY = "SymPy" # SymPy symbolic mathematics
|
|
322
|
+
|
|
298
323
|
@classmethod
|
|
299
324
|
def from_string(cls, value: str) -> Optional['LanguageEnum']:
|
|
300
325
|
if not isinstance(value, str):
|
|
301
326
|
return None
|
|
327
|
+
|
|
328
|
+
# Normalize input value
|
|
329
|
+
normalized_value = value.lower().strip()
|
|
330
|
+
|
|
331
|
+
# Special handling for C++ variants
|
|
332
|
+
if normalized_value in ['c++', 'cpp']:
|
|
333
|
+
return cls.CPP
|
|
334
|
+
if normalized_value in ['js', 'javascript']:
|
|
335
|
+
return cls.JAVASCRIPT
|
|
336
|
+
if normalized_value in ['py', 'python']:
|
|
337
|
+
return cls.PYTHON
|
|
338
|
+
if normalized_value in ['latex', 'tex']:
|
|
339
|
+
return cls.LATEX
|
|
340
|
+
if normalized_value in ['mathml']:
|
|
341
|
+
return cls.MATHML
|
|
342
|
+
if normalized_value in ['asciimath']:
|
|
343
|
+
return cls.ASCIIMATH
|
|
344
|
+
if normalized_value in ['mathjax']:
|
|
345
|
+
return cls.MATHJAX
|
|
346
|
+
if normalized_value in ['katex']:
|
|
347
|
+
return cls.KATEX
|
|
348
|
+
if normalized_value in ['sympy']:
|
|
349
|
+
return cls.SYMPY
|
|
302
350
|
for member in cls:
|
|
303
|
-
if member.value.lower() ==
|
|
351
|
+
if member.value.lower() == normalized_value or member.name.lower() == normalized_value:
|
|
304
352
|
return member
|
|
305
353
|
return None
|
|
306
354
|
|
|
@@ -316,15 +364,15 @@ class LanguageEnum(str, ComparableEnum):
|
|
|
316
364
|
|
|
317
365
|
@classmethod
|
|
318
366
|
def is_programming_language(cls, value: Union[str, 'LanguageEnum']) -> bool:
|
|
319
|
-
"""Check if the given value represents a programming language."""
|
|
367
|
+
"""Check if the given value represents a programming or formula language."""
|
|
320
368
|
if isinstance(value, str):
|
|
321
369
|
enum_value = cls.from_string(value)
|
|
322
370
|
else:
|
|
323
371
|
enum_value = value
|
|
324
|
-
|
|
372
|
+
|
|
325
373
|
if enum_value is None:
|
|
326
374
|
return False
|
|
327
|
-
|
|
328
|
-
# Natural languages and special values are not programming languages
|
|
375
|
+
|
|
376
|
+
# Natural languages and special values are not programming or formula languages
|
|
329
377
|
natural_languages = {cls.UNKNOWN, cls.EN, cls.RU, cls.UK, cls.DE, cls.FR, cls.ES, cls.ZH, cls.JA}
|
|
330
378
|
return enum_value not in natural_languages
|
{chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/chunk_metadata_adapter/examples.py
RENAMED
|
@@ -83,7 +83,6 @@ def example_structured_chunk():
|
|
|
83
83
|
subtask_id="DOC-123-A",
|
|
84
84
|
tags=["introduction", "documentation", "overview"],
|
|
85
85
|
links=[f"parent:{str(uuid.uuid4())}"],
|
|
86
|
-
chunks=["# Introduction", "This is the documentation", "for the system."],
|
|
87
86
|
start=0,
|
|
88
87
|
end=56,
|
|
89
88
|
category="документация",
|
|
@@ -190,42 +189,6 @@ def example_chain_processing():
|
|
|
190
189
|
return chunks
|
|
191
190
|
|
|
192
191
|
|
|
193
|
-
def example_vectorization_chunks():
|
|
194
|
-
"""Example showing how chunks field is used after vectorization."""
|
|
195
|
-
builder = ChunkMetadataBuilder(project="VectorizationExample")
|
|
196
|
-
|
|
197
|
-
# Original text before vectorization
|
|
198
|
-
original_text = "Machine learning is a subset of artificial intelligence. It enables computers to learn without being explicitly programmed."
|
|
199
|
-
|
|
200
|
-
# After vectorization, the text is split into chunks
|
|
201
|
-
text_chunks = [
|
|
202
|
-
"Machine learning is a subset of artificial intelligence.",
|
|
203
|
-
"It enables computers to learn without being explicitly programmed."
|
|
204
|
-
]
|
|
205
|
-
|
|
206
|
-
# Create chunk with vectorization results
|
|
207
|
-
chunk = builder.build_semantic_chunk(
|
|
208
|
-
body=original_text,
|
|
209
|
-
text=original_text,
|
|
210
|
-
language=LanguageEnum.EN,
|
|
211
|
-
chunk_type=ChunkType.DOC_BLOCK,
|
|
212
|
-
source_id=str(uuid.uuid4()),
|
|
213
|
-
summary="Definition of machine learning",
|
|
214
|
-
chunks=text_chunks, # Text chunks returned after vectorization
|
|
215
|
-
tags=["machine-learning", "ai", "definition"],
|
|
216
|
-
start=0,
|
|
217
|
-
end=len(original_text)
|
|
218
|
-
)
|
|
219
|
-
|
|
220
|
-
# The chunks field contains the text pieces alongside the embedding vector
|
|
221
|
-
print(f"Original text length: {len(original_text)} characters")
|
|
222
|
-
print(f"Split into {len(chunk.chunks)} chunks:")
|
|
223
|
-
for i, text_chunk in enumerate(chunk.chunks):
|
|
224
|
-
print(f" Chunk {i+1}: {text_chunk}")
|
|
225
|
-
|
|
226
|
-
return chunk
|
|
227
|
-
|
|
228
|
-
|
|
229
192
|
def example_is_code_detection():
|
|
230
193
|
"""Example demonstrating automatic is_code field detection."""
|
|
231
194
|
builder = ChunkMetadataBuilder(project="CodeDetectionExample")
|
|
@@ -85,7 +85,6 @@ class ChunkMetadataBuilder:
|
|
|
85
85
|
source_lines_end: Optional[int] = None,
|
|
86
86
|
summary: Optional[str] = None,
|
|
87
87
|
tags: Optional[List[str]] = None,
|
|
88
|
-
chunks: Optional[List[str]] = None,
|
|
89
88
|
role: Optional[Union[str, ChunkRole]] = None,
|
|
90
89
|
task_id: Optional[str] = None,
|
|
91
90
|
subtask_id: Optional[str] = None,
|
|
@@ -182,16 +181,11 @@ class ChunkMetadataBuilder:
|
|
|
182
181
|
elif body in (None, "") and text not in (None, ""):
|
|
183
182
|
body = text
|
|
184
183
|
|
|
185
|
-
# tags
|
|
184
|
+
# tags should be a list of strings
|
|
186
185
|
if tags is not None and not isinstance(tags, list):
|
|
187
186
|
raise ValueError(f"tags must be a list of strings or None, got: {tags.__class__.__name__}")
|
|
188
187
|
if tags:
|
|
189
188
|
tags = list_to_str(tags, separator=',', allow_none=True)
|
|
190
|
-
|
|
191
|
-
if chunks is not None and not isinstance(chunks, list):
|
|
192
|
-
raise ValueError(f"chunks must be a list of strings or None, got: {chunks.__class__.__name__}")
|
|
193
|
-
if chunks:
|
|
194
|
-
chunks = list_to_str(chunks, separator=',', allow_none=True)
|
|
195
189
|
|
|
196
190
|
return {
|
|
197
191
|
"uuid": self.generate_uuid(),
|
|
@@ -213,7 +207,6 @@ class ChunkMetadataBuilder:
|
|
|
213
207
|
"unit_id": self.unit_id if self.unit_id else None,
|
|
214
208
|
"created_at": self._get_iso_timestamp(),
|
|
215
209
|
"tags": tags,
|
|
216
|
-
"chunks": chunks,
|
|
217
210
|
"role": role if role else None,
|
|
218
211
|
"link_parent": link_parent,
|
|
219
212
|
"link_related": link_related,
|
|
@@ -255,7 +248,6 @@ class ChunkMetadataBuilder:
|
|
|
255
248
|
subtask_id: Optional[str] = None,
|
|
256
249
|
links: Optional[List[str]] = None,
|
|
257
250
|
tags: Optional[List[str]] = None,
|
|
258
|
-
chunks: Optional[List[str]] = None,
|
|
259
251
|
status: Union[str, ChunkStatus] = ChunkStatus.RAW,
|
|
260
252
|
metrics: Optional[ChunkMetrics] = None,
|
|
261
253
|
quality_score: Optional[float] = None,
|
|
@@ -293,7 +285,6 @@ class ChunkMetadataBuilder:
|
|
|
293
285
|
subtask_id: Subtask identifier
|
|
294
286
|
links: List of links to other chunks (format: "relation:uuid")
|
|
295
287
|
tags: List of tags
|
|
296
|
-
chunks: List of text chunks returned after vectorization alongside embedding
|
|
297
288
|
status: Processing status of the chunk (default: RAW for initial data ingestion)
|
|
298
289
|
The data lifecycle includes these states:
|
|
299
290
|
- RAW: Initial raw data as ingested into the system
|
|
@@ -391,11 +382,6 @@ class ChunkMetadataBuilder:
|
|
|
391
382
|
tags = str_to_list(tags, separator=',', allow_none=True)
|
|
392
383
|
elif not isinstance(tags, list):
|
|
393
384
|
raise ValueError(f"tags must be a list of strings, got: {type(tags)}")
|
|
394
|
-
|
|
395
|
-
if chunks is None:
|
|
396
|
-
chunks = []
|
|
397
|
-
elif not isinstance(chunks, list):
|
|
398
|
-
raise ValueError(f"chunks must be a list of strings, got: {type(chunks)}")
|
|
399
385
|
|
|
400
386
|
return SemanticChunk(
|
|
401
387
|
uuid=chunk_uuid if chunk_uuid is not None else self.generate_uuid(),
|
|
@@ -418,7 +404,6 @@ class ChunkMetadataBuilder:
|
|
|
418
404
|
status=status,
|
|
419
405
|
links=links or [],
|
|
420
406
|
tags=tags,
|
|
421
|
-
chunks=chunks,
|
|
422
407
|
metrics=metrics if metrics is not None else ChunkMetrics(),
|
|
423
408
|
created_at=self._get_iso_timestamp(),
|
|
424
409
|
start=start,
|
|
@@ -129,7 +129,6 @@ class SemanticChunk(BaseModel):
|
|
|
129
129
|
# Коллекционные и бизнес-поля
|
|
130
130
|
tags: Optional[List[str]] = Field(default=None, min_length=0, max_length=32, description="Categorical tags for the chunk.")
|
|
131
131
|
links: Optional[List[str]] = Field(default=None, min_length=0, max_length=32, description="References to other chunks in the format 'relation:uuid'.")
|
|
132
|
-
chunks: Optional[List[str]] = Field(default=None, min_length=0, max_length=100, description="List of text chunks returned after vectorization alongside the embedding vector.")
|
|
133
132
|
block_meta: Optional[dict] = Field(default=None, description="Additional metadata about the block.")
|
|
134
133
|
# Flat-only поля (были только в FlatSemanticChunk)
|
|
135
134
|
tags_flat: Optional[str] = Field(default=None, min_length=0, max_length=1024, description="Comma-separated tags for flat storage.")
|
|
@@ -159,8 +158,6 @@ class SemanticChunk(BaseModel):
|
|
|
159
158
|
return []
|
|
160
159
|
if prop_name == "links":
|
|
161
160
|
return []
|
|
162
|
-
if prop_name == "chunks":
|
|
163
|
-
return []
|
|
164
161
|
raise ValueError(f"No such property: {prop_name}")
|
|
165
162
|
|
|
166
163
|
@property
|
|
@@ -243,7 +240,7 @@ class SemanticChunk(BaseModel):
|
|
|
243
240
|
if "chunking_version" in restored and not isinstance(restored["chunking_version"], str):
|
|
244
241
|
restored["chunking_version"] = str(restored["chunking_version"])
|
|
245
242
|
# Гарантируем правильные типы для списков/массивов
|
|
246
|
-
for field in ["tags", "links", "
|
|
243
|
+
for field in ["tags", "links", "embedding"]:
|
|
247
244
|
val = restored.get(field)
|
|
248
245
|
if isinstance(val, str):
|
|
249
246
|
if val.strip() == "" or val.strip() == "null":
|
|
@@ -254,12 +251,12 @@ class SemanticChunk(BaseModel):
|
|
|
254
251
|
if isinstance(parsed, list):
|
|
255
252
|
restored[field] = parsed
|
|
256
253
|
else:
|
|
257
|
-
if field
|
|
254
|
+
if field == "tags":
|
|
258
255
|
restored[field] = [x.strip() for x in val.split(",") if x.strip()]
|
|
259
256
|
else:
|
|
260
257
|
raise ValueError(f"Field '{field}' must be a list, got: {val}")
|
|
261
258
|
except Exception:
|
|
262
|
-
if field
|
|
259
|
+
if field == "tags":
|
|
263
260
|
restored[field] = [x.strip() for x in val.split(",") if x.strip()]
|
|
264
261
|
else:
|
|
265
262
|
raise ValueError(f"Field '{field}' must be a list, got: {val}")
|
|
@@ -315,8 +312,6 @@ class SemanticChunk(BaseModel):
|
|
|
315
312
|
raise ValueError("tags must be a list for structured metadata")
|
|
316
313
|
if not isinstance(self.links, list):
|
|
317
314
|
raise ValueError("links must be a list for structured metadata")
|
|
318
|
-
if not isinstance(self.chunks, list):
|
|
319
|
-
raise ValueError("chunks must be a list for structured metadata")
|
|
320
315
|
self.__class__.model_validate(self)
|
|
321
316
|
|
|
322
317
|
@model_validator(mode="before")
|
|
@@ -345,7 +340,7 @@ class SemanticChunk(BaseModel):
|
|
|
345
340
|
if not data.get("uuid"):
|
|
346
341
|
data["uuid"] = str(uuid.uuid4())
|
|
347
342
|
# --- Автодесериализация списков/массивов из строк ---
|
|
348
|
-
for field in ["tags", "links", "
|
|
343
|
+
for field in ["tags", "links", "embedding", "source_lines"]:
|
|
349
344
|
val = data.get(field)
|
|
350
345
|
if isinstance(val, str):
|
|
351
346
|
if val.strip() == "" or val.strip() == "null":
|
|
@@ -130,7 +130,6 @@ def test_from_dict_with_validation_allowed_list_fields():
|
|
|
130
130
|
data = {
|
|
131
131
|
"tags": ["tag1", "tag2"], # list allowed
|
|
132
132
|
"links": ["parent:uuid1", "related:uuid2"], # list allowed
|
|
133
|
-
"chunks": ["chunk1", "chunk2"], # list allowed
|
|
134
133
|
"block_meta": {"key": "value"} # dict allowed
|
|
135
134
|
}
|
|
136
135
|
|
|
@@ -140,7 +139,6 @@ def test_from_dict_with_validation_allowed_list_fields():
|
|
|
140
139
|
assert errors is None
|
|
141
140
|
assert query.tags == ["tag1", "tag2"]
|
|
142
141
|
assert query.links == ["parent:uuid1", "related:uuid2"]
|
|
143
|
-
assert query.chunks == ["chunk1", "chunk2"]
|
|
144
142
|
assert query.block_meta == {"key": "value"}
|
|
145
143
|
|
|
146
144
|
|
|
@@ -146,4 +146,31 @@ def test_language_enum_comprehensive_guesslang_coverage():
|
|
|
146
146
|
assert "1C" in additional_languages, "1C language should be present as additional language"
|
|
147
147
|
|
|
148
148
|
# Total should be at least 54 + 1 = 55
|
|
149
|
-
assert len(actual_programming_languages) >= 55, f"Expected at least 55 programming languages, got {len(actual_programming_languages)}"
|
|
149
|
+
assert len(actual_programming_languages) >= 55, f"Expected at least 55 programming languages, got {len(actual_programming_languages)}"
|
|
150
|
+
|
|
151
|
+
def test_language_enum_formula_languages():
|
|
152
|
+
"""Test formula languages in LanguageEnum."""
|
|
153
|
+
formula_languages = [
|
|
154
|
+
("LaTeX", LanguageEnum.LATEX),
|
|
155
|
+
("latex", LanguageEnum.LATEX),
|
|
156
|
+
("tex", LanguageEnum.LATEX),
|
|
157
|
+
("MathML", LanguageEnum.MATHML),
|
|
158
|
+
("mathml", LanguageEnum.MATHML),
|
|
159
|
+
("AsciiMath", LanguageEnum.ASCIIMATH),
|
|
160
|
+
("asciimath", LanguageEnum.ASCIIMATH),
|
|
161
|
+
("MathJax", LanguageEnum.MATHJAX),
|
|
162
|
+
("mathjax", LanguageEnum.MATHJAX),
|
|
163
|
+
("KaTeX", LanguageEnum.KATEX),
|
|
164
|
+
("katex", LanguageEnum.KATEX),
|
|
165
|
+
("SymPy", LanguageEnum.SYMPY),
|
|
166
|
+
("sympy", LanguageEnum.SYMPY),
|
|
167
|
+
]
|
|
168
|
+
for input_str, expected_enum in formula_languages:
|
|
169
|
+
enum_val = LanguageEnum.from_string(input_str)
|
|
170
|
+
assert enum_val == expected_enum, f"{input_str} should map to {expected_enum}"
|
|
171
|
+
assert expected_enum.eqstr(input_str) is True
|
|
172
|
+
assert LanguageEnum.is_programming_language(enum_val) is True
|
|
173
|
+
assert LanguageEnum.is_programming_language(enum_val.value) is True
|
|
174
|
+
# They are not natural languages
|
|
175
|
+
natural_languages = {LanguageEnum.UNKNOWN, LanguageEnum.EN, LanguageEnum.RU, LanguageEnum.UK, LanguageEnum.DE, LanguageEnum.FR, LanguageEnum.ES, LanguageEnum.ZH, LanguageEnum.JA}
|
|
176
|
+
assert enum_val not in natural_languages
|
|
@@ -18,8 +18,7 @@ from chunk_metadata_adapter.examples import (
|
|
|
18
18
|
example_data_lifecycle,
|
|
19
19
|
example_metrics_extension,
|
|
20
20
|
example_full_chain_structured_semantic_flat,
|
|
21
|
-
example_is_code_detection
|
|
22
|
-
example_vectorization_chunks
|
|
21
|
+
example_is_code_detection
|
|
23
22
|
)
|
|
24
23
|
|
|
25
24
|
|
|
@@ -193,24 +192,4 @@ def test_basic_flat_metadata_includes_is_code():
|
|
|
193
192
|
# Should include is_code_chunk field
|
|
194
193
|
assert "is_code_chunk" in metadata
|
|
195
194
|
# Should be True for Python code
|
|
196
|
-
assert metadata["is_code_chunk"] == "true" # String in flat format
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
def test_example_vectorization_chunks():
|
|
200
|
-
"""Test example_vectorization_chunks functionality."""
|
|
201
|
-
chunk = example_vectorization_chunks()
|
|
202
|
-
|
|
203
|
-
# Verify chunks field exists and has correct content
|
|
204
|
-
assert chunk.chunks is not None
|
|
205
|
-
assert isinstance(chunk.chunks, list)
|
|
206
|
-
assert len(chunk.chunks) == 2
|
|
207
|
-
assert chunk.chunks[0] == "Machine learning is a subset of artificial intelligence."
|
|
208
|
-
assert chunk.chunks[1] == "It enables computers to learn without being explicitly programmed."
|
|
209
|
-
|
|
210
|
-
# Verify other fields
|
|
211
|
-
assert chunk.body == chunk.text # Should be the same
|
|
212
|
-
assert "Machine learning" in chunk.body
|
|
213
|
-
assert chunk.tags is not None
|
|
214
|
-
assert "machine-learning" in chunk.tags
|
|
215
|
-
|
|
216
|
-
print("✓ example_vectorization_chunks works correctly")
|
|
195
|
+
assert metadata["is_code_chunk"] == "true" # String in flat format
|
|
@@ -332,4 +332,34 @@ class TestLanguageEnumPerformance:
|
|
|
332
332
|
|
|
333
333
|
# Should complete in reasonable time (less than 0.5 seconds for 5000 calls)
|
|
334
334
|
elapsed = end_time - start_time
|
|
335
|
-
assert elapsed < 0.5, f"is_code method took too long: {elapsed:.3f} seconds"
|
|
335
|
+
assert elapsed < 0.5, f"is_code method took too long: {elapsed:.3f} seconds"
|
|
336
|
+
|
|
337
|
+
def test_formula_languages_integration(self):
|
|
338
|
+
"""Test SemanticChunk and builder with formula languages (LaTeX, MathML, AsciiMath, MathJax, KaTeX, SymPy)."""
|
|
339
|
+
builder = ChunkMetadataBuilder()
|
|
340
|
+
source_id = str(uuid.uuid4())
|
|
341
|
+
formula_cases = [
|
|
342
|
+
(LanguageEnum.LATEX, r"\\frac{a}{b} = c"),
|
|
343
|
+
(LanguageEnum.MATHML, "<math><mfrac><mi>a</mi><mi>b</mi></mfrac></math>"),
|
|
344
|
+
(LanguageEnum.ASCIIMATH, "a/b = c"),
|
|
345
|
+
(LanguageEnum.MATHJAX, r"\\(a^2 + b^2 = c^2\\)"),
|
|
346
|
+
(LanguageEnum.KATEX, r"c = \sqrt{a^2 + b^2}"),
|
|
347
|
+
(LanguageEnum.SYMPY, "Eq(a/b, c)"),
|
|
348
|
+
]
|
|
349
|
+
for language, formula in formula_cases:
|
|
350
|
+
chunk = builder.build_semantic_chunk(
|
|
351
|
+
body=formula,
|
|
352
|
+
language=language,
|
|
353
|
+
chunk_type=ChunkType.DOC_BLOCK,
|
|
354
|
+
source_id=source_id,
|
|
355
|
+
start=0,
|
|
356
|
+
end=len(formula)
|
|
357
|
+
)
|
|
358
|
+
assert chunk.language == language
|
|
359
|
+
assert chunk.is_code() is True, f"{language} should be detected as code"
|
|
360
|
+
assert LanguageEnum.is_programming_language(chunk.language) is True
|
|
361
|
+
# Test round-trip serialization
|
|
362
|
+
flat_dict = chunk.to_flat_dict()
|
|
363
|
+
restored_chunk = SemanticChunk.from_flat_dict(flat_dict)
|
|
364
|
+
assert restored_chunk.language == language
|
|
365
|
+
assert restored_chunk.is_code() is True
|
{chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/tests/test_metadata_builder.py
RENAMED
|
@@ -635,425 +635,4 @@ def test_tokens_flat_and_semantic():
|
|
|
635
635
|
assert flat["tokens"] == 123
|
|
636
636
|
# flat -> semantic
|
|
637
637
|
restored = builder.flat_to_semantic(flat)
|
|
638
|
-
assert restored.metrics.tokens == 123
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
def test_build_flat_metadata_invalid_source_id():
|
|
642
|
-
"""Test build_flat_metadata with invalid source_id."""
|
|
643
|
-
builder = ChunkMetadataBuilder()
|
|
644
|
-
|
|
645
|
-
# Invalid source_id (not UUID)
|
|
646
|
-
with pytest.raises(ValueError, match="badly formed hexadecimal UUID string"):
|
|
647
|
-
builder.build_flat_metadata(
|
|
648
|
-
body="test",
|
|
649
|
-
source_id="invalid-uuid",
|
|
650
|
-
ordinal=0,
|
|
651
|
-
type=ChunkType.DRAFT,
|
|
652
|
-
language="en"
|
|
653
|
-
)
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
def test_build_flat_metadata_invalid_link_parent():
|
|
657
|
-
"""Test build_flat_metadata with invalid link_parent."""
|
|
658
|
-
builder = ChunkMetadataBuilder()
|
|
659
|
-
|
|
660
|
-
with pytest.raises(ValueError, match="badly formed hexadecimal UUID string"):
|
|
661
|
-
builder.build_flat_metadata(
|
|
662
|
-
body="test",
|
|
663
|
-
source_id=str(uuid.uuid4()),
|
|
664
|
-
ordinal=0,
|
|
665
|
-
type=ChunkType.DRAFT,
|
|
666
|
-
language="en",
|
|
667
|
-
link_parent="invalid-uuid"
|
|
668
|
-
)
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
def test_build_flat_metadata_invalid_link_related():
|
|
672
|
-
"""Test build_flat_metadata with invalid link_related."""
|
|
673
|
-
builder = ChunkMetadataBuilder()
|
|
674
|
-
|
|
675
|
-
with pytest.raises(ValueError, match="badly formed hexadecimal UUID string"):
|
|
676
|
-
builder.build_flat_metadata(
|
|
677
|
-
body="test",
|
|
678
|
-
source_id=str(uuid.uuid4()),
|
|
679
|
-
ordinal=0,
|
|
680
|
-
type=ChunkType.DRAFT,
|
|
681
|
-
language="en",
|
|
682
|
-
link_related="invalid-uuid"
|
|
683
|
-
)
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
def test_build_flat_metadata_invalid_coverage():
|
|
687
|
-
"""Test build_flat_metadata with invalid coverage values."""
|
|
688
|
-
builder = ChunkMetadataBuilder()
|
|
689
|
-
source_id = str(uuid.uuid4())
|
|
690
|
-
|
|
691
|
-
# Coverage not a float
|
|
692
|
-
with pytest.raises(ValueError, match="coverage must be a float"):
|
|
693
|
-
builder.build_flat_metadata(
|
|
694
|
-
body="test",
|
|
695
|
-
source_id=source_id,
|
|
696
|
-
ordinal=0,
|
|
697
|
-
type=ChunkType.DRAFT,
|
|
698
|
-
language="en",
|
|
699
|
-
coverage="invalid"
|
|
700
|
-
)
|
|
701
|
-
|
|
702
|
-
# Coverage out of range
|
|
703
|
-
with pytest.raises(ValueError, match="coverage must be in \\[0, 1\\]"):
|
|
704
|
-
builder.build_flat_metadata(
|
|
705
|
-
body="test",
|
|
706
|
-
source_id=source_id,
|
|
707
|
-
ordinal=0,
|
|
708
|
-
type=ChunkType.DRAFT,
|
|
709
|
-
language="en",
|
|
710
|
-
coverage=1.5
|
|
711
|
-
)
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
def test_build_flat_metadata_invalid_tags():
|
|
715
|
-
"""Test build_flat_metadata with invalid tags."""
|
|
716
|
-
builder = ChunkMetadataBuilder()
|
|
717
|
-
|
|
718
|
-
with pytest.raises(ValueError, match="tags must be a list of strings"):
|
|
719
|
-
builder.build_flat_metadata(
|
|
720
|
-
body="test",
|
|
721
|
-
source_id=str(uuid.uuid4()),
|
|
722
|
-
ordinal=0,
|
|
723
|
-
type=ChunkType.DRAFT,
|
|
724
|
-
language="en",
|
|
725
|
-
tags="not-a-list"
|
|
726
|
-
)
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
def test_build_flat_metadata_invalid_chunks():
|
|
730
|
-
"""Test build_flat_metadata with invalid chunks."""
|
|
731
|
-
builder = ChunkMetadataBuilder()
|
|
732
|
-
|
|
733
|
-
with pytest.raises(ValueError, match="chunks must be a list of strings"):
|
|
734
|
-
builder.build_flat_metadata(
|
|
735
|
-
body="test",
|
|
736
|
-
source_id=str(uuid.uuid4()),
|
|
737
|
-
ordinal=0,
|
|
738
|
-
type=ChunkType.DRAFT,
|
|
739
|
-
language="en",
|
|
740
|
-
chunks="not-a-list"
|
|
741
|
-
)
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
def test_build_flat_metadata_enum_conversions():
|
|
745
|
-
"""Test build_flat_metadata with enum conversions."""
|
|
746
|
-
builder = ChunkMetadataBuilder()
|
|
747
|
-
source_id = str(uuid.uuid4())
|
|
748
|
-
|
|
749
|
-
# Test with enum instances
|
|
750
|
-
metadata = builder.build_flat_metadata(
|
|
751
|
-
body="test",
|
|
752
|
-
source_id=source_id,
|
|
753
|
-
ordinal=0,
|
|
754
|
-
type=ChunkType.CODE_BLOCK, # Enum instance
|
|
755
|
-
language="Python",
|
|
756
|
-
role=ChunkRole.DEVELOPER, # Enum instance
|
|
757
|
-
status=ChunkStatus.VERIFIED # Enum instance
|
|
758
|
-
)
|
|
759
|
-
|
|
760
|
-
assert metadata["type"] == "CodeBlock"
|
|
761
|
-
assert metadata["role"] == "developer"
|
|
762
|
-
assert metadata["status"] == "verified"
|
|
763
|
-
assert metadata["is_code_chunk"] == "true" # Should be detected as code
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
def test_build_flat_metadata_text_body_normalization():
|
|
767
|
-
"""Test build_flat_metadata text/body normalization."""
|
|
768
|
-
builder = ChunkMetadataBuilder()
|
|
769
|
-
source_id = str(uuid.uuid4())
|
|
770
|
-
|
|
771
|
-
# Test text=None, body provided
|
|
772
|
-
metadata1 = builder.build_flat_metadata(
|
|
773
|
-
body="test content",
|
|
774
|
-
text=None,
|
|
775
|
-
source_id=source_id,
|
|
776
|
-
ordinal=0,
|
|
777
|
-
type=ChunkType.DRAFT,
|
|
778
|
-
language="en"
|
|
779
|
-
)
|
|
780
|
-
assert metadata1["text"] == "test content"
|
|
781
|
-
|
|
782
|
-
# Test body=None, text provided
|
|
783
|
-
metadata2 = builder.build_flat_metadata(
|
|
784
|
-
body=None,
|
|
785
|
-
text="test content",
|
|
786
|
-
source_id=source_id,
|
|
787
|
-
ordinal=0,
|
|
788
|
-
type=ChunkType.DRAFT,
|
|
789
|
-
language="en"
|
|
790
|
-
)
|
|
791
|
-
assert metadata2["body"] == "test content"
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
def test_build_semantic_chunk_invalid_tags():
|
|
795
|
-
"""Test build_semantic_chunk with invalid tags."""
|
|
796
|
-
builder = ChunkMetadataBuilder()
|
|
797
|
-
|
|
798
|
-
with pytest.raises(ValueError, match="tags must be a list of strings"):
|
|
799
|
-
builder.build_semantic_chunk(
|
|
800
|
-
body="test",
|
|
801
|
-
language="en",
|
|
802
|
-
chunk_type=ChunkType.DRAFT,
|
|
803
|
-
tags="not-a-list"
|
|
804
|
-
)
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
def test_build_semantic_chunk_invalid_links():
|
|
808
|
-
"""Test build_semantic_chunk with invalid links."""
|
|
809
|
-
builder = ChunkMetadataBuilder()
|
|
810
|
-
|
|
811
|
-
with pytest.raises(ValueError, match="links must be a list of strings"):
|
|
812
|
-
builder.build_semantic_chunk(
|
|
813
|
-
body="test",
|
|
814
|
-
language="en",
|
|
815
|
-
chunk_type=ChunkType.DRAFT,
|
|
816
|
-
links="not-a-list"
|
|
817
|
-
)
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
def test_build_semantic_chunk_invalid_source_id():
|
|
821
|
-
"""Test build_semantic_chunk with invalid source_id."""
|
|
822
|
-
builder = ChunkMetadataBuilder()
|
|
823
|
-
|
|
824
|
-
with pytest.raises(ValueError, match="badly formed hexadecimal UUID string"):
|
|
825
|
-
builder.build_semantic_chunk(
|
|
826
|
-
body="test",
|
|
827
|
-
language="en",
|
|
828
|
-
chunk_type=ChunkType.DRAFT,
|
|
829
|
-
source_id="invalid-uuid"
|
|
830
|
-
)
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
def test_build_semantic_chunk_invalid_link_format():
|
|
834
|
-
"""Test build_semantic_chunk with invalid link format."""
|
|
835
|
-
builder = ChunkMetadataBuilder()
|
|
836
|
-
|
|
837
|
-
with pytest.raises(ValueError, match="Link must follow 'relation:uuid' format"):
|
|
838
|
-
builder.build_semantic_chunk(
|
|
839
|
-
body="test",
|
|
840
|
-
language="en",
|
|
841
|
-
chunk_type=ChunkType.DRAFT,
|
|
842
|
-
links=["invalid-format"]
|
|
843
|
-
)
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
def test_build_semantic_chunk_invalid_link_uuid():
|
|
847
|
-
"""Test build_semantic_chunk with invalid UUID in link."""
|
|
848
|
-
builder = ChunkMetadataBuilder()
|
|
849
|
-
|
|
850
|
-
with pytest.raises(ValueError, match="Invalid UUID4 in link"):
|
|
851
|
-
builder.build_semantic_chunk(
|
|
852
|
-
body="test",
|
|
853
|
-
language="en",
|
|
854
|
-
chunk_type=ChunkType.DRAFT,
|
|
855
|
-
links=["parent:invalid-uuid"]
|
|
856
|
-
)
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
def test_build_semantic_chunk_enum_conversions():
|
|
860
|
-
"""Test build_semantic_chunk with enum conversions."""
|
|
861
|
-
builder = ChunkMetadataBuilder()
|
|
862
|
-
|
|
863
|
-
# Test string to enum conversion
|
|
864
|
-
chunk = builder.build_semantic_chunk(
|
|
865
|
-
body="test",
|
|
866
|
-
language="Python",
|
|
867
|
-
chunk_type=ChunkType.CODE_BLOCK, # Use enum directly
|
|
868
|
-
role=ChunkRole.DEVELOPER, # Use enum directly
|
|
869
|
-
status=ChunkStatus.VERIFIED # Use enum directly
|
|
870
|
-
)
|
|
871
|
-
|
|
872
|
-
assert chunk.type == ChunkType.CODE_BLOCK
|
|
873
|
-
assert chunk.role == ChunkRole.DEVELOPER
|
|
874
|
-
assert chunk.status == ChunkStatus.VERIFIED
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
def test_build_semantic_chunk_metrics_handling():
|
|
878
|
-
"""Test build_semantic_chunk metrics handling."""
|
|
879
|
-
builder = ChunkMetadataBuilder()
|
|
880
|
-
|
|
881
|
-
# Test with explicit metrics
|
|
882
|
-
custom_metrics = ChunkMetrics(quality_score=0.9)
|
|
883
|
-
chunk1 = builder.build_semantic_chunk(
|
|
884
|
-
body="test",
|
|
885
|
-
language="en",
|
|
886
|
-
chunk_type=ChunkType.DRAFT,
|
|
887
|
-
metrics=custom_metrics,
|
|
888
|
-
tokens=100
|
|
889
|
-
)
|
|
890
|
-
assert chunk1.metrics.quality_score == 0.9
|
|
891
|
-
assert chunk1.metrics.tokens == 100 # Should be updated
|
|
892
|
-
|
|
893
|
-
# Test with None metrics and individual values
|
|
894
|
-
chunk2 = builder.build_semantic_chunk(
|
|
895
|
-
body="test",
|
|
896
|
-
language="en",
|
|
897
|
-
chunk_type=ChunkType.DRAFT,
|
|
898
|
-
quality_score=0.8,
|
|
899
|
-
coverage=0.7,
|
|
900
|
-
tokens=50
|
|
901
|
-
)
|
|
902
|
-
assert chunk2.metrics.quality_score == 0.8
|
|
903
|
-
assert chunk2.metrics.coverage == 0.7
|
|
904
|
-
assert chunk2.metrics.tokens == 50
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
def test_build_semantic_chunk_string_validation():
|
|
908
|
-
"""Test build_semantic_chunk string field validation."""
|
|
909
|
-
builder = ChunkMetadataBuilder(project=None)
|
|
910
|
-
|
|
911
|
-
chunk = builder.build_semantic_chunk(
|
|
912
|
-
body="", # Empty body
|
|
913
|
-
text="test",
|
|
914
|
-
language="en",
|
|
915
|
-
chunk_type=ChunkType.DRAFT,
|
|
916
|
-
summary="", # Empty summary
|
|
917
|
-
source_path="" # Empty source_path
|
|
918
|
-
)
|
|
919
|
-
|
|
920
|
-
# Empty strings should be replaced with minimum length strings
|
|
921
|
-
assert len(chunk.body) >= 1
|
|
922
|
-
assert len(chunk.summary) >= 1
|
|
923
|
-
assert len(chunk.source_path) >= 1
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
def test_build_semantic_chunk_uuid_validation():
|
|
927
|
-
"""Test build_semantic_chunk UUID validation and generation."""
|
|
928
|
-
builder = ChunkMetadataBuilder(unit_id="invalid-uuid") # Invalid unit_id
|
|
929
|
-
|
|
930
|
-
chunk = builder.build_semantic_chunk(
|
|
931
|
-
body="test",
|
|
932
|
-
language="en",
|
|
933
|
-
chunk_type=ChunkType.DRAFT,
|
|
934
|
-
task_id="invalid-task-uuid", # Invalid task_id
|
|
935
|
-
subtask_id="invalid-subtask-uuid" # Invalid subtask_id
|
|
936
|
-
)
|
|
937
|
-
|
|
938
|
-
# Invalid UUIDs should be replaced with valid ones
|
|
939
|
-
import re
|
|
940
|
-
UUID_PATTERN = re.compile(r'^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$', re.IGNORECASE)
|
|
941
|
-
assert UUID_PATTERN.match(chunk.unit_id)
|
|
942
|
-
assert UUID_PATTERN.match(chunk.task_id)
|
|
943
|
-
assert UUID_PATTERN.match(chunk.subtask_id)
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
def test_build_semantic_chunk_tags_string_conversion():
|
|
947
|
-
"""Test build_semantic_chunk tags string conversion."""
|
|
948
|
-
# This test is for build_semantic_chunk which expects list, not string
|
|
949
|
-
# String conversion happens in build_flat_metadata
|
|
950
|
-
builder = ChunkMetadataBuilder()
|
|
951
|
-
|
|
952
|
-
# Test that string tags are converted in flat metadata
|
|
953
|
-
metadata = builder.build_flat_metadata(
|
|
954
|
-
body="test",
|
|
955
|
-
source_id=str(uuid.uuid4()),
|
|
956
|
-
ordinal=0,
|
|
957
|
-
type=ChunkType.DRAFT,
|
|
958
|
-
language="en",
|
|
959
|
-
tags=["tag1", "tag2", "tag3"] # List tags should be converted to string
|
|
960
|
-
)
|
|
961
|
-
|
|
962
|
-
assert "tags" in metadata
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
def test_build_semantic_chunk_chunks_validation():
|
|
966
|
-
"""Test build_semantic_chunk chunks validation."""
|
|
967
|
-
builder = ChunkMetadataBuilder()
|
|
968
|
-
|
|
969
|
-
with pytest.raises(ValueError, match="chunks must be a list of strings"):
|
|
970
|
-
builder.build_semantic_chunk(
|
|
971
|
-
body="test",
|
|
972
|
-
language="en",
|
|
973
|
-
chunk_type=ChunkType.DRAFT,
|
|
974
|
-
chunks="not-a-list"
|
|
975
|
-
)
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
def test_semantic_to_flat_link_extraction():
|
|
979
|
-
"""Test semantic_to_flat link extraction."""
|
|
980
|
-
builder = ChunkMetadataBuilder()
|
|
981
|
-
|
|
982
|
-
# Create chunk with parent and related links
|
|
983
|
-
chunk = builder.build_semantic_chunk(
|
|
984
|
-
body="test",
|
|
985
|
-
language="en",
|
|
986
|
-
chunk_type=ChunkType.DRAFT,
|
|
987
|
-
links=[
|
|
988
|
-
f"parent:{str(uuid.uuid4())}",
|
|
989
|
-
f"related:{str(uuid.uuid4())}"
|
|
990
|
-
]
|
|
991
|
-
)
|
|
992
|
-
|
|
993
|
-
flat_dict = builder.semantic_to_flat(chunk)
|
|
994
|
-
|
|
995
|
-
assert flat_dict["link_parent"] is not None
|
|
996
|
-
assert flat_dict["link_related"] is not None
|
|
997
|
-
assert flat_dict["tags"] == "" # Empty list converted to empty string
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
def test_semantic_to_flat_no_links():
|
|
1001
|
-
"""Test semantic_to_flat with no links."""
|
|
1002
|
-
builder = ChunkMetadataBuilder()
|
|
1003
|
-
|
|
1004
|
-
chunk = builder.build_semantic_chunk(
|
|
1005
|
-
body="test",
|
|
1006
|
-
language="en",
|
|
1007
|
-
chunk_type=ChunkType.DRAFT,
|
|
1008
|
-
links=[]
|
|
1009
|
-
)
|
|
1010
|
-
|
|
1011
|
-
flat_dict = builder.semantic_to_flat(chunk)
|
|
1012
|
-
|
|
1013
|
-
assert flat_dict["link_parent"] is None
|
|
1014
|
-
assert flat_dict["link_related"] is None
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
def test_wrapper_methods():
|
|
1018
|
-
"""Test wrapper methods for conversion."""
|
|
1019
|
-
builder = ChunkMetadataBuilder()
|
|
1020
|
-
|
|
1021
|
-
# Create a chunk
|
|
1022
|
-
chunk = builder.build_semantic_chunk(
|
|
1023
|
-
body="test",
|
|
1024
|
-
language="en",
|
|
1025
|
-
chunk_type=ChunkType.DRAFT,
|
|
1026
|
-
tags=["test"]
|
|
1027
|
-
)
|
|
1028
|
-
|
|
1029
|
-
# Test all wrapper methods
|
|
1030
|
-
flat_dict = {"body": "test", "type": "Draft", "language": "en"}
|
|
1031
|
-
|
|
1032
|
-
# flat_to_semantic
|
|
1033
|
-
restored_chunk = builder.flat_to_semantic(flat_dict)
|
|
1034
|
-
assert restored_chunk.body == "test"
|
|
1035
|
-
|
|
1036
|
-
# semantic_to_json_dict
|
|
1037
|
-
json_dict = builder.semantic_to_json_dict(chunk)
|
|
1038
|
-
assert json_dict["body"] == "test"
|
|
1039
|
-
|
|
1040
|
-
# json_dict_to_semantic
|
|
1041
|
-
restored_from_json = builder.json_dict_to_semantic(json_dict)
|
|
1042
|
-
assert restored_from_json.body == "test"
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
def test_build_metadata_alias():
|
|
1046
|
-
"""Test build_metadata alias for backward compatibility."""
|
|
1047
|
-
builder = ChunkMetadataBuilder()
|
|
1048
|
-
|
|
1049
|
-
# build_metadata should work the same as build_flat_metadata
|
|
1050
|
-
metadata = builder.build_metadata(
|
|
1051
|
-
body="test",
|
|
1052
|
-
source_id=str(uuid.uuid4()),
|
|
1053
|
-
ordinal=0,
|
|
1054
|
-
type=ChunkType.DRAFT,
|
|
1055
|
-
language="en"
|
|
1056
|
-
)
|
|
1057
|
-
|
|
1058
|
-
assert metadata["body"] == "test"
|
|
1059
|
-
assert metadata["type"] == "Draft"
|
|
638
|
+
assert restored.metrics.tokens == 123
|
{chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/tests/test_round_trip_redis.py
RENAMED
|
@@ -127,12 +127,12 @@ class TestRoundTripRedis:
|
|
|
127
127
|
self._compare_chunks(original_chunk, restored_chunk)
|
|
128
128
|
|
|
129
129
|
def test_tags_and_links_always_lists(self, valid_uuid, sample_text):
|
|
130
|
-
"""Verify tags
|
|
131
|
-
# Test with various tag/link
|
|
130
|
+
"""Verify tags and links are always List[str] regardless of format"""
|
|
131
|
+
# Test with various tag/link formats
|
|
132
132
|
test_cases = [
|
|
133
|
-
{"tags": ["a", "b", "c"], "links": ["rel:uuid1", "rel:uuid2"]
|
|
134
|
-
{"tags": [], "links": []
|
|
135
|
-
{"tags": None, "links": None
|
|
133
|
+
{"tags": ["a", "b", "c"], "links": ["rel:uuid1", "rel:uuid2"]},
|
|
134
|
+
{"tags": [], "links": []},
|
|
135
|
+
{"tags": None, "links": None},
|
|
136
136
|
]
|
|
137
137
|
|
|
138
138
|
for case in test_cases:
|
|
@@ -149,22 +149,18 @@ class TestRoundTripRedis:
|
|
|
149
149
|
restored_chunk = self._normalize_types(restored_chunk)
|
|
150
150
|
|
|
151
151
|
# Both should have lists (empty if None was provided)
|
|
152
|
-
# Note: SemanticChunk may initialize None tags/links
|
|
152
|
+
# Note: SemanticChunk may initialize None tags/links as None, but from_flat_dict converts to []
|
|
153
153
|
orig_tags = original_chunk.tags if original_chunk.tags is not None else []
|
|
154
154
|
orig_links = original_chunk.links if original_chunk.links is not None else []
|
|
155
|
-
orig_chunks = original_chunk.chunks if original_chunk.chunks is not None else []
|
|
156
155
|
rest_tags = restored_chunk.tags if restored_chunk.tags is not None else []
|
|
157
156
|
rest_links = restored_chunk.links if restored_chunk.links is not None else []
|
|
158
|
-
rest_chunks = restored_chunk.chunks if restored_chunk.chunks is not None else []
|
|
159
157
|
|
|
160
158
|
assert isinstance(rest_tags, list)
|
|
161
159
|
assert isinstance(rest_links, list)
|
|
162
|
-
assert isinstance(rest_chunks, list)
|
|
163
160
|
|
|
164
161
|
# Content should match
|
|
165
162
|
assert orig_tags == rest_tags
|
|
166
163
|
assert orig_links == rest_links
|
|
167
|
-
assert orig_chunks == rest_chunks
|
|
168
164
|
|
|
169
165
|
def _normalize_types(self, chunk: SemanticChunk) -> SemanticChunk:
|
|
170
166
|
"""
|
|
@@ -253,8 +249,8 @@ class TestRoundTripRedis:
|
|
|
253
249
|
orig_val = original_dict[key]
|
|
254
250
|
rest_val = restored_dict.get(key)
|
|
255
251
|
|
|
256
|
-
# Special handling for list fields (tags, links,
|
|
257
|
-
if key in ["tags", "links", "
|
|
252
|
+
# Special handling for list fields (tags, links, embedding)
|
|
253
|
+
if key in ["tags", "links", "embedding"]:
|
|
258
254
|
orig_list = orig_val if orig_val is not None else []
|
|
259
255
|
rest_list = rest_val if rest_val is not None else []
|
|
260
256
|
assert orig_list == rest_list, f"Mismatch in {key}: {orig_list} != {rest_list}"
|
|
@@ -295,7 +291,7 @@ class TestRoundTripRedis:
|
|
|
295
291
|
|
|
296
292
|
# All values should be strings except lists
|
|
297
293
|
for key, value in flat_dict.items():
|
|
298
|
-
if key in ["tags", "links"
|
|
294
|
+
if key in ["tags", "links"]:
|
|
299
295
|
assert isinstance(value, list), f"{key} should be list, got {type(value)}"
|
|
300
296
|
else:
|
|
301
297
|
assert isinstance(value, str), f"{key} should be string, got {type(value)}: {value}"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/chunk_metadata_adapter/utils.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/tests/test_array_validation.py
RENAMED
|
File without changes
|
{chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/tests/test_coverage_improvement.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/tests/test_full_cycle_all_fields.py
RENAMED
|
File without changes
|
{chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/tests/test_full_cycle_integration.py
RENAMED
|
File without changes
|
{chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/tests/test_language_integration.py
RENAMED
|
File without changes
|
|
File without changes
|
{chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/tests/test_metadata_builder_part1.py
RENAMED
|
File without changes
|
{chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/tests/test_metadata_builder_part2.py
RENAMED
|
File without changes
|
{chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/tests/test_metrics_extension.py
RENAMED
|
File without changes
|
|
File without changes
|
{chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/tests/test_query_business_examples.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/tests/test_semantic_chunk_advanced.py
RENAMED
|
File without changes
|
{chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/tests/test_semantic_chunk_basic.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/tests/test_semantic_chunk_edge.py
RENAMED
|
File without changes
|
{chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/tests/test_semantic_chunk_is_code.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|