chunk-metadata-adapter 2.2.0__tar.gz → 2.2.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/PKG-INFO +1 -1
  2. {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/chunk_metadata_adapter/__init__.py +1 -1
  3. {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/chunk_metadata_adapter/chunk_query.py +2 -3
  4. {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/chunk_metadata_adapter/data_types.py +55 -7
  5. {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/chunk_metadata_adapter/examples.py +0 -37
  6. {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/chunk_metadata_adapter/metadata_builder.py +1 -16
  7. {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/chunk_metadata_adapter/semantic_chunk.py +4 -9
  8. {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/chunk_metadata_adapter.egg-info/PKG-INFO +1 -1
  9. {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/pyproject.toml +1 -1
  10. {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/tests/test_chunk_query.py +0 -2
  11. {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/tests/test_data_types.py +28 -1
  12. {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/tests/test_examples.py +2 -23
  13. {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/tests/test_language_enum_integration.py +31 -1
  14. {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/tests/test_metadata_builder.py +1 -422
  15. {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/tests/test_round_trip_redis.py +9 -13
  16. {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/README.md +0 -0
  17. {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/chunk_metadata_adapter/query_business_examples.py +0 -0
  18. {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/chunk_metadata_adapter/query_examples.py +0 -0
  19. {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/chunk_metadata_adapter/query_serialization_examples.py +0 -0
  20. {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/chunk_metadata_adapter/utils.py +0 -0
  21. {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/chunk_metadata_adapter.egg-info/SOURCES.txt +0 -0
  22. {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/chunk_metadata_adapter.egg-info/dependency_links.txt +0 -0
  23. {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/chunk_metadata_adapter.egg-info/requires.txt +0 -0
  24. {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/chunk_metadata_adapter.egg-info/top_level.txt +0 -0
  25. {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/setup.cfg +0 -0
  26. {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/tests/test_array_validation.py +0 -0
  27. {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/tests/test_coverage_improvement.py +0 -0
  28. {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/tests/test_examples_part1.py +0 -0
  29. {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/tests/test_examples_part2.py +0 -0
  30. {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/tests/test_examples_part3.py +0 -0
  31. {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/tests/test_full_cycle_all_fields.py +0 -0
  32. {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/tests/test_full_cycle_integration.py +0 -0
  33. {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/tests/test_language_integration.py +0 -0
  34. {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/tests/test_lifecycle.py +0 -0
  35. {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/tests/test_metadata_builder_part1.py +0 -0
  36. {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/tests/test_metadata_builder_part2.py +0 -0
  37. {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/tests/test_metrics_extension.py +0 -0
  38. {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/tests/test_models_part1.py +0 -0
  39. {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/tests/test_query_business_examples.py +0 -0
  40. {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/tests/test_query_examples.py +0 -0
  41. {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/tests/test_query_serialization_examples.py +0 -0
  42. {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/tests/test_semantic_chunk_advanced.py +0 -0
  43. {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/tests/test_semantic_chunk_basic.py +0 -0
  44. {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/tests/test_semantic_chunk_coverage_part1.py +0 -0
  45. {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/tests/test_semantic_chunk_coverage_part2.py +0 -0
  46. {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/tests/test_semantic_chunk_edge.py +0 -0
  47. {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/tests/test_semantic_chunk_is_code.py +0 -0
  48. {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/tests/test_semantic_chunk_serialization.py +0 -0
  49. {chunk_metadata_adapter-2.2.0 → chunk_metadata_adapter-2.2.2}/tests/test_utils.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: chunk_metadata_adapter
3
- Version: 2.2.0
3
+ Version: 2.2.2
4
4
  Summary: Reusable metadata builder for chunk-based systems
5
5
  Author: SmartAssistant
6
6
  License-Expression: MIT
@@ -20,7 +20,7 @@ from .semantic_chunk import (
20
20
  # Если потребуется фильтр — импортировать так:
21
21
  # from .chunk_query import ChunkQuery
22
22
 
23
- __version__ = "2.2.0"
23
+ __version__ = "2.2.2"
24
24
  __all__ = [
25
25
  "ChunkMetadataBuilder",
26
26
  "ChunkType",
@@ -64,7 +64,6 @@ class ChunkQuery(BaseModel):
64
64
  source_lines_end: Optional[Union[str, int, float, bool, dict, list]] = Field(default=None, description="End line in the source file")
65
65
  tags: Optional[List[str]] = Field(default=None, description="Categorical tags for the chunk.")
66
66
  links: Optional[List[str]] = Field(default=None, description="References to other chunks in the format 'relation:uuid'.")
67
- chunks: Optional[List[str]] = Field(default=None, description="List of text chunks returned after vectorization (search by chunk content).")
68
67
  block_meta: Optional[dict] = Field(default=None, description="Additional metadata about the block.")
69
68
  tags_flat: Optional[str] = Field(default=None, description="Comma-separated tags for flat storage.")
70
69
  link_related: Optional[Union[str, int, float, bool, dict, list]] = Field(default=None, description="Related chunk UUID")
@@ -143,8 +142,8 @@ class ChunkQuery(BaseModel):
143
142
  if field in uuid_fields or field in enum_fields:
144
143
  continue
145
144
  if isinstance(value, (list, dict)):
146
- # Для фильтра допускается list только для tags, links, chunks и dict для block_meta
147
- if field not in ["tags", "links", "chunks", "block_meta"]:
145
+ # Для фильтра допускается list только для tags, links и dict для block_meta
146
+ if field not in ["tags", "links", "block_meta"]:
148
147
  errors.setdefault(field, []).append(f"{field} must be str, int, float, bool or dict (got {type(value).__name__})")
149
148
  elif not isinstance(value, allowed_types):
150
149
  errors.setdefault(field, []).append(f"{field} must be str/int/float/bool/None, got {type(value).__name__}")
@@ -219,9 +219,26 @@ class BlockType(str, ComparableEnum):
219
219
 
220
220
 
221
221
  class LanguageEnum(str, ComparableEnum):
222
- """Programming and natural languages supported by the system.
222
+ """Programming, natural, and formula languages supported by the system.
223
223
 
224
- Includes all 54 programming languages supported by guesslang plus additional languages.
224
+ Includes all 54 programming languages supported by guesslang, natural languages, and formula languages (e.g., LaTeX, MathML, AsciiMath, MathJax, KaTeX, SymPy).
225
+
226
+ Special values:
227
+ - UNKNOWN: Unknown or unspecified language
228
+
229
+ Natural languages:
230
+ - EN, RU, UK, DE, FR, ES, ZH, JA
231
+
232
+ Programming languages:
233
+ - Assembly, Batchfile, C, C#, C++, Clojure, CMake, COBOL, CoffeeScript, CSS, CSV, Dart, DM, Dockerfile, Elixir, Erlang, Fortran, Go, Groovy, Haskell, HTML, INI, Java, JavaScript, JSON, Julia, Kotlin, Lisp, Lua, Makefile, Markdown, Matlab, Objective-C, OCaml, Pascal, Perl, PHP, PowerShell, Prolog, Python, R, Ruby, Rust, Scala, Shell, SQL, Swift, TeX, TOML, TypeScript, Verilog, Visual Basic, XML, YAML, 1C
234
+
235
+ Formula languages (for mathematical and scientific notation):
236
+ - LATEX: LaTeX markup language for scientific documents
237
+ - MATHML: MathML (Mathematical Markup Language)
238
+ - ASCIIMATH: AsciiMath notation
239
+ - MATHJAX: MathJax JavaScript display engine
240
+ - KATEX: KaTeX JavaScript display engine
241
+ - SYMPY: SymPy symbolic mathematics language
225
242
  """
226
243
  # Special values
227
244
  UNKNOWN = "UNKNOWN"
@@ -295,12 +312,43 @@ class LanguageEnum(str, ComparableEnum):
295
312
  # Additional languages
296
313
  ONEC = "1C" # 1С programming language
297
314
 
315
+ # Formula languages
316
+ LATEX = "LaTeX" # LaTeX markup language
317
+ MATHML = "MathML" # Mathematical Markup Language
318
+ ASCIIMATH = "AsciiMath" # AsciiMath notation
319
+ MATHJAX = "MathJax" # MathJax JavaScript engine
320
+ KATEX = "KaTeX" # KaTeX JavaScript engine
321
+ SYMPY = "SymPy" # SymPy symbolic mathematics
322
+
298
323
  @classmethod
299
324
  def from_string(cls, value: str) -> Optional['LanguageEnum']:
300
325
  if not isinstance(value, str):
301
326
  return None
327
+
328
+ # Normalize input value
329
+ normalized_value = value.lower().strip()
330
+
331
+ # Special handling for C++ variants
332
+ if normalized_value in ['c++', 'cpp']:
333
+ return cls.CPP
334
+ if normalized_value in ['js', 'javascript']:
335
+ return cls.JAVASCRIPT
336
+ if normalized_value in ['py', 'python']:
337
+ return cls.PYTHON
338
+ if normalized_value in ['latex', 'tex']:
339
+ return cls.LATEX
340
+ if normalized_value in ['mathml']:
341
+ return cls.MATHML
342
+ if normalized_value in ['asciimath']:
343
+ return cls.ASCIIMATH
344
+ if normalized_value in ['mathjax']:
345
+ return cls.MATHJAX
346
+ if normalized_value in ['katex']:
347
+ return cls.KATEX
348
+ if normalized_value in ['sympy']:
349
+ return cls.SYMPY
302
350
  for member in cls:
303
- if member.value.lower() == value.lower() or member.name.lower() == value.lower():
351
+ if member.value.lower() == normalized_value or member.name.lower() == normalized_value:
304
352
  return member
305
353
  return None
306
354
 
@@ -316,15 +364,15 @@ class LanguageEnum(str, ComparableEnum):
316
364
 
317
365
  @classmethod
318
366
  def is_programming_language(cls, value: Union[str, 'LanguageEnum']) -> bool:
319
- """Check if the given value represents a programming language."""
367
+ """Check if the given value represents a programming or formula language."""
320
368
  if isinstance(value, str):
321
369
  enum_value = cls.from_string(value)
322
370
  else:
323
371
  enum_value = value
324
-
372
+
325
373
  if enum_value is None:
326
374
  return False
327
-
328
- # Natural languages and special values are not programming languages
375
+
376
+ # Natural languages and special values are not programming or formula languages
329
377
  natural_languages = {cls.UNKNOWN, cls.EN, cls.RU, cls.UK, cls.DE, cls.FR, cls.ES, cls.ZH, cls.JA}
330
378
  return enum_value not in natural_languages
@@ -83,7 +83,6 @@ def example_structured_chunk():
83
83
  subtask_id="DOC-123-A",
84
84
  tags=["introduction", "documentation", "overview"],
85
85
  links=[f"parent:{str(uuid.uuid4())}"],
86
- chunks=["# Introduction", "This is the documentation", "for the system."],
87
86
  start=0,
88
87
  end=56,
89
88
  category="документация",
@@ -190,42 +189,6 @@ def example_chain_processing():
190
189
  return chunks
191
190
 
192
191
 
193
- def example_vectorization_chunks():
194
- """Example showing how chunks field is used after vectorization."""
195
- builder = ChunkMetadataBuilder(project="VectorizationExample")
196
-
197
- # Original text before vectorization
198
- original_text = "Machine learning is a subset of artificial intelligence. It enables computers to learn without being explicitly programmed."
199
-
200
- # After vectorization, the text is split into chunks
201
- text_chunks = [
202
- "Machine learning is a subset of artificial intelligence.",
203
- "It enables computers to learn without being explicitly programmed."
204
- ]
205
-
206
- # Create chunk with vectorization results
207
- chunk = builder.build_semantic_chunk(
208
- body=original_text,
209
- text=original_text,
210
- language=LanguageEnum.EN,
211
- chunk_type=ChunkType.DOC_BLOCK,
212
- source_id=str(uuid.uuid4()),
213
- summary="Definition of machine learning",
214
- chunks=text_chunks, # Text chunks returned after vectorization
215
- tags=["machine-learning", "ai", "definition"],
216
- start=0,
217
- end=len(original_text)
218
- )
219
-
220
- # The chunks field contains the text pieces alongside the embedding vector
221
- print(f"Original text length: {len(original_text)} characters")
222
- print(f"Split into {len(chunk.chunks)} chunks:")
223
- for i, text_chunk in enumerate(chunk.chunks):
224
- print(f" Chunk {i+1}: {text_chunk}")
225
-
226
- return chunk
227
-
228
-
229
192
  def example_is_code_detection():
230
193
  """Example demonstrating automatic is_code field detection."""
231
194
  builder = ChunkMetadataBuilder(project="CodeDetectionExample")
@@ -85,7 +85,6 @@ class ChunkMetadataBuilder:
85
85
  source_lines_end: Optional[int] = None,
86
86
  summary: Optional[str] = None,
87
87
  tags: Optional[List[str]] = None,
88
- chunks: Optional[List[str]] = None,
89
88
  role: Optional[Union[str, ChunkRole]] = None,
90
89
  task_id: Optional[str] = None,
91
90
  subtask_id: Optional[str] = None,
@@ -182,16 +181,11 @@ class ChunkMetadataBuilder:
182
181
  elif body in (None, "") and text not in (None, ""):
183
182
  body = text
184
183
 
185
- # tags and chunks should be lists of strings
184
+ # tags should be a list of strings
186
185
  if tags is not None and not isinstance(tags, list):
187
186
  raise ValueError(f"tags must be a list of strings or None, got: {tags.__class__.__name__}")
188
187
  if tags:
189
188
  tags = list_to_str(tags, separator=',', allow_none=True)
190
-
191
- if chunks is not None and not isinstance(chunks, list):
192
- raise ValueError(f"chunks must be a list of strings or None, got: {chunks.__class__.__name__}")
193
- if chunks:
194
- chunks = list_to_str(chunks, separator=',', allow_none=True)
195
189
 
196
190
  return {
197
191
  "uuid": self.generate_uuid(),
@@ -213,7 +207,6 @@ class ChunkMetadataBuilder:
213
207
  "unit_id": self.unit_id if self.unit_id else None,
214
208
  "created_at": self._get_iso_timestamp(),
215
209
  "tags": tags,
216
- "chunks": chunks,
217
210
  "role": role if role else None,
218
211
  "link_parent": link_parent,
219
212
  "link_related": link_related,
@@ -255,7 +248,6 @@ class ChunkMetadataBuilder:
255
248
  subtask_id: Optional[str] = None,
256
249
  links: Optional[List[str]] = None,
257
250
  tags: Optional[List[str]] = None,
258
- chunks: Optional[List[str]] = None,
259
251
  status: Union[str, ChunkStatus] = ChunkStatus.RAW,
260
252
  metrics: Optional[ChunkMetrics] = None,
261
253
  quality_score: Optional[float] = None,
@@ -293,7 +285,6 @@ class ChunkMetadataBuilder:
293
285
  subtask_id: Subtask identifier
294
286
  links: List of links to other chunks (format: "relation:uuid")
295
287
  tags: List of tags
296
- chunks: List of text chunks returned after vectorization alongside embedding
297
288
  status: Processing status of the chunk (default: RAW for initial data ingestion)
298
289
  The data lifecycle includes these states:
299
290
  - RAW: Initial raw data as ingested into the system
@@ -391,11 +382,6 @@ class ChunkMetadataBuilder:
391
382
  tags = str_to_list(tags, separator=',', allow_none=True)
392
383
  elif not isinstance(tags, list):
393
384
  raise ValueError(f"tags must be a list of strings, got: {type(tags)}")
394
-
395
- if chunks is None:
396
- chunks = []
397
- elif not isinstance(chunks, list):
398
- raise ValueError(f"chunks must be a list of strings, got: {type(chunks)}")
399
385
 
400
386
  return SemanticChunk(
401
387
  uuid=chunk_uuid if chunk_uuid is not None else self.generate_uuid(),
@@ -418,7 +404,6 @@ class ChunkMetadataBuilder:
418
404
  status=status,
419
405
  links=links or [],
420
406
  tags=tags,
421
- chunks=chunks,
422
407
  metrics=metrics if metrics is not None else ChunkMetrics(),
423
408
  created_at=self._get_iso_timestamp(),
424
409
  start=start,
@@ -129,7 +129,6 @@ class SemanticChunk(BaseModel):
129
129
  # Коллекционные и бизнес-поля
130
130
  tags: Optional[List[str]] = Field(default=None, min_length=0, max_length=32, description="Categorical tags for the chunk.")
131
131
  links: Optional[List[str]] = Field(default=None, min_length=0, max_length=32, description="References to other chunks in the format 'relation:uuid'.")
132
- chunks: Optional[List[str]] = Field(default=None, min_length=0, max_length=100, description="List of text chunks returned after vectorization alongside the embedding vector.")
133
132
  block_meta: Optional[dict] = Field(default=None, description="Additional metadata about the block.")
134
133
  # Flat-only поля (были только в FlatSemanticChunk)
135
134
  tags_flat: Optional[str] = Field(default=None, min_length=0, max_length=1024, description="Comma-separated tags for flat storage.")
@@ -159,8 +158,6 @@ class SemanticChunk(BaseModel):
159
158
  return []
160
159
  if prop_name == "links":
161
160
  return []
162
- if prop_name == "chunks":
163
- return []
164
161
  raise ValueError(f"No such property: {prop_name}")
165
162
 
166
163
  @property
@@ -243,7 +240,7 @@ class SemanticChunk(BaseModel):
243
240
  if "chunking_version" in restored and not isinstance(restored["chunking_version"], str):
244
241
  restored["chunking_version"] = str(restored["chunking_version"])
245
242
  # Гарантируем правильные типы для списков/массивов
246
- for field in ["tags", "links", "chunks", "embedding"]:
243
+ for field in ["tags", "links", "embedding"]:
247
244
  val = restored.get(field)
248
245
  if isinstance(val, str):
249
246
  if val.strip() == "" or val.strip() == "null":
@@ -254,12 +251,12 @@ class SemanticChunk(BaseModel):
254
251
  if isinstance(parsed, list):
255
252
  restored[field] = parsed
256
253
  else:
257
- if field in ["tags", "chunks"]:
254
+ if field == "tags":
258
255
  restored[field] = [x.strip() for x in val.split(",") if x.strip()]
259
256
  else:
260
257
  raise ValueError(f"Field '{field}' must be a list, got: {val}")
261
258
  except Exception:
262
- if field in ["tags", "chunks"]:
259
+ if field == "tags":
263
260
  restored[field] = [x.strip() for x in val.split(",") if x.strip()]
264
261
  else:
265
262
  raise ValueError(f"Field '{field}' must be a list, got: {val}")
@@ -315,8 +312,6 @@ class SemanticChunk(BaseModel):
315
312
  raise ValueError("tags must be a list for structured metadata")
316
313
  if not isinstance(self.links, list):
317
314
  raise ValueError("links must be a list for structured metadata")
318
- if not isinstance(self.chunks, list):
319
- raise ValueError("chunks must be a list for structured metadata")
320
315
  self.__class__.model_validate(self)
321
316
 
322
317
  @model_validator(mode="before")
@@ -345,7 +340,7 @@ class SemanticChunk(BaseModel):
345
340
  if not data.get("uuid"):
346
341
  data["uuid"] = str(uuid.uuid4())
347
342
  # --- Автодесериализация списков/массивов из строк ---
348
- for field in ["tags", "links", "chunks", "embedding", "source_lines"]:
343
+ for field in ["tags", "links", "embedding", "source_lines"]:
349
344
  val = data.get(field)
350
345
  if isinstance(val, str):
351
346
  if val.strip() == "" or val.strip() == "null":
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: chunk_metadata_adapter
3
- Version: 2.2.0
3
+ Version: 2.2.2
4
4
  Summary: Reusable metadata builder for chunk-based systems
5
5
  Author: SmartAssistant
6
6
  License-Expression: MIT
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "chunk_metadata_adapter"
7
- version = "2.2.0"
7
+ version = "2.2.2"
8
8
  description = "Reusable metadata builder for chunk-based systems"
9
9
  readme = "README.md"
10
10
  authors = [{name = "SmartAssistant"}]
@@ -130,7 +130,6 @@ def test_from_dict_with_validation_allowed_list_fields():
130
130
  data = {
131
131
  "tags": ["tag1", "tag2"], # list allowed
132
132
  "links": ["parent:uuid1", "related:uuid2"], # list allowed
133
- "chunks": ["chunk1", "chunk2"], # list allowed
134
133
  "block_meta": {"key": "value"} # dict allowed
135
134
  }
136
135
 
@@ -140,7 +139,6 @@ def test_from_dict_with_validation_allowed_list_fields():
140
139
  assert errors is None
141
140
  assert query.tags == ["tag1", "tag2"]
142
141
  assert query.links == ["parent:uuid1", "related:uuid2"]
143
- assert query.chunks == ["chunk1", "chunk2"]
144
142
  assert query.block_meta == {"key": "value"}
145
143
 
146
144
 
@@ -146,4 +146,31 @@ def test_language_enum_comprehensive_guesslang_coverage():
146
146
  assert "1C" in additional_languages, "1C language should be present as additional language"
147
147
 
148
148
  # Total should be at least 54 + 1 = 55
149
- assert len(actual_programming_languages) >= 55, f"Expected at least 55 programming languages, got {len(actual_programming_languages)}"
149
+ assert len(actual_programming_languages) >= 55, f"Expected at least 55 programming languages, got {len(actual_programming_languages)}"
150
+
151
+ def test_language_enum_formula_languages():
152
+ """Test formula languages in LanguageEnum."""
153
+ formula_languages = [
154
+ ("LaTeX", LanguageEnum.LATEX),
155
+ ("latex", LanguageEnum.LATEX),
156
+ ("tex", LanguageEnum.LATEX),
157
+ ("MathML", LanguageEnum.MATHML),
158
+ ("mathml", LanguageEnum.MATHML),
159
+ ("AsciiMath", LanguageEnum.ASCIIMATH),
160
+ ("asciimath", LanguageEnum.ASCIIMATH),
161
+ ("MathJax", LanguageEnum.MATHJAX),
162
+ ("mathjax", LanguageEnum.MATHJAX),
163
+ ("KaTeX", LanguageEnum.KATEX),
164
+ ("katex", LanguageEnum.KATEX),
165
+ ("SymPy", LanguageEnum.SYMPY),
166
+ ("sympy", LanguageEnum.SYMPY),
167
+ ]
168
+ for input_str, expected_enum in formula_languages:
169
+ enum_val = LanguageEnum.from_string(input_str)
170
+ assert enum_val == expected_enum, f"{input_str} should map to {expected_enum}"
171
+ assert expected_enum.eqstr(input_str) is True
172
+ assert LanguageEnum.is_programming_language(enum_val) is True
173
+ assert LanguageEnum.is_programming_language(enum_val.value) is True
174
+ # They are not natural languages
175
+ natural_languages = {LanguageEnum.UNKNOWN, LanguageEnum.EN, LanguageEnum.RU, LanguageEnum.UK, LanguageEnum.DE, LanguageEnum.FR, LanguageEnum.ES, LanguageEnum.ZH, LanguageEnum.JA}
176
+ assert enum_val not in natural_languages
@@ -18,8 +18,7 @@ from chunk_metadata_adapter.examples import (
18
18
  example_data_lifecycle,
19
19
  example_metrics_extension,
20
20
  example_full_chain_structured_semantic_flat,
21
- example_is_code_detection,
22
- example_vectorization_chunks
21
+ example_is_code_detection
23
22
  )
24
23
 
25
24
 
@@ -193,24 +192,4 @@ def test_basic_flat_metadata_includes_is_code():
193
192
  # Should include is_code_chunk field
194
193
  assert "is_code_chunk" in metadata
195
194
  # Should be True for Python code
196
- assert metadata["is_code_chunk"] == "true" # String in flat format
197
-
198
-
199
- def test_example_vectorization_chunks():
200
- """Test example_vectorization_chunks functionality."""
201
- chunk = example_vectorization_chunks()
202
-
203
- # Verify chunks field exists and has correct content
204
- assert chunk.chunks is not None
205
- assert isinstance(chunk.chunks, list)
206
- assert len(chunk.chunks) == 2
207
- assert chunk.chunks[0] == "Machine learning is a subset of artificial intelligence."
208
- assert chunk.chunks[1] == "It enables computers to learn without being explicitly programmed."
209
-
210
- # Verify other fields
211
- assert chunk.body == chunk.text # Should be the same
212
- assert "Machine learning" in chunk.body
213
- assert chunk.tags is not None
214
- assert "machine-learning" in chunk.tags
215
-
216
- print("✓ example_vectorization_chunks works correctly")
195
+ assert metadata["is_code_chunk"] == "true" # String in flat format
@@ -332,4 +332,34 @@ class TestLanguageEnumPerformance:
332
332
 
333
333
  # Should complete in reasonable time (less than 0.5 seconds for 5000 calls)
334
334
  elapsed = end_time - start_time
335
- assert elapsed < 0.5, f"is_code method took too long: {elapsed:.3f} seconds"
335
+ assert elapsed < 0.5, f"is_code method took too long: {elapsed:.3f} seconds"
336
+
337
+ def test_formula_languages_integration(self):
338
+ """Test SemanticChunk and builder with formula languages (LaTeX, MathML, AsciiMath, MathJax, KaTeX, SymPy)."""
339
+ builder = ChunkMetadataBuilder()
340
+ source_id = str(uuid.uuid4())
341
+ formula_cases = [
342
+ (LanguageEnum.LATEX, r"\\frac{a}{b} = c"),
343
+ (LanguageEnum.MATHML, "<math><mfrac><mi>a</mi><mi>b</mi></mfrac></math>"),
344
+ (LanguageEnum.ASCIIMATH, "a/b = c"),
345
+ (LanguageEnum.MATHJAX, r"\\(a^2 + b^2 = c^2\\)"),
346
+ (LanguageEnum.KATEX, r"c = \sqrt{a^2 + b^2}"),
347
+ (LanguageEnum.SYMPY, "Eq(a/b, c)"),
348
+ ]
349
+ for language, formula in formula_cases:
350
+ chunk = builder.build_semantic_chunk(
351
+ body=formula,
352
+ language=language,
353
+ chunk_type=ChunkType.DOC_BLOCK,
354
+ source_id=source_id,
355
+ start=0,
356
+ end=len(formula)
357
+ )
358
+ assert chunk.language == language
359
+ assert chunk.is_code() is True, f"{language} should be detected as code"
360
+ assert LanguageEnum.is_programming_language(chunk.language) is True
361
+ # Test round-trip serialization
362
+ flat_dict = chunk.to_flat_dict()
363
+ restored_chunk = SemanticChunk.from_flat_dict(flat_dict)
364
+ assert restored_chunk.language == language
365
+ assert restored_chunk.is_code() is True
@@ -635,425 +635,4 @@ def test_tokens_flat_and_semantic():
635
635
  assert flat["tokens"] == 123
636
636
  # flat -> semantic
637
637
  restored = builder.flat_to_semantic(flat)
638
- assert restored.metrics.tokens == 123
639
-
640
-
641
- def test_build_flat_metadata_invalid_source_id():
642
- """Test build_flat_metadata with invalid source_id."""
643
- builder = ChunkMetadataBuilder()
644
-
645
- # Invalid source_id (not UUID)
646
- with pytest.raises(ValueError, match="badly formed hexadecimal UUID string"):
647
- builder.build_flat_metadata(
648
- body="test",
649
- source_id="invalid-uuid",
650
- ordinal=0,
651
- type=ChunkType.DRAFT,
652
- language="en"
653
- )
654
-
655
-
656
- def test_build_flat_metadata_invalid_link_parent():
657
- """Test build_flat_metadata with invalid link_parent."""
658
- builder = ChunkMetadataBuilder()
659
-
660
- with pytest.raises(ValueError, match="badly formed hexadecimal UUID string"):
661
- builder.build_flat_metadata(
662
- body="test",
663
- source_id=str(uuid.uuid4()),
664
- ordinal=0,
665
- type=ChunkType.DRAFT,
666
- language="en",
667
- link_parent="invalid-uuid"
668
- )
669
-
670
-
671
- def test_build_flat_metadata_invalid_link_related():
672
- """Test build_flat_metadata with invalid link_related."""
673
- builder = ChunkMetadataBuilder()
674
-
675
- with pytest.raises(ValueError, match="badly formed hexadecimal UUID string"):
676
- builder.build_flat_metadata(
677
- body="test",
678
- source_id=str(uuid.uuid4()),
679
- ordinal=0,
680
- type=ChunkType.DRAFT,
681
- language="en",
682
- link_related="invalid-uuid"
683
- )
684
-
685
-
686
- def test_build_flat_metadata_invalid_coverage():
687
- """Test build_flat_metadata with invalid coverage values."""
688
- builder = ChunkMetadataBuilder()
689
- source_id = str(uuid.uuid4())
690
-
691
- # Coverage not a float
692
- with pytest.raises(ValueError, match="coverage must be a float"):
693
- builder.build_flat_metadata(
694
- body="test",
695
- source_id=source_id,
696
- ordinal=0,
697
- type=ChunkType.DRAFT,
698
- language="en",
699
- coverage="invalid"
700
- )
701
-
702
- # Coverage out of range
703
- with pytest.raises(ValueError, match="coverage must be in \\[0, 1\\]"):
704
- builder.build_flat_metadata(
705
- body="test",
706
- source_id=source_id,
707
- ordinal=0,
708
- type=ChunkType.DRAFT,
709
- language="en",
710
- coverage=1.5
711
- )
712
-
713
-
714
- def test_build_flat_metadata_invalid_tags():
715
- """Test build_flat_metadata with invalid tags."""
716
- builder = ChunkMetadataBuilder()
717
-
718
- with pytest.raises(ValueError, match="tags must be a list of strings"):
719
- builder.build_flat_metadata(
720
- body="test",
721
- source_id=str(uuid.uuid4()),
722
- ordinal=0,
723
- type=ChunkType.DRAFT,
724
- language="en",
725
- tags="not-a-list"
726
- )
727
-
728
-
729
- def test_build_flat_metadata_invalid_chunks():
730
- """Test build_flat_metadata with invalid chunks."""
731
- builder = ChunkMetadataBuilder()
732
-
733
- with pytest.raises(ValueError, match="chunks must be a list of strings"):
734
- builder.build_flat_metadata(
735
- body="test",
736
- source_id=str(uuid.uuid4()),
737
- ordinal=0,
738
- type=ChunkType.DRAFT,
739
- language="en",
740
- chunks="not-a-list"
741
- )
742
-
743
-
744
- def test_build_flat_metadata_enum_conversions():
745
- """Test build_flat_metadata with enum conversions."""
746
- builder = ChunkMetadataBuilder()
747
- source_id = str(uuid.uuid4())
748
-
749
- # Test with enum instances
750
- metadata = builder.build_flat_metadata(
751
- body="test",
752
- source_id=source_id,
753
- ordinal=0,
754
- type=ChunkType.CODE_BLOCK, # Enum instance
755
- language="Python",
756
- role=ChunkRole.DEVELOPER, # Enum instance
757
- status=ChunkStatus.VERIFIED # Enum instance
758
- )
759
-
760
- assert metadata["type"] == "CodeBlock"
761
- assert metadata["role"] == "developer"
762
- assert metadata["status"] == "verified"
763
- assert metadata["is_code_chunk"] == "true" # Should be detected as code
764
-
765
-
766
- def test_build_flat_metadata_text_body_normalization():
767
- """Test build_flat_metadata text/body normalization."""
768
- builder = ChunkMetadataBuilder()
769
- source_id = str(uuid.uuid4())
770
-
771
- # Test text=None, body provided
772
- metadata1 = builder.build_flat_metadata(
773
- body="test content",
774
- text=None,
775
- source_id=source_id,
776
- ordinal=0,
777
- type=ChunkType.DRAFT,
778
- language="en"
779
- )
780
- assert metadata1["text"] == "test content"
781
-
782
- # Test body=None, text provided
783
- metadata2 = builder.build_flat_metadata(
784
- body=None,
785
- text="test content",
786
- source_id=source_id,
787
- ordinal=0,
788
- type=ChunkType.DRAFT,
789
- language="en"
790
- )
791
- assert metadata2["body"] == "test content"
792
-
793
-
794
- def test_build_semantic_chunk_invalid_tags():
795
- """Test build_semantic_chunk with invalid tags."""
796
- builder = ChunkMetadataBuilder()
797
-
798
- with pytest.raises(ValueError, match="tags must be a list of strings"):
799
- builder.build_semantic_chunk(
800
- body="test",
801
- language="en",
802
- chunk_type=ChunkType.DRAFT,
803
- tags="not-a-list"
804
- )
805
-
806
-
807
- def test_build_semantic_chunk_invalid_links():
808
- """Test build_semantic_chunk with invalid links."""
809
- builder = ChunkMetadataBuilder()
810
-
811
- with pytest.raises(ValueError, match="links must be a list of strings"):
812
- builder.build_semantic_chunk(
813
- body="test",
814
- language="en",
815
- chunk_type=ChunkType.DRAFT,
816
- links="not-a-list"
817
- )
818
-
819
-
820
- def test_build_semantic_chunk_invalid_source_id():
821
- """Test build_semantic_chunk with invalid source_id."""
822
- builder = ChunkMetadataBuilder()
823
-
824
- with pytest.raises(ValueError, match="badly formed hexadecimal UUID string"):
825
- builder.build_semantic_chunk(
826
- body="test",
827
- language="en",
828
- chunk_type=ChunkType.DRAFT,
829
- source_id="invalid-uuid"
830
- )
831
-
832
-
833
- def test_build_semantic_chunk_invalid_link_format():
834
- """Test build_semantic_chunk with invalid link format."""
835
- builder = ChunkMetadataBuilder()
836
-
837
- with pytest.raises(ValueError, match="Link must follow 'relation:uuid' format"):
838
- builder.build_semantic_chunk(
839
- body="test",
840
- language="en",
841
- chunk_type=ChunkType.DRAFT,
842
- links=["invalid-format"]
843
- )
844
-
845
-
846
- def test_build_semantic_chunk_invalid_link_uuid():
847
- """Test build_semantic_chunk with invalid UUID in link."""
848
- builder = ChunkMetadataBuilder()
849
-
850
- with pytest.raises(ValueError, match="Invalid UUID4 in link"):
851
- builder.build_semantic_chunk(
852
- body="test",
853
- language="en",
854
- chunk_type=ChunkType.DRAFT,
855
- links=["parent:invalid-uuid"]
856
- )
857
-
858
-
859
- def test_build_semantic_chunk_enum_conversions():
860
- """Test build_semantic_chunk with enum conversions."""
861
- builder = ChunkMetadataBuilder()
862
-
863
- # Test string to enum conversion
864
- chunk = builder.build_semantic_chunk(
865
- body="test",
866
- language="Python",
867
- chunk_type=ChunkType.CODE_BLOCK, # Use enum directly
868
- role=ChunkRole.DEVELOPER, # Use enum directly
869
- status=ChunkStatus.VERIFIED # Use enum directly
870
- )
871
-
872
- assert chunk.type == ChunkType.CODE_BLOCK
873
- assert chunk.role == ChunkRole.DEVELOPER
874
- assert chunk.status == ChunkStatus.VERIFIED
875
-
876
-
877
- def test_build_semantic_chunk_metrics_handling():
878
- """Test build_semantic_chunk metrics handling."""
879
- builder = ChunkMetadataBuilder()
880
-
881
- # Test with explicit metrics
882
- custom_metrics = ChunkMetrics(quality_score=0.9)
883
- chunk1 = builder.build_semantic_chunk(
884
- body="test",
885
- language="en",
886
- chunk_type=ChunkType.DRAFT,
887
- metrics=custom_metrics,
888
- tokens=100
889
- )
890
- assert chunk1.metrics.quality_score == 0.9
891
- assert chunk1.metrics.tokens == 100 # Should be updated
892
-
893
- # Test with None metrics and individual values
894
- chunk2 = builder.build_semantic_chunk(
895
- body="test",
896
- language="en",
897
- chunk_type=ChunkType.DRAFT,
898
- quality_score=0.8,
899
- coverage=0.7,
900
- tokens=50
901
- )
902
- assert chunk2.metrics.quality_score == 0.8
903
- assert chunk2.metrics.coverage == 0.7
904
- assert chunk2.metrics.tokens == 50
905
-
906
-
907
- def test_build_semantic_chunk_string_validation():
908
- """Test build_semantic_chunk string field validation."""
909
- builder = ChunkMetadataBuilder(project=None)
910
-
911
- chunk = builder.build_semantic_chunk(
912
- body="", # Empty body
913
- text="test",
914
- language="en",
915
- chunk_type=ChunkType.DRAFT,
916
- summary="", # Empty summary
917
- source_path="" # Empty source_path
918
- )
919
-
920
- # Empty strings should be replaced with minimum length strings
921
- assert len(chunk.body) >= 1
922
- assert len(chunk.summary) >= 1
923
- assert len(chunk.source_path) >= 1
924
-
925
-
926
- def test_build_semantic_chunk_uuid_validation():
927
- """Test build_semantic_chunk UUID validation and generation."""
928
- builder = ChunkMetadataBuilder(unit_id="invalid-uuid") # Invalid unit_id
929
-
930
- chunk = builder.build_semantic_chunk(
931
- body="test",
932
- language="en",
933
- chunk_type=ChunkType.DRAFT,
934
- task_id="invalid-task-uuid", # Invalid task_id
935
- subtask_id="invalid-subtask-uuid" # Invalid subtask_id
936
- )
937
-
938
- # Invalid UUIDs should be replaced with valid ones
939
- import re
940
- UUID_PATTERN = re.compile(r'^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$', re.IGNORECASE)
941
- assert UUID_PATTERN.match(chunk.unit_id)
942
- assert UUID_PATTERN.match(chunk.task_id)
943
- assert UUID_PATTERN.match(chunk.subtask_id)
944
-
945
-
946
- def test_build_semantic_chunk_tags_string_conversion():
947
- """Test build_semantic_chunk tags string conversion."""
948
- # This test is for build_semantic_chunk which expects list, not string
949
- # String conversion happens in build_flat_metadata
950
- builder = ChunkMetadataBuilder()
951
-
952
- # Test that string tags are converted in flat metadata
953
- metadata = builder.build_flat_metadata(
954
- body="test",
955
- source_id=str(uuid.uuid4()),
956
- ordinal=0,
957
- type=ChunkType.DRAFT,
958
- language="en",
959
- tags=["tag1", "tag2", "tag3"] # List tags should be converted to string
960
- )
961
-
962
- assert "tags" in metadata
963
-
964
-
965
- def test_build_semantic_chunk_chunks_validation():
966
- """Test build_semantic_chunk chunks validation."""
967
- builder = ChunkMetadataBuilder()
968
-
969
- with pytest.raises(ValueError, match="chunks must be a list of strings"):
970
- builder.build_semantic_chunk(
971
- body="test",
972
- language="en",
973
- chunk_type=ChunkType.DRAFT,
974
- chunks="not-a-list"
975
- )
976
-
977
-
978
- def test_semantic_to_flat_link_extraction():
979
- """Test semantic_to_flat link extraction."""
980
- builder = ChunkMetadataBuilder()
981
-
982
- # Create chunk with parent and related links
983
- chunk = builder.build_semantic_chunk(
984
- body="test",
985
- language="en",
986
- chunk_type=ChunkType.DRAFT,
987
- links=[
988
- f"parent:{str(uuid.uuid4())}",
989
- f"related:{str(uuid.uuid4())}"
990
- ]
991
- )
992
-
993
- flat_dict = builder.semantic_to_flat(chunk)
994
-
995
- assert flat_dict["link_parent"] is not None
996
- assert flat_dict["link_related"] is not None
997
- assert flat_dict["tags"] == "" # Empty list converted to empty string
998
-
999
-
1000
- def test_semantic_to_flat_no_links():
1001
- """Test semantic_to_flat with no links."""
1002
- builder = ChunkMetadataBuilder()
1003
-
1004
- chunk = builder.build_semantic_chunk(
1005
- body="test",
1006
- language="en",
1007
- chunk_type=ChunkType.DRAFT,
1008
- links=[]
1009
- )
1010
-
1011
- flat_dict = builder.semantic_to_flat(chunk)
1012
-
1013
- assert flat_dict["link_parent"] is None
1014
- assert flat_dict["link_related"] is None
1015
-
1016
-
1017
- def test_wrapper_methods():
1018
- """Test wrapper methods for conversion."""
1019
- builder = ChunkMetadataBuilder()
1020
-
1021
- # Create a chunk
1022
- chunk = builder.build_semantic_chunk(
1023
- body="test",
1024
- language="en",
1025
- chunk_type=ChunkType.DRAFT,
1026
- tags=["test"]
1027
- )
1028
-
1029
- # Test all wrapper methods
1030
- flat_dict = {"body": "test", "type": "Draft", "language": "en"}
1031
-
1032
- # flat_to_semantic
1033
- restored_chunk = builder.flat_to_semantic(flat_dict)
1034
- assert restored_chunk.body == "test"
1035
-
1036
- # semantic_to_json_dict
1037
- json_dict = builder.semantic_to_json_dict(chunk)
1038
- assert json_dict["body"] == "test"
1039
-
1040
- # json_dict_to_semantic
1041
- restored_from_json = builder.json_dict_to_semantic(json_dict)
1042
- assert restored_from_json.body == "test"
1043
-
1044
-
1045
- def test_build_metadata_alias():
1046
- """Test build_metadata alias for backward compatibility."""
1047
- builder = ChunkMetadataBuilder()
1048
-
1049
- # build_metadata should work the same as build_flat_metadata
1050
- metadata = builder.build_metadata(
1051
- body="test",
1052
- source_id=str(uuid.uuid4()),
1053
- ordinal=0,
1054
- type=ChunkType.DRAFT,
1055
- language="en"
1056
- )
1057
-
1058
- assert metadata["body"] == "test"
1059
- assert metadata["type"] == "Draft"
638
+ assert restored.metrics.tokens == 123
@@ -127,12 +127,12 @@ class TestRoundTripRedis:
127
127
  self._compare_chunks(original_chunk, restored_chunk)
128
128
 
129
129
  def test_tags_and_links_always_lists(self, valid_uuid, sample_text):
130
- """Verify tags, links and chunks are always List[str] regardless of format"""
131
- # Test with various tag/link/chunks formats
130
+ """Verify tags and links are always List[str] regardless of format"""
131
+ # Test with various tag/link formats
132
132
  test_cases = [
133
- {"tags": ["a", "b", "c"], "links": ["rel:uuid1", "rel:uuid2"], "chunks": ["chunk1", "chunk2"]},
134
- {"tags": [], "links": [], "chunks": []},
135
- {"tags": None, "links": None, "chunks": None},
133
+ {"tags": ["a", "b", "c"], "links": ["rel:uuid1", "rel:uuid2"]},
134
+ {"tags": [], "links": []},
135
+ {"tags": None, "links": None},
136
136
  ]
137
137
 
138
138
  for case in test_cases:
@@ -149,22 +149,18 @@ class TestRoundTripRedis:
149
149
  restored_chunk = self._normalize_types(restored_chunk)
150
150
 
151
151
  # Both should have lists (empty if None was provided)
152
- # Note: SemanticChunk may initialize None tags/links/chunks as None, but from_flat_dict converts to []
152
+ # Note: SemanticChunk may initialize None tags/links as None, but from_flat_dict converts to []
153
153
  orig_tags = original_chunk.tags if original_chunk.tags is not None else []
154
154
  orig_links = original_chunk.links if original_chunk.links is not None else []
155
- orig_chunks = original_chunk.chunks if original_chunk.chunks is not None else []
156
155
  rest_tags = restored_chunk.tags if restored_chunk.tags is not None else []
157
156
  rest_links = restored_chunk.links if restored_chunk.links is not None else []
158
- rest_chunks = restored_chunk.chunks if restored_chunk.chunks is not None else []
159
157
 
160
158
  assert isinstance(rest_tags, list)
161
159
  assert isinstance(rest_links, list)
162
- assert isinstance(rest_chunks, list)
163
160
 
164
161
  # Content should match
165
162
  assert orig_tags == rest_tags
166
163
  assert orig_links == rest_links
167
- assert orig_chunks == rest_chunks
168
164
 
169
165
  def _normalize_types(self, chunk: SemanticChunk) -> SemanticChunk:
170
166
  """
@@ -253,8 +249,8 @@ class TestRoundTripRedis:
253
249
  orig_val = original_dict[key]
254
250
  rest_val = restored_dict.get(key)
255
251
 
256
- # Special handling for list fields (tags, links, chunks, embedding)
257
- if key in ["tags", "links", "chunks", "embedding"]:
252
+ # Special handling for list fields (tags, links, embedding)
253
+ if key in ["tags", "links", "embedding"]:
258
254
  orig_list = orig_val if orig_val is not None else []
259
255
  rest_list = rest_val if rest_val is not None else []
260
256
  assert orig_list == rest_list, f"Mismatch in {key}: {orig_list} != {rest_list}"
@@ -295,7 +291,7 @@ class TestRoundTripRedis:
295
291
 
296
292
  # All values should be strings except lists
297
293
  for key, value in flat_dict.items():
298
- if key in ["tags", "links", "chunks"]:
294
+ if key in ["tags", "links"]:
299
295
  assert isinstance(value, list), f"{key} should be list, got {type(value)}"
300
296
  else:
301
297
  assert isinstance(value, str), f"{key} should be string, got {type(value)}: {value}"