ddi-fw 0.0.257__py3-none-any.whl → 0.0.259__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (6) hide show
  1. ddi_fw/langchain/faiss_storage.py +5 -1
  2. ddi_fw/utils/kaggle.py +2 -1
  3. {ddi_fw-0.0.257.dist-info → ddi_fw-0.0.259.dist-info}/METADATA +1 -1
  4. {ddi_fw-0.0.257.dist-info → ddi_fw-0.0.259.dist-info}/RECORD +6 -6
  5. {ddi_fw-0.0.257.dist-info → ddi_fw-0.0.259.dist-info}/WHEEL +0 -0
  6. {ddi_fw-0.0.257.dist-info → ddi_fw-0.0.259.dist-info}/top_level.txt +0 -0
ddi_fw/langchain/faiss_storage.py CHANGED
@@ -1,4 +1,5 @@
1
1
  import faiss
2
+ from langchain_text_splitters import TextSplitter
2
3
  import pandas as pd
3
4
  from uuid import uuid4
4
5
  from langchain_community.vectorstores.faiss import FAISS
@@ -45,6 +46,7 @@ class FaissVectorStoreManager(BaseVectorStoreManager):
45
46
  persist_directory: str = Field(default="./embeddings/faiss")
46
47
  index: Any = None
47
48
  vector_store: Optional[FAISS] | None = None
49
+ text_splitter: Optional[TextSplitter] = None
48
50
  class Config:
49
51
  arbitrary_types_allowed = True
50
52
  # def generate_vector_store(self, docs):
@@ -111,7 +113,9 @@ class FaissVectorStoreManager(BaseVectorStoreManager):
111
113
 
112
114
  valid_docs = []
113
115
  valid_ids = []
114
-
116
+ if self.text_splitter:
117
+ docs = self.text_splitter.split_documents(docs)
118
+
115
119
  for doc in docs:
116
120
  content = doc.page_content if hasattr(doc, 'page_content') else ""
117
121
  if content and content.strip():
ddi_fw/utils/kaggle.py CHANGED
@@ -22,7 +22,8 @@ def create_kaggle_dataset(base_path: str, collections: list):
22
22
  # Step 2: Get metadata for the current folder
23
23
  model_info = next((c for c in collections if c['id'] == folder_name), None)
24
24
  if model_info is None:
25
- continue # Skip if model info is not found
25
+ raise FileNotFoundError(f"Model info for {folder_name} not exists")
26
+ # continue # Skip if model info is not found
26
27
 
27
28
  title = model_info['kaggle_title']
28
29
 
{ddi_fw-0.0.257.dist-info → ddi_fw-0.0.259.dist-info}/METADATA RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: ddi_fw
3
- Version: 0.0.257
3
+ Version: 0.0.259
4
4
  Summary: Do not use :)
5
5
  Author-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>
6
6
  Maintainer-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>
{ddi_fw-0.0.257.dist-info → ddi_fw-0.0.259.dist-info}/RECORD RENAMED
@@ -6,7 +6,7 @@ ddi_fw/datasets/setup_._py,sha256=khYVJuW5PlOY_i_A16F3UbSZ6s6o_ljw33Byw3C-A8E,10
6
6
  ddi_fw/langchain/__init__.py,sha256=Kk2Yr7vemjy9MNB_ImAWET808zt1JkLsWqsgEXpVPJk,421
7
7
  ddi_fw/langchain/chroma_storage.py,sha256=Xh0p2d8bI9U-NYkhuu7fLtsgbExP7phkqPE17PBYWd4,15634
8
8
  ddi_fw/langchain/embeddings.py,sha256=eEWy4okcjdhUJHi4N48Wd8XauPXyeaQVLUdNWEvtEcY,6754
9
- ddi_fw/langchain/faiss_storage.py,sha256=kFnvsRCE1Kdv77XgC2jLlYrAHZ9Qoz-ygs-IPB2Sg6g,18275
9
+ ddi_fw/langchain/faiss_storage.py,sha256=VCKvBB50t3krO5isqRMSPT5yFB9d9s1jQUWiWsmOiak,18481
10
10
  ddi_fw/langchain/sentence_splitter.py,sha256=h_bYElx4Ud1mwDNJfL7mUwvgadwKX3GKlSzu5L2PXzg,280
11
11
  ddi_fw/langchain/storage.py,sha256=OizKyWm74Js7T6Q9kez-ulUoBGzIMFo4R46h4kjUyIM,11200
12
12
  ddi_fw/ml/__init__.py,sha256=FteYEawCkVQOaK-cTv2VrHZ2ZnfeFr31BD6VucO7_DQ,268
@@ -29,7 +29,7 @@ ddi_fw/utils/__init__.py,sha256=WNxkQXk-694roG50D355TGLXstfdWVb_tUyr-PM-8rg,537
29
29
  ddi_fw/utils/categorical_data_encoding_checker.py,sha256=T1X70Rh4atucAuqyUZmz-iFULllY9dY0NRyV9-jTjJ0,3438
30
30
  ddi_fw/utils/enums.py,sha256=19eJ3fX5eRK_xPvkYcukmug144jXPH4X9zQqtsFBj5A,671
31
31
  ddi_fw/utils/json_helper.py,sha256=BVU6wmJgdXPxyqLPu3Ck_9Es5RrP1PDanKvE-OSj1D4,571
32
- ddi_fw/utils/kaggle.py,sha256=wKRJ18KpQ6P-CubpZklEgsDtyFpR9RUL1_HyyF6ttEE,2425
32
+ ddi_fw/utils/kaggle.py,sha256=ZlKS4kZVsNV_JNYO4IvqY4GkhQCvl-_5saaz_bZzgrQ,2508
33
33
  ddi_fw/utils/numpy_utils.py,sha256=gd1WNq5NpWD2MBEMTtFuS5I0h8B6FAUNcq6BVOlxdhY,797
34
34
  ddi_fw/utils/package_helper.py,sha256=erl8_onmhK-41zQoaED2qyDUV9GQxmT9sdoyRp9_q5I,1056
35
35
  ddi_fw/utils/py7zr_helper.py,sha256=gOqaFIyJvTjUM-btO2x9AQ69jZOS8PoKN0wetYIckJw,4747
@@ -38,7 +38,7 @@ ddi_fw/utils/zip_helper.py,sha256=YRZA4tKZVBJwGQM0_WK6L-y5MoqkKoC-nXuuHK6CU9I,55
38
38
  ddi_fw/vectorization/__init__.py,sha256=LcJOpLVoLvHPDw9phGFlUQGeNcST_zKV-Oi1Pm5h_nE,110
39
39
  ddi_fw/vectorization/feature_vector_generation.py,sha256=92bhZw4Qxh0hqPK-bPHm9bUO7pg2p4cStQYtVrOtetE,7919
40
40
  ddi_fw/vectorization/idf_helper.py,sha256=_Gd1dtDSLaw8o-o0JugzSKMt9FpeXewTh4wGEaUd4VQ,2571
41
- ddi_fw-0.0.257.dist-info/METADATA,sha256=Iz86o_WMv72q5U6wIPZ7Wq4mG-4WAwXl1VFWdvmTrTA,2623
42
- ddi_fw-0.0.257.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
43
- ddi_fw-0.0.257.dist-info/top_level.txt,sha256=PMwHICFZTZtcpzQNPV4UQnfNXYIeLR_Ste-Wfc1h810,7
44
- ddi_fw-0.0.257.dist-info/RECORD,,
41
+ ddi_fw-0.0.259.dist-info/METADATA,sha256=PJfj1EQLb65en5lYWyhYA7izycUHXLEF_MlCllV6qwg,2623
42
+ ddi_fw-0.0.259.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
43
+ ddi_fw-0.0.259.dist-info/top_level.txt,sha256=PMwHICFZTZtcpzQNPV4UQnfNXYIeLR_Ste-Wfc1h810,7
44
+ ddi_fw-0.0.259.dist-info/RECORD,,