snplib 1.1.10__tar.gz → 1.2.10__tar.gz

{snplib-1.1.10/src/snplib.egg-info → snplib-1.2.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: snplib
-Version: 1.1.10
+Version: 1.2.10
 Summary: Snptools is a tool for Single Nucleotide Polymorphism (SNP) data processing
 Author-email: Igor <igor.loschinin@gmail.com>
 License: GNU

{snplib-1.1.10 → snplib-1.2.10}/docs/examples.rst

@@ -73,6 +73,31 @@ Output::
 
         ...
 
+To handle large files, use `usecols` and `dtype`. This reduces memory
+consumption and speeds up processing.
+
+.. note::
+    `usecols` is used when `allele` is **None**.
+
+.. code-block:: python
+
+        alleles_ab = FinalReport(
+            usecols=['SNP Name', 'Sample ID', 'Allele1 - AB', 'Allele2 - AB'],
+            dtype={'SNP Name': 'category'}
+        )
+        alleles_ab.handle("path/to/finalreport.txt")
+        data_ab = alleles_ab.snp_data
+
+Output::
+
+        SNP Name    Sample ID    Allele1 - AB  Allele2 - AB
+        ARS-BFGL-BAC-10172 HO840M003135245650 B B
+        ARS-BFGL-BAC-1020 HO840M003135245650 B B
+        ARS-BFGL-BAC-10245 HO840M003135245650 B B
+        ARS-BFGL-BAC-10345 HO840M003135245650 A B
+        ARS-BFGL-BAC-10375 HO840M003135245650 A B
+        ...
+
 Preparation SNP files
 ---------------------
 

{snplib-1.1.10 → snplib-1.2.10}/pyproject.toml

@@ -17,7 +17,7 @@ snplib = ["*.pl"]
 
 [project]
 name = "snplib"
-version = "1.1.10"
+version = "1.2.10"
 description = "Snptools is a tool for Single Nucleotide Polymorphism (SNP) data processing"
 authors = [
     {name = "Igor", email = "igor.loschinin@gmail.com"}

{snplib-1.1.10 → snplib-1.2.10}/src/snplib/finalreport/_finalreport.py

@@ -3,13 +3,12 @@
 __author__ = "Igor Loschinin (igor.loschinin@gmail.com)"
 __all__ = ("FinalReport",)
 
-from pathlib import Path
-from functools import reduce
-
 import re
+from functools import reduce
+from pathlib import Path
 
-from numpy import nan
 import pandas as pd
+from numpy import nan
 
 
 class FinalReport(object):
@@ -17,10 +16,14 @@ class FinalReport(object):
 	handle method. If values in 'SID' or 'UNIQ_KEY' were missing in the xlsx
 	conversion file, the processed data will contain NAN values.
 
-	:argument allele: A variant form of a single nucleotide polymorphism
-		(SNP), a specific polymorphic site or a whole gene detectable at
-		a locus.  Type: 'AB', 'Forward', 'Top', 'Plus', 'Design'
-	:argument sep: Delimiter to use. Default value: "\\t"
+	:param allele: A variant form of a single nucleotide polymorphism (SNP), a
+		specific polymorphic site or a whole gene detectable at a locus. Type:
+		'AB', 'Forward', 'Top', 'Plus', 'Design'.
+	:param sep: Delimiter to use. Default value: "\\t".
+	:param usecols: Selection of fields for reading. Accelerates processing
+		and reduces memory.
+	:param dtype: Data type(s) to apply to either the whole dataset or
+		individual columns. E.g., {'a': np.float64, 'b': np.int32, 'c': 'Int64'}.
 
 	Example:
 		[Header]
@@ -38,20 +41,34 @@ class FinalReport(object):
 		...
 	"""
 
-	__PATTERN_HEADER = re.compile(r'(^\[Header\])')
-	__PATTERN_DATA = re.compile(r'(^\[Data\])')
+	__PATTERN_HEADER = re.compile(r'(^\[Header])')
+	__PATTERN_DATA = re.compile(r'(^\[Data])')
+
+	__slots__ = (
+		"_delimiter",
+		"__allele",
+		"__usecols",
+		"__dtype",
+		"__snp_data",
+		"__header",
+		"_map_rn",
+	)
 
 	def __init__(
 			self,
 			allele: str | list | None = None,
+			usecols: list[str] | None = None,
+			dtype: dict | None = None,
 			sep: str = "\t"
 	) -> None:
 		self._delimiter = sep
-		self._full_data = None
+		self.__allele = allele
+		self.__usecols = usecols
+		self.__dtype = dtype
 
+		# self._full_data = None
+		self.__snp_data: pd.DataFrame | None = None
 		self.__header = {}
-		self.__snp_data = None
-		self.__allele = allele
 		self._map_rn = None
 
 	@property
@@ -77,6 +94,9 @@ class FinalReport(object):
 
 		try:
 
+			if self.__allele is not None and self.__usecols is not None:
+				raise Exception("Error. Usecols is used for allele is none.")
+
 			if isinstance(file_rep, str):
 				file_rep = Path(file_rep)
 
@@ -93,17 +113,11 @@ class FinalReport(object):
 
 				self.__convert_s_id(conv_file)
 
-			# Processing report file
-			if not self.read(file_rep):
-				return False
-
-			if self._full_data is None:
-				raise Exception("Not data in file FinalReport.txt")
-
-			self.__handler_header()
-			self.__handler_data()
+			# # Processing report file
+			self.__handler_header(file_rep)
+			self.__handler_data(file_rep)
 
-			if self._map_rn is not None:
+			if not self.__snp_data.empty and self._map_rn is not None:
 				self.__snp_data['Sample ID'] = \
 					self.__snp_data['Sample ID'].map(
 						dict(zip(self._map_rn.SID, self._map_rn.UNIQ_KEY))
@@ -114,62 +128,99 @@ class FinalReport(object):
 
 		return True
 
-	def read(self, file_rep: Path) -> bool:
-		""" Reading data from the final_report file
+	def __handler_header(self, file_rep: Path) -> None:
+		""" Processes data from a file, selects meta-information.
 
 		:param file_rep: path, pointer to the file to be read.
-		:return: Returns true if the read was successful, false if it failed.
 		"""
-		try:
-			if len(data := file_rep.read_text()) != 0:
-				self._full_data = data.strip().split("\n")
-				return True
 
-			self._full_data = None
+		with open(file_rep, 'r') as file:
 
-		except Exception as e:
-			return False
+			for line in file:
+				if self.__class__.__PATTERN_DATA.findall(line.strip()):
+					return
 
-		return True
+				if self.__class__.__PATTERN_HEADER.findall(line.strip()) or\
+					len(line.strip()) == 0:
+					continue
+
+				key = line.strip().split("\t")[0]
+				value = line.strip().split("\t")[1]
+
+				self.__header[key] = value
+
+	def __handler_data(self, file_rep: Path) -> None:
+		""" Processes data and forms an array for further processing.
+
+		:param file_rep: path, pointer to the file to be read.
+		"""
 
-	def __handler_header(self) -> None:
-		""" Processes data from a file, selects meta-information. """
+		with open(file_rep, 'r') as file:
+
+			# Search for the data start index and skip
+			for line in file:
+				if self.__class__.__PATTERN_DATA.findall(line.strip()):
+					break
+
+			# line column
+			orig_name_col = file.readline().strip().split(self._delimiter)
+
+			if self.__allele is None and self.__usecols is None:
+				self.__snp_data = pd.read_csv(
+					file,
+					sep=self._delimiter,
+					header=None,
+					names=orig_name_col,
+					dtype=self.__dtype,
+					low_memory=True,
+					na_filter=True
+				)
 
-		for line in self._full_data:
-			if self.__class__.__PATTERN_DATA.findall(line):
 				return
 
-			if self.__class__.__PATTERN_HEADER.findall(line):
-				continue
+			sub_n_col = self.__processing_columns(orig_name_col)
+			self.__snp_data = pd.read_csv(
+				file,
+				sep=self._delimiter,
+				header=None,
+				names=orig_name_col,
+				usecols=sub_n_col,
+				dtype=self.__dtype,
+				low_memory=True,
+				na_filter=True
+			)
 
-			key = line.strip().split("\t")[0]
-			value = line.strip().split("\t")[1]
+			return
 
-			self.__header[key] = value
+	def __processing_columns(self, lst_col: list[str]) -> list[str] | None:
+		""" Processing the line with all the names of the fields and the
+		sample of them.
 
-	def __handler_data(self) -> None:
-		""" Processes data and forms an array for further processing. """
+		:param lst_col: List of all fields.
+		:return: Returns a tuple with a list of names of selected fields.
+		"""
 
-		temp = 1
-		for line in self._full_data:
-			if self.__class__.__PATTERN_DATA.findall(line):
-				break
-			temp += 1
+		if self.__usecols is not None:
+			check_n_col = [
+				item for item in self.__usecols if item in lst_col
+			]
 
-		names_col = self.__sample_by_allele(
-			self._full_data[temp].split(f"{self._delimiter}")
-		)
+			# Check on empty list
+			if check_n_col:
+				return self.__usecols
+
+			raise Exception(
+				f"Error. The USECOLS list contains not true fields."
+			)
 
-		if names_col is None:
-			raise Exception(f"Error. Allele {self.__allele} not in data.")
+		# processing alleles
+		sample_n_col = self.__sample_by_allele(lst_col)
+		if sample_n_col is None:
+			raise Exception(
+				f"Error. Allele {self.__allele} not in data."
+			)
 
-		self.__snp_data = pd.DataFrame(
-			[
-				item_data.split(f"{self._delimiter}")
-				for item_data in self._full_data[temp + 1:]
-			],
-			columns=self._full_data[temp].split(f"{self._delimiter}")
-		)[names_col]
+		return sample_n_col
 
 	def __sample_by_allele(self, names: list[str]) -> list[str] | None:
 		""" Method that generates a list of field names choosing which alleles

{snplib-1.1.10 → snplib-1.2.10/src/snplib.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: snplib
-Version: 1.1.10
+Version: 1.2.10
 Summary: Snptools is a tool for Single Nucleotide Polymorphism (SNP) data processing
 Author-email: Igor <igor.loschinin@gmail.com>
 License: GNU

{snplib-1.1.10 → snplib-1.2.10}/tests/finalreport/test_finalreport.py

@@ -11,12 +11,12 @@ import pytest
 
 @pytest.fixture
 def report(request) -> FinalReport:
-	return FinalReport(allele=request.param)
+	return FinalReport(**request.param)
 
 
 class TestFinalReport(object):
 
-	@pytest.mark.parametrize("report", [None], indirect=True)
+	@pytest.mark.parametrize("report", [{'allele': None}], indirect=True)
 	def test_handle_1(self, report: FinalReport) -> None:
 		""" If both files do not exist """
 
@@ -24,7 +24,17 @@ class TestFinalReport(object):
 			DIR_FILES / "fr/f.txt", DIR_FILES / "fr/f.xlsx",
 		)
 
-	@pytest.mark.parametrize("report", [None], indirect=True)
+	@pytest.mark.parametrize("report", [{'allele': None}], indirect=True)
+	def test_handle_8(self, report: FinalReport) -> None:
+		""" If files exist """
+
+		assert report.handle(
+			DIR_FILES / "fr/file1.txt", DIR_FILES / "fr/file1.xlsx",
+		)
+
+		assert len(report.header) != 0 and not report.snp_data.empty
+
+	@pytest.mark.parametrize("report", [{'allele': None}], indirect=True)
 	def test_handle_2(self, report: FinalReport) -> None:
 		""" If the file to convert does not exist """
 
@@ -32,7 +42,7 @@ class TestFinalReport(object):
 			DIR_FILES / "fr/file1.txt", DIR_FILES / "fr/f.xlsx",
 		)
 
-	@pytest.mark.parametrize("report", [None], indirect=True)
+	@pytest.mark.parametrize("report", [{'allele': None}], indirect=True)
 	def test_handle_3(self, report: FinalReport) -> None:
 		""" If the data does not contain header data """
 
@@ -42,7 +52,7 @@ class TestFinalReport(object):
 
 		assert len(report.header) == 0 and not report.snp_data.empty
 
-	@pytest.mark.parametrize("report", [None], indirect=True)
+	@pytest.mark.parametrize("report", [{'allele': None}], indirect=True)
 	def test_handle_4(self, report: FinalReport) -> None:
 		""" If the file contains only header and field names """
 
@@ -52,20 +62,18 @@ class TestFinalReport(object):
 
 		assert report.snp_data is not None and report.snp_data.empty
 
-	@pytest.mark.parametrize("report", [None], indirect=True)
+	@pytest.mark.parametrize("report", [{'allele': None}], indirect=True)
 	def test_handle_5(self, report: FinalReport) -> None:
 		""" If the data file is empty """
 
-		with pytest.raises(
-				Exception, match="Not data in file FinalReport.txt"
-		):
-			report.handle(
-				DIR_FILES / "fr/file5.txt", DIR_FILES / "fr/file5.xlsx",
-			)
+		report.handle(
+			DIR_FILES / "fr/file5.txt", DIR_FILES / "fr/file5.xlsx",
+		)
 
-		assert report.snp_data is None
+		assert len(report.header) == 0
+		assert report.snp_data is not None and report.snp_data.empty
 
-	@pytest.mark.parametrize("report", [None], indirect=True)
+	@pytest.mark.parametrize("report", [{'allele': None}], indirect=True)
 	def test_handle_6(self, report: FinalReport) -> None:
 		""" If the conversion file is empty """
 
@@ -76,7 +84,7 @@ class TestFinalReport(object):
 		assert not report.snp_data.empty
 		assert len(report.header) != 0
 
-	@pytest.mark.parametrize("report", [None], indirect=True)
+	@pytest.mark.parametrize("report", [{'allele': None}], indirect=True)
 	def test_handle_7(self, report: FinalReport) -> None:
 		""" If the data file is not needed to convert ID name """
 
@@ -85,15 +93,7 @@ class TestFinalReport(object):
 		assert not report.snp_data.empty
 		assert len(report.header) != 0
 
-	@pytest.mark.parametrize("report", [None], indirect=True)
-	def test_handle_8(self, report: FinalReport) -> None:
-		""" If files exist """
-
-		assert report.handle(
-			DIR_FILES / "fr/file1.txt", DIR_FILES / "fr/file1.xlsx",
-		)
-
-	@pytest.mark.parametrize("report", [None], indirect=True)
+	@pytest.mark.parametrize("report", [{'allele': None}], indirect=True)
 	def test_allele_none(self, report: FinalReport) -> None:
 		report.handle(DIR_FILES / "fr/file4.txt", None)
 
@@ -105,7 +105,7 @@ class TestFinalReport(object):
 
 		assert report.snp_data.columns.difference(_fields).empty
 
-	@pytest.mark.parametrize("report", ["AB"], indirect=True)
+	@pytest.mark.parametrize("report", [{'allele': "AB"}], indirect=True)
 	def test_sample_allele_ab(self, report: FinalReport) -> None:
 		report.handle(DIR_FILES / "fr/file4.txt", None)
 
@@ -116,7 +116,7 @@ class TestFinalReport(object):
 
 		assert report.snp_data.columns.difference(_fields).empty
 
-	@pytest.mark.parametrize("report", ["Forward"], indirect=True)
+	@pytest.mark.parametrize("report", [{'allele': "Forward"}], indirect=True)
 	def test_sample_allele_forward(self, report: FinalReport) -> None:
 		report.handle(DIR_FILES / "fr/file4.txt", None)
 
@@ -127,7 +127,7 @@ class TestFinalReport(object):
 
 		assert report.snp_data.columns.difference(_fields).empty
 
-	@pytest.mark.parametrize("report", ["Top"], indirect=True)
+	@pytest.mark.parametrize("report", [{'allele': "Top"}], indirect=True)
 	def test_sample_allele_top(self, report: FinalReport) -> None:
 		report.handle(DIR_FILES / "fr/file4.txt", None)
 
@@ -138,7 +138,9 @@ class TestFinalReport(object):
 
 		assert report.snp_data.columns.difference(_fields).empty
 
-	@pytest.mark.parametrize("report", [["AB", "Top"]], indirect=True)
+	@pytest.mark.parametrize(
+		"report", [{'allele': ["AB", "Top"]}], indirect=True
+	)
 	def test_sample_allele_list1(self, report: FinalReport) -> None:
 		report.handle(DIR_FILES / "fr/file4.txt", None)
 
@@ -149,7 +151,7 @@ class TestFinalReport(object):
 
 		assert report.snp_data.columns.difference(_fields).empty
 
-	@pytest.mark.parametrize("report", [["AB"]], indirect=True)
+	@pytest.mark.parametrize("report", [{'allele': ["AB"]}], indirect=True)
 	def test_sample_allele_list2(self, report: FinalReport) -> None:
 		report.handle(DIR_FILES / "fr/file4.txt", None)
 
@@ -160,7 +162,9 @@ class TestFinalReport(object):
 
 		assert report.snp_data.columns.difference(_fields).empty
 
-	@pytest.mark.parametrize("report", [("AB", "Top")], indirect=True)
+	@pytest.mark.parametrize(
+		"report", [{'allele': ("AB", "Top")}], indirect=True
+	)
 	def test_sample_allele_tuple(self, report: FinalReport) -> None:
 		report.handle(DIR_FILES / "fr/file4.txt", None)
 
@@ -171,7 +175,9 @@ class TestFinalReport(object):
 
 		assert report.snp_data.columns.difference(_fields).empty
 
-	@pytest.mark.parametrize("report", [{"AB", "Top"}], indirect=True)
+	@pytest.mark.parametrize(
+		"report", [{'allele': {"AB", "Top"}}], indirect=True
+	)
 	def test_sample_allele_set(self, report: FinalReport) -> None:
 		report.handle(DIR_FILES / "fr/file4.txt", None)
 
@@ -182,7 +188,7 @@ class TestFinalReport(object):
 
 		assert report.snp_data.columns.difference(_fields).empty
 
-	@pytest.mark.parametrize("report", ["GG"], indirect=True)
+	@pytest.mark.parametrize("report", [{'allele': "GG"}], indirect=True)
 	def test_sample_allele_not_exist(self, report: FinalReport) -> None:
 
 		with pytest.raises(
@@ -190,8 +196,8 @@ class TestFinalReport(object):
 		):
 			report.handle(DIR_FILES / "fr/file4.txt", None)
 
-	@pytest.mark.parametrize("report", ["AB"], indirect=True)
-	def test_7(self, report: FinalReport) -> None:
+	@pytest.mark.parametrize("report", [{'allele': ["AB"]}], indirect=True)
+	def test_on_ru_symbol(self, report: FinalReport) -> None:
 		""" An error is checked if the name of the number is Kirilitsa """
 
 		with pytest.raises(
@@ -201,10 +207,10 @@ class TestFinalReport(object):
 				DIR_FILES / "fr/file7.txt", DIR_FILES / "fr/file7.xlsx"
 			)
 
-		assert report.snp_data is None
+		assert report.snp_data is None and len(report.header) == 0
 
-	@pytest.mark.parametrize("report", ["AB"], indirect=True)
-	def test_8(self, report: FinalReport) -> None:
+	@pytest.mark.parametrize("report", [{'allele': ["AB"]}], indirect=True)
+	def test_on_nan_in_sid_file_xlsx(self, report: FinalReport) -> None:
 		""" Checking for processing empty values in SID """
 
 		report.handle(
@@ -212,18 +218,89 @@ class TestFinalReport(object):
 			DIR_FILES / "fr/file8.xlsx"
 		)
 
-		assert report.snp_data is not None
-		assert not report.snp_data.empty
+		assert report.snp_data is not None and not report.snp_data.empty
 		assert report.snp_data['Sample ID'].isna().any()
 
-	@pytest.mark.parametrize("report", ["AB"], indirect=True)
-	def test_9(self, report: FinalReport) -> None:
+	@pytest.mark.parametrize("report", [{'allele': "AB"}], indirect=True)
+	def test_on_nan_in_fr(self, report: FinalReport) -> None:
 		""" Checking for missing values in SID """
 		report.handle(
 			DIR_FILES / "fr/file9.txt",
 			DIR_FILES / "fr/file9.xlsx"
 		)
 
-		assert report.snp_data is not None
-		assert not report.snp_data.empty
+		assert report.snp_data is not None and not report.snp_data.empty
 		assert report.snp_data['Sample ID'].isna().any()
+
+	@pytest.mark.parametrize(
+		"report",
+		[{'allele': "AB", 'usecols': ['SNP Name', 'Sample ID']}],
+		indirect=True
+	)
+	def test_raise_use_allele_and_usecols(self, report: FinalReport) -> None:
+		""" Checking for allele and usecols sharing error """
+
+		with pytest.raises(
+			Exception, match="Error. Usecols is used for allele is none."
+		):
+			report.handle(
+				DIR_FILES / "fr/file1.txt", DIR_FILES / "fr/file1.xlsx"
+			)
+
+		assert report.snp_data is None and len(report.header) == 0
+
+	@pytest.mark.parametrize(
+		"report",
+		[{'usecols': ['SNP Name', 'Sample ID']}],
+		indirect=True
+	)
+	def test_using_usecols(self, report: FinalReport) -> None:
+		""" Checking for use setting usecols """
+
+		_fields = ['SNP Name', 'Sample ID']
+
+		report.handle(
+			DIR_FILES / "fr/file1.txt", DIR_FILES / "fr/file1.xlsx"
+		)
+
+		assert not report.snp_data.empty and len(report.header) != 0
+		assert report.snp_data.columns.difference(_fields).empty
+
+	@pytest.mark.parametrize(
+		"report",
+		[{
+			'usecols': ['SNP Name', 'Sample ID'],
+			'dtype': {'SNP Name': 'category'}
+		}],
+		indirect=True
+	)
+	def test_using_dtype_and_combo_usecols(self, report: FinalReport) -> None:
+		""" Checking the use of dtype """
+
+		_fields = ['SNP Name', 'Sample ID']
+
+		report.handle(
+			DIR_FILES / "fr/file1.txt", DIR_FILES / "fr/file1.xlsx"
+		)
+
+		assert not report.snp_data.empty and len(report.header) != 0
+		assert report.snp_data.columns.difference(_fields).empty
+		assert report.snp_data['SNP Name'].dtypes == 'category'
+
+	@pytest.mark.parametrize(
+		"report",
+		[{
+			'allele': "AB",
+			'dtype': {'SNP Name': 'category'}
+		}],
+		indirect=True
+	)
+	def test_using_dtype_and_combo_usecols(self, report: FinalReport) -> None:
+		""" Checking the use combo dtype and allele """
+
+		report.handle(
+			DIR_FILES / "fr/file1.txt", DIR_FILES / "fr/file1.xlsx"
+		)
+
+		assert not report.snp_data.empty and len(report.header) != 0
+		assert report.snp_data['SNP Name'].dtypes == 'category'