gtfparse 1.3.0__tar.gz → 2.1.0__tar.gz

{gtfparse-1.3.0 → gtfparse-2.1.0}/PKG-INFO

@@ -1,10 +1,10 @@
 Metadata-Version: 2.1
 Name: gtfparse
-Version: 1.3.0
-Summary: GTF Parsing
-Home-page: https://github.com/openvax/gtfparse
-Author: Alex Rubinsteyn
-License: http://www.apache.org/licenses/LICENSE-2.0.html
+Version: 2.1.0
+Summary: Parsing library for extracting data frames of genomic features from GTF files
+Author-email: Alex Rubinsteyn <alex.rubinsteyn@unc.edu>
+Project-URL: Homepage, https://github.com/openvax/gtfparse
+Project-URL: Bug Tracker, https://github.com/openvax/gtfparse
 Classifier: Development Status :: 4 - Beta
 Classifier: Environment :: Console
 Classifier: Operating System :: OS Independent
@@ -12,8 +12,11 @@ Classifier: Intended Audience :: Science/Research
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Programming Language :: Python
 Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
+Requires-Python: >=3.7
 Description-Content-Type: text/markdown
 License-File: LICENSE
+Requires-Dist: polars<0.21.0,>=0.20.2
+Requires-Dist: pyarrow<14.1.0,>=14.0.2
 
 [![Build Status](https://travis-ci.org/openvax/gtfparse.svg?branch=master)](https://travis-ci.org/openvax/gtfparse) [![Coverage Status](https://coveralls.io/repos/openvax/gtfparse/badge.svg?branch=master&service=github)](https://coveralls.io/github/openvax/gtfparse?branch=master)
 <a href="https://pypi.python.org/pypi/gtfparse/">

{gtfparse-1.3.0 → gtfparse-2.1.0}/gtfparse/__init__.py

@@ -12,17 +12,26 @@
 
 from .attribute_parsing import expand_attribute_strings
 from .create_missing_features import create_missing_features
-from .required_columns import REQUIRED_COLUMNS
 from .parsing_error import ParsingError
-from .read_gtf import read_gtf, parse_gtf, parse_gtf_and_expand_attributes
+from .read_gtf import (
+    read_gtf, 
+    parse_gtf,
+    parse_gtf_pandas, 
+    parse_gtf_and_expand_attributes, 
+    REQUIRED_COLUMNS,
+)
 
+__version__ = "2.1.0"
 
 __all__ = [
+    "__version__",
     "expand_attribute_strings",
     "create_missing_features",
-    "parse_gtf",
+
     "parse_gtf_and_expand_attributes",
     "REQUIRED_COLUMNS",
     "ParsingError",
     "read_gtf",
+    "parse_gtf",
+    "parse_gtf_pandas",
 ]

{gtfparse-1.3.0 → gtfparse-2.1.0}/gtfparse/attribute_parsing.py

@@ -18,9 +18,10 @@ logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
 
 
+
 def expand_attribute_strings(
         attribute_strings,
-        quote_char='\"',
+        quote_char="'",
         missing_value="",
         usecols=None):
     """
@@ -64,10 +65,11 @@ def expand_attribute_strings(
     # using a local dictionary, hence the two dictionaries below
     # and pair of try/except blocks in the loop.
     column_interned_strings = {}
-    value_interned_strings = {}
 
-    for (i, attribute_string) in enumerate(attribute_strings):
-        for kv in attribute_string.split(";"):
+    for (i, kv_strings) in enumerate(attribute_strings):
+        if type(kv_strings) is str:
+            kv_strings = kv_strings.split(";")
+        for kv in kv_strings:
             # We're slicing the first two elements out of split() because
             # Ensembl release 79 added values like:
             #   transcript_support_level "1 (assigned to previous version 5)";
@@ -88,28 +90,25 @@ def expand_attribute_strings(
             if usecols is not None and column_name not in usecols:
                 continue
 
+            if value[0] == quote_char:
+                value = value.replace(quote_char, "")
+                
             try:
                 column = extra_columns[column_name]
+                # if an attribute is used repeatedly then
+                # keep track of all its values in a list
+                old_value = column[i]
+                if old_value is missing_value:
+                    column[i] = value
+                else:
+                    column[i] = "%s,%s" % (old_value, value)
             except KeyError:
                 column = [missing_value] * n
+                column[i] = value
                 extra_columns[column_name] = column
                 column_order.append(column_name)
 
-            value = value.replace(quote_char, "") if value.startswith(quote_char) else value
-
-            try:
-                value = value_interned_strings[value]
-            except KeyError:
-                value = intern(str(value))
-                value_interned_strings[value] = value
 
-            # if an attribute is used repeatedly then
-            # keep track of all its values in a list
-            old_value = column[i]
-            if old_value is missing_value:
-                column[i] = value
-            else:
-                column[i] = "%s,%s" % (old_value, value)
 
     logging.info("Extracted GTF attributes: %s" % column_order)
     return OrderedDict(

{gtfparse-1.3.0 → gtfparse-2.1.0}/gtfparse/create_missing_features.py

@@ -55,7 +55,7 @@ def create_missing_features(
     extra_dataframes = []
 
     existing_features = set(dataframe["feature"])
-    existing_columns = set(dataframe.keys())
+    existing_columns = set(dataframe.columns)
 
     for (feature_name, groupby_key) in unique_keys.items():
         if feature_name in existing_features:

gtfparse-2.1.0/gtfparse/read_gtf.py

@@ -0,0 +1,297 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+from os.path import exists
+from io import StringIO
+import gzip 
+
+import polars 
+
+from .attribute_parsing import expand_attribute_strings
+from .parsing_error import ParsingError
+
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+"""
+Columns of a GTF file:
+
+    seqname   - name of the chromosome or scaffold; chromosome names
+                without a 'chr' in Ensembl (but sometimes with a 'chr'
+                elsewhere)
+    source    - name of the program that generated this feature, or
+                the data source (database or project name)
+    feature   - feature type name.
+                Features currently in Ensembl GTFs:
+                    gene
+                    transcript
+                    exon
+                    CDS
+                    Selenocysteine
+                    start_codon
+                    stop_codon
+                    UTR
+                Older Ensembl releases may be missing some of these features.
+    start     - start position of the feature, with sequence numbering
+                starting at 1.
+    end       - end position of the feature, with sequence numbering
+                starting at 1.
+    score     - a floating point value indiciating the score of a feature
+    strand    - defined as + (forward) or - (reverse).
+    frame     - one of '0', '1' or '2'. Frame indicates the number of base pairs
+                before you encounter a full codon. '0' indicates the feature
+                begins with a whole codon. '1' indicates there is an extra
+                base (the 3rd base of the prior codon) at the start of this feature.
+                '2' indicates there are two extra bases (2nd and 3rd base of the
+                prior exon) before the first codon. All values are given with
+                relation to the 5' end.
+    attribute - a semicolon-separated list of tag-value pairs (separated by a space),
+                providing additional information about each feature. A key can be
+                repeated multiple times.
+
+(from ftp://ftp.ensembl.org/pub/release-75/gtf/homo_sapiens/README)
+"""
+
+REQUIRED_COLUMNS = [
+    "seqname",
+    "source",
+    "feature",
+    "start",
+    "end",
+    "score",
+    "strand",
+    "frame",
+    "attribute",
+]
+
+
+def parse_with_polars_lazy(
+        filepath_or_buffer,
+        split_attributes=True,
+        features=None,
+        fix_quotes_columns=["attribute"]):
+    # use a global string cache so that all strings get intern'd into
+    # a single numbering system
+    polars.enable_string_cache()
+    kwargs = dict(
+        has_header=False,
+        separator="\t",
+        comment_prefix="#",
+        null_values=".",
+        dtypes={
+            "seqname": polars.Categorical, 
+            "source": polars.Categorical, 
+            
+            "start": polars.Int64,
+            "end": polars.Int64,
+            "score": polars.Float32,
+
+            "feature": polars.Categorical, 
+            "strand": polars.Categorical, 
+            "frame": polars.UInt32,
+        })
+    try:
+        if type(filepath_or_buffer) is StringIO:
+            df = polars.read_csv(
+                filepath_or_buffer,
+                new_columns=REQUIRED_COLUMNS,
+                **kwargs).lazy()
+        elif filepath_or_buffer.endswith(".gz") or filepath_or_buffer.endswith(".gzip"):
+            with gzip.open(filepath_or_buffer) as f:
+                df = polars.read_csv(
+                    f,
+                    new_columns=REQUIRED_COLUMNS,
+                    **kwargs).lazy()
+        else:
+            df = polars.scan_csv(
+                filepath_or_buffer, 
+                with_column_names=lambda cols: REQUIRED_COLUMNS,
+                **kwargs).lazy()
+    except polars.ShapeError:
+        raise ParsingError("Wrong number of columns")
+
+    df = df.with_columns([
+        polars.col("frame").fill_null(0),
+        polars.col("attribute").str.replace_all('"', "'")
+    ])
+    
+    for fix_quotes_column in fix_quotes_columns:
+        # Catch mistaken semicolons by replacing "xyz;" with "xyz"
+        # Required to do this since the Ensembl GTF for Ensembl
+        # release 78 has mistakes such as:
+        #   gene_name = "PRAMEF6;" transcript_name = "PRAMEF6;-201"
+        df = df.with_columns([
+            polars.col(fix_quotes_column).str.replace(';\"', '\"').str.replace(";-", "-")
+        ])
+
+    if features is not None:
+        features = sorted(set(features))
+        df = df.filter(polars.col("feature").is_in(features))
+
+
+    if split_attributes:
+        df = df.with_columns([
+            polars.col("attribute").str.split(";").alias("attribute_split")
+        ])
+    return df
+
+def parse_gtf(
+        filepath_or_buffer, 
+        split_attributes=True, 
+        features=None,
+        fix_quotes_columns=["attribute"]):
+    df_lazy = parse_with_polars_lazy(
+        filepath_or_buffer=filepath_or_buffer,
+        split_attributes=split_attributes,
+        features=features,
+        fix_quotes_columns=fix_quotes_columns)
+    return df_lazy.collect()
+
+def parse_gtf_pandas(*args, **kwargs):
+    return parse_gtf(*args, **kwargs).to_pandas()
+
+    
+def parse_gtf_and_expand_attributes(
+        filepath_or_buffer,
+        restrict_attribute_columns=None,
+        features=None):
+    """
+    Parse lines into column->values dictionary and then expand
+    the 'attribute' column into multiple columns. This expansion happens
+    by replacing strings of semi-colon separated key-value values in the
+    'attribute' column with one column per distinct key, with a list of
+    values for each row (using None for rows where key didn't occur).
+
+    Parameters
+    ----------
+    filepath_or_buffer : str or buffer object
+
+    chunksize : int
+
+    restrict_attribute_columns : list/set of str or None
+        If given, then only use these attribute columns.
+
+    features : set or None
+        Ignore entries which don't correspond to one of the supplied features
+    """
+    df = parse_gtf(
+        filepath_or_buffer=filepath_or_buffer, 
+        features=features,
+        split_attributes=True)
+    if type(restrict_attribute_columns) is str:
+        restrict_attribute_columns = {restrict_attribute_columns}
+    elif restrict_attribute_columns:
+        restrict_attribute_columns = set(restrict_attribute_columns)
+    df.drop_in_place("attribute")
+    attribute_pairs = df.drop_in_place("attribute_split")
+    return df.with_columns([
+        polars.Series(k, vs)
+        for (k, vs) in 
+        expand_attribute_strings(attribute_pairs).items()
+        if restrict_attribute_columns is None or k in restrict_attribute_columns
+    ])
+    
+
+def read_gtf(
+        filepath_or_buffer,
+        expand_attribute_column=True,
+        infer_biotype_column=False,
+        column_converters={},
+        usecols=None,
+        features=None,
+        result_type='polars'):
+    """
+    Parse a GTF into a dictionary mapping column names to sequences of values.
+
+    Parameters
+    ----------
+    filepath_or_buffer : str or buffer object
+        Path to GTF file (may be gzip compressed) or buffer object
+        such as StringIO
+
+    expand_attribute_column : bool
+        Replace strings of semi-colon separated key-value values in the
+        'attribute' column with one column per distinct key, with a list of
+        values for each row (using None for rows where key didn't occur).
+
+    infer_biotype_column : bool
+        Due to the annoying ambiguity of the second GTF column across multiple
+        Ensembl releases, figure out if an older GTF's source column is actually
+        the gene_biotype or transcript_biotype.
+
+    column_converters : dict, optional
+        Dictionary mapping column names to conversion functions. Will replace
+        empty strings with None and otherwise passes them to given conversion
+        function.
+
+    usecols : list of str or None
+        Restrict which columns are loaded to the give set. If None, then
+        load all columns.
+
+    features : set of str or None
+        Drop rows which aren't one of the features in the supplied set
+
+    result_type : One of 'polars', 'pandas', or 'dict'
+        Default behavior is to return a Polars DataFrame, but will convert to 
+        Pandas DataFrame or dictionary if specified.
+    """
+    if type(filepath_or_buffer) is str and not exists(filepath_or_buffer):
+        raise ValueError("GTF file does not exist: %s" % filepath_or_buffer)
+
+    if expand_attribute_column:
+        result_df = parse_gtf_and_expand_attributes(
+            filepath_or_buffer,
+            restrict_attribute_columns=usecols,
+            features=features)
+    else:
+        result_df = parse_gtf(result_df, features=features)
+
+    result_df = result_df.with_columns(
+        [
+            polars.col(column_name).map_elements(lambda x: column_type(x) if len(x) > 0 else None)
+            for column_name, column_type in column_converters.items()
+        ]
+    )
+
+    # Hackishly infer whether the values in the 'source' column of this GTF
+    # are actually representing a biotype by checking for the most common
+    # gene_biotype and transcript_biotype value 'protein_coding'
+    if infer_biotype_column:
+        unique_source_values = set(result_df["source"])
+        if "protein_coding" in unique_source_values:
+            column_names = set(result_df.columns)
+            # Disambiguate between the two biotypes by checking if
+            # gene_biotype is already present in another column. If it is,
+            # the 2nd column is the transcript_biotype (otherwise, it's the
+            # gene_biotype)
+            if "gene_biotype" not in column_names:
+                logging.info("Using column 'source' to replace missing 'gene_biotype'")
+                result_df = result_df.with_column(polars.col("source").alias("gene_biotype"))
+            if "transcript_biotype" not in column_names:
+                logging.info("Using column 'source' to replace missing 'transcript_biotype'")
+                result_df = result_df.with_column(polars.col("source").alias("transcript_biotype"))
+
+    if usecols is not None:
+        column_names = set(result_df.columns)
+        valid_columns = [c for c in usecols if c in column_names]
+        result_df = result_df.select(valid_columns)
+
+    if result_type == "pandas":
+        result = result_df.to_pandas()
+    elif result_type == "polars":
+        result = result_df
+    elif result_type == "dict":
+        result = result_df.to_dict()
+    return result

{gtfparse-1.3.0 → gtfparse-2.1.0}/gtfparse.egg-info/PKG-INFO

@@ -1,10 +1,10 @@
 Metadata-Version: 2.1
 Name: gtfparse
-Version: 1.3.0
-Summary: GTF Parsing
-Home-page: https://github.com/openvax/gtfparse
-Author: Alex Rubinsteyn
-License: http://www.apache.org/licenses/LICENSE-2.0.html
+Version: 2.1.0
+Summary: Parsing library for extracting data frames of genomic features from GTF files
+Author-email: Alex Rubinsteyn <alex.rubinsteyn@unc.edu>
+Project-URL: Homepage, https://github.com/openvax/gtfparse
+Project-URL: Bug Tracker, https://github.com/openvax/gtfparse
 Classifier: Development Status :: 4 - Beta
 Classifier: Environment :: Console
 Classifier: Operating System :: OS Independent
@@ -12,8 +12,11 @@ Classifier: Intended Audience :: Science/Research
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Programming Language :: Python
 Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
+Requires-Python: >=3.7
 Description-Content-Type: text/markdown
 License-File: LICENSE
+Requires-Dist: polars<0.21.0,>=0.20.2
+Requires-Dist: pyarrow<14.1.0,>=14.0.2
 
 [![Build Status](https://travis-ci.org/openvax/gtfparse.svg?branch=master)](https://travis-ci.org/openvax/gtfparse) [![Coverage Status](https://coveralls.io/repos/openvax/gtfparse/badge.svg?branch=master&service=github)](https://coveralls.io/github/openvax/gtfparse?branch=master)
 <a href="https://pypi.python.org/pypi/gtfparse/">

{gtfparse-1.3.0 → gtfparse-2.1.0}/gtfparse.egg-info/SOURCES.txt

@@ -1,22 +1,22 @@
 LICENSE
 README.md
-setup.py
+pyproject.toml
+requirements.txt
 gtfparse/__init__.py
 gtfparse/attribute_parsing.py
 gtfparse/create_missing_features.py
 gtfparse/parsing_error.py
 gtfparse/read_gtf.py
-gtfparse/required_columns.py
-gtfparse/version.py
 gtfparse.egg-info/PKG-INFO
 gtfparse.egg-info/SOURCES.txt
 gtfparse.egg-info/dependency_links.txt
 gtfparse.egg-info/requires.txt
 gtfparse.egg-info/top_level.txt
-test/test_create_missing_features.py
-test/test_ensembl_gtf.py
-test/test_expand_attributes.py
-test/test_multiple_values_for_tag_attribute.py
-test/test_parse_gtf_lines.py
-test/test_read_stringtie_gtf.py
-test/test_refseq_gtf.py
+gtfparse/../requirements.txt
+tests/test_create_missing_features.py
+tests/test_ensembl_gtf.py
+tests/test_expand_attributes.py
+tests/test_multiple_values_for_tag_attribute.py
+tests/test_parse_gtf_lines.py
+tests/test_read_stringtie_gtf.py
+tests/test_refseq_gtf.py

gtfparse-2.1.0/gtfparse.egg-info/requires.txt

@@ -0,0 +1,2 @@
+polars<0.21.0,>=0.20.2
+pyarrow<14.1.0,>=14.0.2

gtfparse-2.1.0/pyproject.toml

@@ -0,0 +1,27 @@
+[project]
+name = "gtfparse"
+requires-python = ">=3.7"
+authors = [ {name="Alex Rubinsteyn", email="alex.rubinsteyn@unc.edu" } ]
+description = "Parsing library for extracting data frames of genomic features from GTF files"
+classifiers = [
+            'Development Status :: 4 - Beta',
+            'Environment :: Console',
+            'Operating System :: OS Independent',
+            'Intended Audience :: Science/Research',
+            'License :: OSI Approved :: Apache Software License',
+            'Programming Language :: Python',
+            'Topic :: Scientific/Engineering :: Bio-Informatics',
+]
+readme = "README.md"
+dynamic = ["version", "dependencies"]
+
+[tool.setuptools.dynamic]
+version = {attr = "gtfparse.__version__"}
+dependencies = {file = ["requirements.txt"]}
+
+[tool.setuptools]
+packages = ["gtfparse"]
+
+[project.urls]
+"Homepage" = "https://github.com/openvax/gtfparse"
+"Bug Tracker" = "https://github.com/openvax/gtfparse"

gtfparse-2.1.0/requirements.txt

@@ -0,0 +1,2 @@
+polars>=0.20.2,<0.21.0
+pyarrow>=14.0.2,<14.1.0

{gtfparse-1.3.0/test → gtfparse-2.1.0/tests}/test_create_missing_features.py

@@ -1,5 +1,5 @@
 from gtfparse import create_missing_features, parse_gtf_and_expand_attributes
-from six import StringIO
+from io import StringIO
 
 # two lines from the Ensembl 54 human GTF containing only a stop_codon and
 # exon features, but from which gene and transcript information could be
@@ -19,6 +19,7 @@ GTF_TEXT = "\n".join([
 
 
 GTF_DATAFRAME = parse_gtf_and_expand_attributes(StringIO(GTF_TEXT))
+GTF_DATAFRAME = GTF_DATAFRAME.to_pandas()
 
 def test_create_missing_features_identity():
     df_should_be_same = create_missing_features(GTF_DATAFRAME, {})

{gtfparse-1.3.0/test → gtfparse-2.1.0/tests}/test_ensembl_gtf.py

@@ -1,6 +1,6 @@
 from data import data_path
 from gtfparse import read_gtf
-from nose.tools import eq_
+
 
 ENSEMBL_GTF_PATH = data_path("ensembl_grch37.head.gtf")
 
@@ -18,7 +18,7 @@ EXPECTED_FEATURES = set([
 def test_ensembl_gtf_columns():
     df = read_gtf(ENSEMBL_GTF_PATH)
     features = set(df["feature"])
-    eq_(features, EXPECTED_FEATURES)
+    assert features == EXPECTED_FEATURES
 
 # first 1000 lines of GTF only contained these genes
 EXPECTED_GENE_NAMES = {

gtfparse-2.1.0/tests/test_expand_attributes.py

@@ -0,0 +1,36 @@
+from gtfparse import expand_attribute_strings
+
+def test_attributes_in_quotes():
+    attributes = [
+        "gene_id \"ENSG001\"; tag \"bogotron\"; version \"1\";",
+        "gene_id \"ENSG002\"; tag \"wolfpuppy\"; version \"2\";"
+    ]
+    parsed_dict = expand_attribute_strings(attributes, quote_char='"')
+    assert list(sorted(parsed_dict.keys())), ["gene_id", "tag", "version"]
+    assert parsed_dict["gene_id"] == ["ENSG001", "ENSG002"]
+    assert parsed_dict["tag"] == ["bogotron", "wolfpuppy"]
+    assert parsed_dict["version"] == ["1", "2"]
+
+
+def test_attributes_without_quotes():
+    attributes = [
+        "gene_id ENSG001; tag bogotron; version 1;",
+        "gene_id ENSG002; tag wolfpuppy; version 2"
+    ]
+    parsed_dict = expand_attribute_strings(attributes)
+    assert list(sorted(parsed_dict.keys())) == ["gene_id", "tag", "version"]
+    assert parsed_dict["gene_id"] == ["ENSG001", "ENSG002"]
+    assert parsed_dict["tag"] == ["bogotron", "wolfpuppy"]
+    assert parsed_dict["version"] == ["1", "2"]
+
+
+def test_optional_attributes():
+    attributes = [
+        "gene_id ENSG001; sometimes-present bogotron;",
+        "gene_id ENSG002;",
+        "gene_id ENSG003; sometimes-present wolfpuppy;",
+    ]
+    parsed_dict = expand_attribute_strings(attributes)
+    assert list(sorted(parsed_dict.keys())) ==  ["gene_id", "sometimes-present"]
+    assert parsed_dict["gene_id"] ==  ["ENSG001", "ENSG002", "ENSG003"]
+    assert parsed_dict["sometimes-present"] ==  ["bogotron", "", "wolfpuppy"]

{gtfparse-1.3.0/test → gtfparse-2.1.0/tests}/test_multiple_values_for_tag_attribute.py

@@ -1,6 +1,5 @@
-from six import StringIO
+from io import StringIO
 from gtfparse import parse_gtf_and_expand_attributes
-from nose.tools import eq_
 
 # failing example from https://github.com/openvax/gtfparse/issues/2
 GTF_TEXT = (
@@ -15,23 +14,22 @@ GTF_TEXT = (
 def test_parse_tag_attributes():
     parsed = parse_gtf_and_expand_attributes(StringIO(GTF_TEXT))
     tag_column = parsed["tag"]
-    eq_(len(tag_column), 1)
+    assert len(tag_column) == 1
     tags = tag_column[0]
-    eq_(tags, 'cds_end_NF,mRNA_end_NF')
+    assert tags == 'cds_end_NF,mRNA_end_NF'
 
 def test_parse_tag_attributes_with_usecols():
     parsed = parse_gtf_and_expand_attributes(
         StringIO(GTF_TEXT),
         restrict_attribute_columns=["tag"])
     tag_column = parsed["tag"]
-    eq_(len(tag_column), 1)
+    assert len(tag_column) == 1
     tags = tag_column[0]
-    eq_(tags, 'cds_end_NF,mRNA_end_NF')
+    assert tags == 'cds_end_NF,mRNA_end_NF'
 
 def test_parse_tag_attributes_with_usecols_other_column():
     parsed = parse_gtf_and_expand_attributes(
         StringIO(GTF_TEXT),
         restrict_attribute_columns=["exon_id"])
-    tag_column = parsed.get("tag")
 
-    assert tag_column is None, "Expected 'tag' to get dropped but got %s" % (parsed,)
+    assert "tag" not in parsed, "Expected 'tag' to get dropped but got %s" % (parsed,)

{gtfparse-1.3.0/test → gtfparse-2.1.0/tests}/test_parse_gtf_lines.py

@@ -1,12 +1,11 @@
-import numpy as np
-from nose.tools import eq_, assert_raises
+from pytest import raises
 from gtfparse import (
     parse_gtf,
     parse_gtf_and_expand_attributes,
     REQUIRED_COLUMNS,
     ParsingError
 )
-from six import StringIO
+from io import StringIO 
 
 gtf_text = """
 # sample GTF data copied from:
@@ -16,7 +15,9 @@ gtf_text = """
 """
 
 def test_parse_gtf_lines_with_expand_attributes():
-    parsed_dict = parse_gtf_and_expand_attributes(StringIO(gtf_text))
+    df = parse_gtf_and_expand_attributes(StringIO(gtf_text))
+
+
     # excluding 'attribute' column from required names
     expected_columns = REQUIRED_COLUMNS[:8] + [
         "gene_id",
@@ -28,40 +29,31 @@ def test_parse_gtf_lines_with_expand_attributes():
         "transcript_source",
     ]
     # convert to list since Py3's dictionary keys are a distinct collection type
-    eq_(list(parsed_dict.keys()), expected_columns)
-    eq_(list(parsed_dict["seqname"]), ["1", "1"])
+    assert list(df.columns) ==  expected_columns
+    assert list(df["seqname"]) == ["1", "1"]
     # convert to list for comparison since numerical columns may be NumPy arrays
-    eq_(list(parsed_dict["start"]), [11869, 11869])
-    eq_(list(parsed_dict["end"]), [14409, 14409])
-    # can't compare NaN with equality
-    scores = list(parsed_dict["score"])
-    assert np.isnan(scores).all(), "Unexpected scores: %s" % scores
-    eq_(list(parsed_dict["gene_id"]), ["ENSG00000223972", "ENSG00000223972"])
-    eq_(list(parsed_dict["transcript_id"]), ["", "ENST00000456328"])
+    assert list(df["start"]) == [11869, 11869]
+    assert list(df["end"]) == [14409, 14409]
+
+    assert df["score"].is_null().all(), "Unexpected scores: %s" % (df["score"],)
+    assert list(df["gene_id"]) == ["ENSG00000223972", "ENSG00000223972"]
+    assert list(df["transcript_id"]) == ["", "ENST00000456328"]
 
 
 def test_parse_gtf_lines_without_expand_attributes():
-    parsed_dict = parse_gtf(StringIO(gtf_text))
+    df = parse_gtf(StringIO(gtf_text), split_attributes=False)
 
     # convert to list since Py3's dictionary keys are a distinct collection type
-    eq_(list(parsed_dict.keys()), REQUIRED_COLUMNS)
-    eq_(list(parsed_dict["seqname"]), ["1", "1"])
+    assert list(df.columns) == REQUIRED_COLUMNS
+    assert list(df["seqname"]) == ["1", "1"]
     # convert to list for comparison since numerical columns may be NumPy arrays
-    eq_(list(parsed_dict["start"]), [11869, 11869])
-    eq_(list(parsed_dict["end"]), [14409, 14409])
-    # can't compare NaN with equality
-    scores = list(parsed_dict["score"])
-    assert np.isnan(scores).all(), "Unexpected scores: %s" % scores
-    assert len(parsed_dict["attribute"]) == 2
-
-def test_parse_gtf_lines_error_too_many_fields():
-    bad_gtf_text = gtf_text.replace(" ", "\t")
-    # pylint: disable=no-value-for-parameter
-    with assert_raises(ParsingError):
-        parse_gtf(StringIO(bad_gtf_text))
+    assert list(df["start"]) == [11869, 11869]
+    assert list(df["end"]) == [14409, 14409]
+    assert df["score"].is_null().all(), "Unexpected scores: %s" % (df["score"],)
+    assert len(df["attribute"]) == 2
 
 def test_parse_gtf_lines_error_too_few_fields():
     bad_gtf_text = gtf_text.replace("\t", " ")
     # pylint: disable=no-value-for-parameter
-    with assert_raises(ParsingError):
+    with raises(ParsingError):
         parse_gtf(StringIO(bad_gtf_text))

gtfparse-1.3.0/gtfparse/read_gtf.py

@@ -1,243 +0,0 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import logging
-from os.path import exists
-
-from sys import intern
-import numpy as np
-import pandas as pd
-
-from .attribute_parsing import expand_attribute_strings
-from .parsing_error import ParsingError
-from .required_columns import REQUIRED_COLUMNS
-
-logging.basicConfig(level=logging.INFO)
-logger = logging.getLogger(__name__)
-
-
-def parse_gtf(
-        filepath_or_buffer,
-        chunksize=1024 * 1024,
-        features=None,
-        intern_columns=["seqname", "source", "strand", "frame"],
-        fix_quotes_columns=["attribute"]):
-    """
-    Parameters
-    ----------
-
-    filepath_or_buffer : str or buffer object
-
-    chunksize : int
-
-    features : set or None
-        Drop entries which aren't one of these features
-
-    intern_columns : list
-        These columns are short strings which should be interned
-
-    fix_quotes_columns : list
-        Most commonly the 'attribute' column which had broken quotes on
-        some Ensembl release GTF files.
-    """
-    if features is not None:
-        features = set(features)
-
-    dataframes = []
-
-    def parse_frame(s):
-        if s == ".":
-            return 0
-        else:
-            return int(s)
-
-    # GTF columns:
-    # 1) seqname: str ("1", "X", "chrX", etc...)
-    # 2) source : str
-    #      Different versions of GTF use second column as of:
-    #      (a) gene biotype
-    #      (b) transcript biotype
-    #      (c) the annotation source
-    #      See: https://www.biostars.org/p/120306/#120321
-    # 3) feature : str ("gene", "transcript", &c)
-    # 4) start : int
-    # 5) end : int
-    # 6) score : float or "."
-    # 7) strand : "+", "-", or "."
-    # 8) frame : 0, 1, 2 or "."
-    # 9) attribute : key-value pairs separated by semicolons
-    # (see more complete description in docstring at top of file)
-
-    chunk_iterator = pd.read_csv(
-        filepath_or_buffer,
-        sep="\t",
-        comment="#",
-        names=REQUIRED_COLUMNS,
-        skipinitialspace=True,
-        skip_blank_lines=True,
-        on_bad_lines="error",
-        chunksize=chunksize,
-        engine="c",
-        dtype={
-            "start": np.int64,
-            "end": np.int64,
-            "score": np.float32,
-            "seqname": str,
-        },
-        na_values=".",
-        converters={"frame": parse_frame})
-    dataframes = []
-    try:
-        for df in chunk_iterator:
-            for intern_column in intern_columns:
-                df[intern_column] = [intern(str(s)) for s in df[intern_column]]
-
-            # compare feature strings after interning
-            if features is not None:
-                df = df[df["feature"].isin(features)]
-
-            for fix_quotes_column in fix_quotes_columns:
-                # Catch mistaken semicolons by replacing "xyz;" with "xyz"
-                # Required to do this since the Ensembl GTF for Ensembl
-                # release 78 has mistakes such as:
-                #   gene_name = "PRAMEF6;" transcript_name = "PRAMEF6;-201"
-                df[fix_quotes_column] = [
-                    s.replace(';\"', '\"').replace(";-", "-")
-                    for s in df[fix_quotes_column]
-                ]
-            dataframes.append(df)
-    except Exception as e:
-        raise ParsingError(str(e))
-    df = pd.concat(dataframes)
-    return df
-
-
-def parse_gtf_and_expand_attributes(
-        filepath_or_buffer,
-        chunksize=1024 * 1024,
-        restrict_attribute_columns=None,
-        features=None):
-    """
-    Parse lines into column->values dictionary and then expand
-    the 'attribute' column into multiple columns. This expansion happens
-    by replacing strings of semi-colon separated key-value values in the
-    'attribute' column with one column per distinct key, with a list of
-    values for each row (using None for rows where key didn't occur).
-
-    Parameters
-    ----------
-    filepath_or_buffer : str or buffer object
-
-    chunksize : int
-
-    restrict_attribute_columns : list/set of str or None
-        If given, then only usese attribute columns.
-
-    features : set or None
-        Ignore entries which don't correspond to one of the supplied features
-    """
-    result = parse_gtf(
-        filepath_or_buffer,
-        chunksize=chunksize,
-        features=features)
-    attribute_values = result["attribute"]
-    del result["attribute"]
-    for column_name, values in expand_attribute_strings(
-            attribute_values, usecols=restrict_attribute_columns).items():
-        result[column_name] = values
-    return result
-
-
-def read_gtf(
-        filepath_or_buffer,
-        expand_attribute_column=True,
-        infer_biotype_column=False,
-        column_converters={},
-        usecols=None,
-        features=None,
-        chunksize=1024 * 1024):
-    """
-    Parse a GTF into a dictionary mapping column names to sequences of values.
-
-    Parameters
-    ----------
-    filepath_or_buffer : str or buffer object
-        Path to GTF file (may be gzip compressed) or buffer object
-        such as StringIO
-
-    expand_attribute_column : bool
-        Replace strings of semi-colon separated key-value values in the
-        'attribute' column with one column per distinct key, with a list of
-        values for each row (using None for rows where key didn't occur).
-
-    infer_biotype_column : bool
-        Due to the annoying ambiguity of the second GTF column across multiple
-        Ensembl releases, figure out if an older GTF's source column is actually
-        the gene_biotype or transcript_biotype.
-
-    column_converters : dict, optional
-        Dictionary mapping column names to conversion functions. Will replace
-        empty strings with None and otherwise passes them to given conversion
-        function.
-
-    usecols : list of str or None
-        Restrict which columns are loaded to the give set. If None, then
-        load all columns.
-
-    features : set of str or None
-        Drop rows which aren't one of the features in the supplied set
-
-    chunksize : int
-    """
-    if type(filepath_or_buffer) is str and not exists(filepath_or_buffer):
-        raise ValueError("GTF file does not exist: %s" % filepath_or_buffer)
-
-    if expand_attribute_column:
-        result_df = parse_gtf_and_expand_attributes(
-            filepath_or_buffer,
-            chunksize=chunksize,
-            restrict_attribute_columns=usecols,
-            features=features)
-    else:
-        result_df = parse_gtf(result_df, features=features)
-
-    for column_name, column_type in list(column_converters.items()):
-        result_df[column_name] = [
-            column_type(string_value) if len(string_value) > 0 else None
-            for string_value
-            in result_df[column_name]
-        ]
-
-    # Hackishly infer whether the values in the 'source' column of this GTF
-    # are actually representing a biotype by checking for the most common
-    # gene_biotype and transcript_biotype value 'protein_coding'
-    if infer_biotype_column:
-        unique_source_values = set(result_df["source"])
-        if "protein_coding" in unique_source_values:
-            column_names = set(result_df.columns)
-            # Disambiguate between the two biotypes by checking if
-            # gene_biotype is already present in another column. If it is,
-            # the 2nd column is the transcript_biotype (otherwise, it's the
-            # gene_biotype)
-            if "gene_biotype" not in column_names:
-                logging.info("Using column 'source' to replace missing 'gene_biotype'")
-                result_df["gene_biotype"] = result_df["source"]
-            if "transcript_biotype" not in column_names:
-                logging.info("Using column 'source' to replace missing 'transcript_biotype'")
-                result_df["transcript_biotype"] = result_df["source"]
-
-    if usecols is not None:
-        column_names = set(result_df.columns)
-        valid_columns = [c for c in usecols if c in column_names]
-        result_df = result_df[valid_columns]
-
-    return result_df

gtfparse-1.3.0/gtfparse/required_columns.py

@@ -1,62 +0,0 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""
-Columns of a GTF file:
-
-    seqname   - name of the chromosome or scaffold; chromosome names
-                without a 'chr' in Ensembl (but sometimes with a 'chr'
-                elsewhere)
-    source    - name of the program that generated this feature, or
-                the data source (database or project name)
-    feature   - feature type name.
-                Features currently in Ensembl GTFs:
-                    gene
-                    transcript
-                    exon
-                    CDS
-                    Selenocysteine
-                    start_codon
-                    stop_codon
-                    UTR
-                Older Ensembl releases may be missing some of these features.
-    start     - start position of the feature, with sequence numbering
-                starting at 1.
-    end       - end position of the feature, with sequence numbering
-                starting at 1.
-    score     - a floating point value indiciating the score of a feature
-    strand    - defined as + (forward) or - (reverse).
-    frame     - one of '0', '1' or '2'. Frame indicates the number of base pairs
-                before you encounter a full codon. '0' indicates the feature
-                begins with a whole codon. '1' indicates there is an extra
-                base (the 3rd base of the prior codon) at the start of this feature.
-                '2' indicates there are two extra bases (2nd and 3rd base of the
-                prior exon) before the first codon. All values are given with
-                relation to the 5' end.
-    attribute - a semicolon-separated list of tag-value pairs (separated by a space),
-                providing additional information about each feature. A key can be
-                repeated multiple times.
-
-(from ftp://ftp.ensembl.org/pub/release-75/gtf/homo_sapiens/README)
-"""
-
-REQUIRED_COLUMNS = [
-    "seqname",
-    "source",
-    "feature",
-    "start",
-    "end",
-    "score",
-    "strand",
-    "frame",
-    "attribute",
-]

gtfparse-1.3.0/gtfparse/version.py

@@ -1 +0,0 @@
-__version__ = "1.3.0"

gtfparse-1.3.0/gtfparse.egg-info/requires.txt

@@ -1,2 +0,0 @@
-numpy>=1.7
-pandas>=0.15

gtfparse-1.3.0/setup.py

@@ -1,61 +0,0 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from __future__ import print_function
-import os
-import re
-
-from setuptools import setup, find_packages
-
-readme_filename = "README.md"
-current_directory = os.path.dirname(__file__)
-readme_path = os.path.join(current_directory, readme_filename)
-
-readme_markdown = ""
-try:
-    with open(readme_path, 'r') as f:
-        readme_markdown = f.read()
-except Exception as e:
-    print(e)
-    print("Failed to open %s" % readme_path)
-
-with open('gtfparse/version.py', 'r') as f:
-    version = re.search(
-        r'^__version__\s*=\s*[\'"]([^\'"]*)[\'"]',
-        f.read(),
-        re.MULTILINE).group(1)
-
-if __name__ == '__main__':
-    setup(
-        name='gtfparse',
-        packages=find_packages(),
-        version=version,
-        description="GTF Parsing",
-        long_description=readme_markdown,
-        long_description_content_type='text/markdown',
-        url="https://github.com/openvax/gtfparse",
-        author="Alex Rubinsteyn",
-        license="http://www.apache.org/licenses/LICENSE-2.0.html",
-        classifiers=[
-            'Development Status :: 4 - Beta',
-            'Environment :: Console',
-            'Operating System :: OS Independent',
-            'Intended Audience :: Science/Research',
-            'License :: OSI Approved :: Apache Software License',
-            'Programming Language :: Python',
-            'Topic :: Scientific/Engineering :: Bio-Informatics',
-        ],
-        install_requires=[
-            'numpy>=1.7',
-            'pandas>=0.15',
-        ],
-    )

gtfparse-1.3.0/test/test_expand_attributes.py

@@ -1,37 +0,0 @@
-from gtfparse import expand_attribute_strings
-from nose.tools import eq_
-
-def test_attributes_in_quotes():
-    attributes = [
-        "gene_id \"ENSG001\"; tag \"bogotron\"; version \"1\";",
-        "gene_id \"ENSG002\"; tag \"wolfpuppy\"; version \"2\";"
-    ]
-    parsed_dict = expand_attribute_strings(attributes)
-    eq_(list(sorted(parsed_dict.keys())), ["gene_id", "tag", "version"])
-    eq_(parsed_dict["gene_id"], ["ENSG001", "ENSG002"])
-    eq_(parsed_dict["tag"], ["bogotron", "wolfpuppy"])
-    eq_(parsed_dict["version"], ["1", "2"])
-
-
-def test_attributes_without_quotes():
-    attributes = [
-        "gene_id ENSG001; tag bogotron; version 1;",
-        "gene_id ENSG002; tag wolfpuppy; version 2"
-    ]
-    parsed_dict = expand_attribute_strings(attributes)
-    eq_(list(sorted(parsed_dict.keys())), ["gene_id", "tag", "version"])
-    eq_(parsed_dict["gene_id"], ["ENSG001", "ENSG002"])
-    eq_(parsed_dict["tag"], ["bogotron", "wolfpuppy"])
-    eq_(parsed_dict["version"], ["1", "2"])
-
-
-def test_optional_attributes():
-    attributes = [
-        "gene_id ENSG001; sometimes-present bogotron;",
-        "gene_id ENSG002;",
-        "gene_id ENSG003; sometimes-present wolfpuppy;",
-    ]
-    parsed_dict = expand_attribute_strings(attributes)
-    eq_(list(sorted(parsed_dict.keys())), ["gene_id", "sometimes-present"])
-    eq_(parsed_dict["gene_id"], ["ENSG001", "ENSG002", "ENSG003"])
-    eq_(parsed_dict["sometimes-present"], ["bogotron", "", "wolfpuppy"])