pyham 2.0.0__py3-none-any.whl

pyham/parsers.py

@@ -0,0 +1,959 @@
+import logging
+from collections import defaultdict
+from xml.etree.ElementTree import XMLParser
+from tqdm.auto import tqdm
+
+from . import abstractgene
+from .genome import ExtantGenome
+from .taxonomy import build_taxon_node, Taxonomy
+from . import id_formats
+
+logger = logging.getLogger(__name__)
+
+
+def _label_and_species(node):
+    """Return a (label, species_names) pair describing an AbstractGene for conflict reporting."""
+    if isinstance(node, abstractgene.HOG):
+        species = [g.genome.name for g in node.get_all_descendant_genes()]
+        return node.hog_id if node.hog_id is not None else "<no id>", species
+    label = node.prot_id or node.gene_id or node.unique_id
+    return label, [node.genome.name]
+
+
+def short_level_name(genome):
+    """Name of a genome's level, abbreviated if it was synthesized from many child names."""
+    return abstractgene.short_level_name(genome.name)
+
+
+def _format_one_conflict(conflict, index=None, total=None):
+    header = f"Family {conflict.family_id} — {conflict.hog_id} claims level \"{conflict.resolved_level}\" ({conflict.claim_property})"
+    if index is not None:
+        header = f"[{index}/{total}] {header}"
+
+    lines = [header]
+    for label, species, detail in conflict.offending_members:
+        species_str = ", ".join(species)
+        if conflict.kind == "same_rank":
+            lines.append(f"      conflict: {label} ({species_str}) resolves to that SAME level.")
+        elif conflict.kind == "inverted":
+            lines.append(f"      conflict: \"{conflict.resolved_level}\" (claimed level) is itself a descendant of {label} ({species_str}) in the given tree.")
+        else:  # disjoint
+            lines.append(f"      conflict: {label} ({species_str}) is not below that level in the given tree.")
+        if detail:
+            lines.append(f"      reason:   {detail}")
+
+    if conflict.sibling_members:
+        sibling_str = "; ".join(f"{label} ({', '.join(species)})" for label, species in conflict.sibling_members)
+        lines.append(f"      also at \"{conflict.resolved_level}\" in this family: {sibling_str}")
+
+    return "\n".join(lines)
+
+
+def format_taxonomic_conflicts(conflicts: list, cap: int = 20) -> str:
+    """Render a list of :obj:`abstractgene.TaxonomicConflict` records into a human-readable report."""
+    if not conflicts:
+        return "No taxonomic conflicts."
+
+    if len(conflicts) == 1:
+        return _format_one_conflict(conflicts[0])
+
+    total = len(conflicts)
+    shown = conflicts[:cap]
+    blocks = [_format_one_conflict(c, i, total) for i, c in enumerate(shown, start=1)]
+    report = "\n\n".join(blocks)
+    if total > cap:
+        report += f"\n\n... and {total - cap} more conflict(s) not shown here; see the .conflicts attribute for the complete list."
+    report += f"\n\n{total} conflicts total." if total != 1 else f"\n\n{total} conflict total."
+    return report
+
+
+class OrthoXMLParser:
+
+    """
+    Custom parser for OrthoXML containing HOGs. It can take a FilterParser object to restrict the information to parse.
+     
+    The parse goes through the whole XML and create on the fly the required Ham objects:
+        - In the Xref/header, the parser creates the ExtantGenome and Gene objects.
+        - the taxonomy group and its taxon elements are parsed to create the taxonomy tree, if no species tree is provided.
+        - In the Groups section, it creates the HOGs with their hierarchy (parent/children links) and their related
+        AncestralGenomes.
+    
+
+    Attributes:
+        ham_object (:obj:`Ham`): Ham object to feed with created objects.
+        filterObj (:obj:`FilterParser`, optional): FilterParser object used to restrict the parsed information. Defaults
+        to None.
+        
+        extant_gene_map (:obj:`dict`): dictionary of gene unique id mapped to their related :obj:`Gene`.
+        external_id_mapper (:obj:`dict`): dictionary of xref id mapped to their list of possible unique ids.
+        toplevel_hogs (:obj:`dict`): dictionary of top level hog id mapped to their related :obj:`HOG`.
+        
+        cpt (:obj:`int`): Counter of parsed hogs.
+        hog_stack (:obj:`list`): Stack of hogs currently parsed. Reset at each top level hog change.
+        current_species (:obj:`ExtantGenome`): Pointer to the current species parsed during xref first step.
+        paralog_stack (:obj:`list`): Stack of position in hog stack when paralogy event occured and the related DuplicationNode.
+        Reset at each top level hog change.
+        in_paralogGroup (:obj:`int`): last position in the stack where a duplication  occured.
+        skip_this_hog (:obj:`Boolean`): Boolean to skip the current hog or not (used when filtering option is set).     
+    """
+
+    def __init__(self, ham_object, taxonomy=None, filterObject=None, id_schema: str = None, with_progress: bool = False,
+                 fail_fast: bool = False):
+
+        """
+        Args:
+            ham_object (:obj:`Ham`): Ham object to feed with created objects.
+            filterObject (:obj:`FilterParser`, optional): FilterParser object used to restrict the parsed information.
+            with_progress (:bool:, optional): Whether to display a tqdm progress bar whilst parsing. Per-species
+                and per-500-HOGs messages are logged at DEBUG level so they don't garble the bar at INFO.
+            Defaults to None.
+            fail_fast (:bool:, optional): If True, raise :obj:`abstractgene.TaxonomicConflictError` immediately at the
+            first taxonomic conflict found. If False (default), keep parsing and collect every conflict found across
+            the whole file, raising a single combined error once parsing completes.
+            id_schema (:obj:`str`, optional): the HOG id scheme used by the orthoxml file (one of
+            :obj:`pyham.id_formats.SCHEMES`), used to synthesize ids for HOG levels inserted because the
+            species tree implies them but the file doesn't. Defaults to :obj:`pyham.id_formats.GENERIC`
+            (no synthesis) if not given.
+        """
+        self.ham_object = ham_object
+        self.filterObj = filterObject
+        self.taxonomy = ham_object.taxonomy
+        self._id_schema = id_schema if id_schema is not None else id_formats.GENERIC
+
+        # usefull information
+        self.extant_gene_map = {}
+        self.external_id_mapper = defaultdict(list)
+        self.toplevel_hogs = {}
+
+        # On the fly variable
+        self.cpt = 0
+        self.hog_stack = []
+        self.paralog_stack = []
+        self.taxon_stack = []
+        self.current_species = None
+        self.in_paralogGroup = None
+        self.paralogyNode = None
+        self.skip_this_hog = False
+        self._genomes_to_add = []
+
+        # id of the top-level family being parsed (None between families), used to give errors context
+        self.current_family_id = None
+        self._family_index = 0
+        # OrthoXML species that couldn't be mapped onto the species tree, as (name, reason) pairs;
+        # collected over the whole <species> section and reported together before <groups>
+        self.unmapped_species = []
+        self.n_species = 0
+        # ids of HOGs collapsed into their parent because they sat at the same taxonomic level
+        self.collapsed_same_level_hogs = []
+
+        # taxonomic conflict tracking
+        self.fail_fast = fail_fast
+        self.taxonomic_conflicts = []
+        self._conflicted_children = set()
+
+        # counter for freshly-minted duplication-branch indices (see
+        # _assign_duplication_branch_ids) -- seeded far above any realistic real index so
+        # it can never collide with one; only needs to be distinct from other
+        # freshly-minted indices, which a monotonic counter guarantees.
+        self._next_synthetic_dup_idx = 10 ** 4
+
+        # save progress bar option
+        self.with_progress = with_progress
+
+    def _build_gene(self, attrib):
+        gene = abstractgene.Gene(**attrib)
+        self.current_species.add_gene(gene)
+        self.extant_gene_map[gene.unique_id] = gene
+        for type, Id in attrib.items():
+            if type != "id":
+                self.external_id_mapper.setdefault(Id, []).append(gene.unique_id)
+
+    def _build_hog(self, attrib):
+        hog = abstractgene.HOG(arose_by_duplication=False, **attrib)
+
+        if self.in_paralogGroup == len(self.hog_stack):
+            self.paralogyNode.add_child(hog)
+
+        if len(self.hog_stack) > 0:
+            self.hog_stack[-1].add_child(hog)
+        self.hog_stack.append(hog)
+
+    def _update_progress_bar(self, tag):
+        if tag == "{http://orthoXML.org/2011/}species":
+            if not hasattr(self, 'sp_pbar'):
+                self.sp_pbar = tqdm(desc='Parsing Species')
+            self.sp_pbar.update()
+        elif tag == "{http://orthoXML.org/2011/}orthologGroup":
+            if not hasattr(self, 'hog_pbar'):
+                if hasattr(self, 'sp_pbar'):
+                    self.sp_pbar.close()
+                    logger.info("Parsing Species completed.")
+                    delattr(self, 'sp_pbar')
+                self.hog_pbar = tqdm(desc='Parsing HOGs')
+            self.hog_pbar.update()
+        elif tag == "{http://orthoXML.org/2011/}groups":
+            if hasattr(self, 'hog_pbar'):
+                self.hog_pbar.close()
+                delattr(self, 'hog_pbar')
+            logger.info("Parsing completed.")
+
+    def close_progress_bars(self):
+        """Close any progress bar left open, e.g. when parsing aborted before reaching </groups>."""
+        for attr in ('sp_pbar', 'hog_pbar'):
+            if hasattr(self, attr):
+                getattr(self, attr).close()
+                delattr(self, attr)
+
+    def _add_extant_genome_to_taxonomy(self, genome):
+        try:
+            taxon = self.taxonomy.get_extant_taxa_by_name(genome.name)
+        except KeyError as e:
+            self.unmapped_species.append((genome.name, e.args[0] if e.args else str(e)))
+            return
+        self.taxonomy.add_genome_to_node(taxon, genome)
+
+    def _assign_ancestral_genome(self, hog):
+        """This method determines the ancestral genome for a HOG based on its children."""
+        child_genomes = [child.genome for child in hog.children]
+        tax_node = None
+        if hog.taxon_id is not None:
+            # If the HOG has a taxon_id, we can directly assign the ancestral genome based on the taxon.
+            tax_nodes = list(self.taxonomy.nodes_by_attr(id=hog.taxon_id))
+            if len(tax_nodes) == 1:
+                tax_node = tax_nodes[0]
+                if len(set(child_genomes)) == 1 and child_genomes[0].taxon == tax_node:
+                    raise abstractgene.SameLevelHOGError(f"HOG '{hog.hog_id}' at {tax_node.name} has child at same level: {hog.children[0]}.")
+        # let's check if we can map it using the TaxRange or taxid property
+        if tax_node is None:
+            for key in ('TaxRange', 'taxid'):
+                try:
+                    tax_value = hog[key]
+                    tax_node = self.taxonomy.get_node_by_name(tax_value)
+                except (KeyError, AttributeError):
+                    pass
+                else:
+                    failing = [(child, child.genome) for child in hog.children
+                               if not self.taxonomy.is_child_recursive(child.genome.taxon, tax_node)]
+                    if not failing:
+                        break
+
+                    if len(set(child_genomes)) == 1 and failing[0][1].taxon == tax_node:
+                        raise abstractgene.SameLevelHOGError(f"HOG '{hog.hog_id}' at {tax_node.name} has child at same level: {failing[0][1]}.")
+
+                    for conflict in self._build_taxonomic_conflicts(hog, key, tax_value, tax_node, failing):
+                        if self.fail_fast:
+                            raise abstractgene.TaxonomicConflictError(format_taxonomic_conflicts([conflict]), [conflict])
+                        self.taxonomic_conflicts.append(conflict)
+                    for child, _ in failing:
+                        self._conflicted_children.add(child)
+                    break
+
+        # if not, let's try with the children genomes
+        if tax_node is None:
+            if len(set(child_genomes)) == 1:
+                try:
+                    tax_range = hog['TaxRange']
+                    if hog['TaxRange'] == hog.children[0].genome.name:
+                        raise abstractgene.SameLevelHOGError(f"HOG '{hog.hog_id}' at {tax_range} has child at same level: {hog.children[0]}.")
+                except (KeyError, AttributeError):
+                    pass
+                tax_node = child_genomes[0].taxon.up
+            else:
+                tax_node = self.taxonomy.get_mrca_taxnode(*(g.taxon for g in child_genomes))
+        genome = self.taxonomy.get_genome_from_taxnode(tax_node)
+        genome.add_gene(hog)
+        logger.debug(f"Added {hog} to ancestral genome {genome.name}")
+
+    def _classify_conflict(self, child, child_node, tax_node):
+        """Classify why `child_node` doesn't sit below `tax_node`, and build a human-readable reason.
+
+        Returns a (kind, detail) pair where kind is one of "same_rank" (child collapses onto the
+        same node as `tax_node`), "inverted" (tax_node is itself a descendant of child_node), or
+        "disjoint" (the two sit in unrelated branches of the given tree).
+        """
+        if child_node == tax_node:
+            own_claim_prop, own_claim_val = None, None
+            for key in ('TaxRange', 'taxid'):
+                try:
+                    own_claim_val = child[key]
+                    own_claim_prop = key
+                    break
+                except (KeyError, AttributeError):
+                    continue
+            if own_claim_val is not None:
+                detail = (f"its own claimed {own_claim_prop}=\"{own_claim_val}\" is not a taxon in the given tree; "
+                          f"the common ancestor of its members there is \"{tax_node.name}\" itself.")
+            else:
+                detail = f"the common ancestor of its members there is \"{tax_node.name}\" itself, the same level as its own parent."
+            return "same_rank", detail
+
+        if self.taxonomy.is_child_recursive(tax_node, child_node):
+            return "inverted", None
+
+        common = self.taxonomy.get_mrca_taxnode(child_node, tax_node)
+        intermediate = [n.name for n in reversed(self.taxonomy.get_path_up(child_node, common))]
+        chain = " > ".join([common.name] + intermediate + [child_node.name])
+        return "disjoint", f"its lineage there runs {chain}."
+
+    def _build_taxonomic_conflicts(self, hog, claim_property, claim_value, tax_node, failing):
+        """Build one :obj:`abstractgene.TaxonomicConflict` per distinct conflict kind found among
+        `hog`'s failing children (in the overwhelmingly common case, all failing children share one
+        kind and this returns a single-element list)."""
+        failing_children = {child for child, _ in failing}
+        sibling_members = [_label_and_species(c) for c in hog.children if c not in failing_children]
+
+        by_kind = defaultdict(list)
+        for child, child_genome in failing:
+            kind, detail = self._classify_conflict(child, child_genome.taxon, tax_node)
+            label, species = _label_and_species(child)
+            by_kind[kind].append((label, species, detail))
+
+        family_id = (self.hog_stack[0].hog_id if self.hog_stack else hog.hog_id).split('_')[0]
+        return [
+            abstractgene.TaxonomicConflict(
+                family_id=family_id,
+                hog_id=hog.hog_id,
+                claim_property=claim_property,
+                claim_value=str(claim_value),
+                resolved_level=tax_node.name,
+                kind=kind,
+                offending_members=members,
+                sibling_members=sibling_members,
+            )
+            for kind, members in by_kind.items()
+        ]
+
+    def _create_missing_hogs(self, child: abstractgene.AbstractGene, parent: abstractgene.HOG):
+        """Create missing HOGs between a child and a parent genome.
+        Returns the oldest created HOG, or the child if no HOG was created."""
+        if child == parent:
+            raise ValueError("Child and parent genomes are the same, cannot create missing HOGs.")
+
+        missing_taxons = self.taxonomy.get_path_up(child.genome.taxon, parent.genome.taxon)
+        if len(missing_taxons) > 0:
+            # the youngest hog is removed from the oldest hog's children.
+            parent.remove_child(child)
+            # Then for each intermediate level in between the two hogs...
+            current_child = child
+            anchor_id = child.hog_id if hasattr(child, 'hog_id') else parent.hog_id
+            for tax in missing_taxons:
+                # ... we get the related ancestral genome of this level...
+                ancestral_genome = self.taxonomy.get_genome_from_taxnode(tax)
+
+                # ... we create the related hog and add it to the ancestral genome...
+                hog_id = id_formats.make_missing_hog_id(self._id_schema, anchor_id, ancestral_genome.taxon)
+                hog = abstractgene.HOG(id=hog_id)
+                setattr(hog, '_missing_in_xml', parent.genome)
+                ancestral_genome.add_gene(hog)
+
+                # ... we check if taxon correspond to child parent taxon ...
+                if ancestral_genome.taxon is not current_child.genome.taxon.up:
+                    raise TypeError(
+                        "HOG taxon {} is different than child parent taxon {}".format(ancestral_genome.taxon,
+                                                                                      current_child.genome.taxon.up))
+                # ... we add the child if everything is fine.
+                hog.add_child(current_child)
+                current_child = hog
+            parent.add_child(current_child)
+        logger.debug(f"Created {len(missing_taxons)} missing HOGs between {child} and {parent}.")
+        return current_child if len(missing_taxons) > 0 else child
+
+    def _describe_child_not_below_parent(self, parent, child):
+        def describe(node):
+            label, species = _label_and_species(node)
+            species = sorted(set(species))
+            shown = ", ".join(species[:5]) + (f", ... ({len(species)} species)" if len(species) > 5 else "")
+            return f"{label} ({shown})"
+
+        others = "; ".join(describe(c) for c in parent.children if c is not child)
+        where = "the same level as" if child.genome.taxon == parent.genome.taxon else "a level above"
+        return (f"Group {describe(child)} resolves to \"{short_level_name(child.genome)}\", {where} its parent group "
+                f"{_label_and_species(parent)[0]}, but a nested group must sit below its parent. The parent also "
+                f"contains {others or 'nothing else'} as a separate lineage, which the species tree doesn't allow. "
+                f"The group nesting contradicts the species tree, e.g. because it comes from a gene tree that "
+                f"wasn't reconciled with this species tree.")
+
+    def _resolve_missing_levels(self, parent: abstractgene.HOG, children: list):
+        """Insert missing intermediate HOG levels between `parent` and `children`.
+
+        If several children are missing the same intermediate taxon (i.e. the
+        reference species tree resolves this part of the taxonomy more finely
+        than the HOGs were built/augmented at), a single shared ancestral HOG is
+        created for all of them instead of one redundant HOG per child. Children
+        that are missing no shared level are delegated to `_create_missing_hogs`.
+
+        Children already flagged as a taxonomic conflict are skipped entirely:
+        their genome doesn't actually sit below `parent` in the given tree, so
+        there is no valid path to insert missing levels along.
+        """
+        children = [c for c in children if c not in self._conflicted_children]
+        change = {}
+        for child in children:
+            if parent.genome.taxon.props["depth"] != child.genome.taxon.props["depth"] - 1:
+                if not self.taxonomy.is_child_recursive(child.genome.taxon, parent.genome.taxon):
+                    raise abstractgene.EvolutionaryConceptError(self._describe_child_not_below_parent(parent, child))
+                change[child] = self.taxonomy.get_path_up(child.genome.taxon, parent.genome.taxon)
+        if not change:
+            return
+
+        groups = defaultdict(list)
+        for child, missing in change.items():
+            # missing[-1] is the taxon immediately below `parent`; children sharing
+            # any ancestor on the path back to `parent` necessarily share this one too.
+            groups[missing[-1]].append(child)
+
+        for tax_node, group_children in groups.items():
+            if len(group_children) == 1:
+                self._create_missing_hogs(group_children[0], parent)
+            else:
+                shared_genome = self.taxonomy.get_genome_from_taxnode(tax_node)
+                shared_hog_id = id_formats.make_missing_hog_id(self._id_schema, parent.hog_id, tax_node)
+                shared_hog = abstractgene.HOG(id=shared_hog_id)
+                setattr(shared_hog, "_missing_in_xml", parent.genome)
+                shared_genome.add_gene(shared_hog)
+                parent.add_child(shared_hog)
+                for child in group_children:
+                    parent.remove_child(child)
+                    shared_hog.add_child(child)
+                self._resolve_missing_levels(shared_hog, group_children)
+
+    def _assign_duplication_branch_ids(self, parent_hog: abstractgene.HOG, duplication, children: list):
+        """Give every duplication branch in `children` (direct children of `parent_hog`,
+        all flagged as arising from `duplication`) its own distinguishing LOFT dot-chain
+        segment when it lacks a real id of its own -- instead of every id-less sibling
+        falling back to `parent_hog`'s own id unchanged, which collapses distinct paralog
+        lineages onto the same synthesized id (e.g. two independent post-duplication
+        genes both becoming "HOG:X.3a_71" instead of "HOG:X.3a.1a_71"/"HOG:X.3a.1b_71").
+
+        Real ids already present on some siblings are never touched, and their
+        (idx, letter) are reused/avoided when assigning the missing ones. If none of
+        `children` reveal a real index at all, a fresh one is minted from a counter that
+        only needs to be distinct from other freshly-minted indices (any synthesized id
+        is already uniquely prefixed by `parent_hog`'s own -- already unique -- id).
+
+        Returns the subset of `children` that got moved under a freshly-created wrapper
+        HOG, so the caller can exclude them from any further, generic gap-filling over
+        `parent_hog`'s direct children (they're no longer among them).
+        """
+        if self._id_schema not in (id_formats.LOFT, id_formats.LOFT_TAXID):
+            return set()
+
+        needs_assignment = [c for c in children if getattr(c, 'hog_id', None) is None]
+        if not needs_assignment:
+            return set()
+
+        m = id_formats.HOG_ID_RE.match(parent_hog.hog_id) if parent_hog.hog_id else None
+        if not m:
+            return set()
+        prefix = m.group('fam') + m.group('subhog')
+
+        real_segments = [seg for seg in
+                         (id_formats.parse_branch_segment(getattr(c, 'hog_id', None), prefix) for c in children)
+                         if seg is not None]
+        if real_segments:
+            idx = real_segments[0][0]
+            used_numbers = {id_formats.decode_paralog_branch(letters)
+                             for seg_idx, letters in real_segments if seg_idx == idx}
+        else:
+            idx = self._next_synthetic_dup_idx
+            self._next_synthetic_dup_idx += 1
+            used_numbers = set()
+
+        use_taxid = self._id_schema == id_formats.LOFT_TAXID
+        wrapped = set()
+        next_number = 0
+        for child in needs_assignment:
+            while next_number in used_numbers:
+                next_number += 1
+            used_numbers.add(next_number)
+            branch_chain = f".{idx}{id_formats.encode_paralog_branch(next_number)}"
+
+            if isinstance(child, abstractgene.HOG):
+                # a real orthologGroup element with no id attribute: relabel in place.
+                taxid = id_formats.taxid_for_taxon(child.genome.taxon) if use_taxid else None
+                child.hog_id = f"{prefix}{branch_chain}_{taxid}" if taxid is not None else f"{prefix}{branch_chain}"
+                continue
+
+            # bare gene: only worth a wrapper HOG if there's real vertical distance
+            # between the duplication and its species -- a real LOFT="..." geneRef
+            # attribute never carries a taxid either, so a wrapper-less label matches
+            # that same convention.
+            missing_taxons = self.taxonomy.get_path_up(child.genome.taxon, parent_hog.genome.taxon)
+            if not missing_taxons:
+                child.set_LOFT(branch_chain)
+                continue
+
+            branch_taxon = missing_taxons[-1]
+            ancestral_genome = self.taxonomy.get_genome_from_taxnode(branch_taxon)
+            taxid = id_formats.taxid_for_taxon(branch_taxon) if use_taxid else None
+            branch_id = f"{prefix}{branch_chain}_{taxid}" if taxid is not None else f"{prefix}{branch_chain}"
+            wrapper = abstractgene.HOG(id=branch_id)
+            setattr(wrapper, "_missing_in_xml", parent_hog.genome)
+            ancestral_genome.add_gene(wrapper)
+
+            parent_hog.remove_child(child)
+            wrapper.add_child(child)
+            parent_hog.add_child(wrapper)
+            self._create_missing_hogs(child, wrapper)
+
+            duplication.remove_child(child)
+            duplication.add_child(wrapper)
+            wrapped.add(child)
+
+        return wrapped
+
+    def start(self, tag, attrib):
+        if tag == "{http://orthoXML.org/2011/}species":
+            self.current_species = ExtantGenome(**attrib)
+            self.n_species += 1
+            if self.taxonomy is not None:
+                self._add_extant_genome_to_taxonomy(self.current_species)
+            else:
+                self._genomes_to_add.append(self.current_species)
+
+        elif tag == "{http://orthoXML.org/2011/}gene" and (self.filterObj is None or attrib["id"] in self.filterObj.geneUniqueId):
+            self._build_gene(attrib)
+
+        elif tag == "{http://orthoXML.org/2011/}paralogGroup" and self.skip_this_hog is False:
+            cur_depth = len(self.hog_stack)
+            if len(self.paralog_stack) > 0 and self.paralog_stack[-1]['depth'] == cur_depth:
+                # we have a directly nested paralog group. use the same DuplicationNode
+                dNode = self.paralog_stack[-1]['node']
+            else:
+                dNode = abstractgene.DuplicationNode(self, id=attrib.get('og', None))
+
+            self.paralog_stack.append({'depth': cur_depth, 'node': dNode})
+
+            self.in_paralogGroup = self.paralog_stack[-1]['depth']
+            self.paralogyNode = self.paralog_stack[-1]['node']
+
+        elif tag == "{http://orthoXML.org/2011/}geneRef" and self.skip_this_hog is False:
+
+            gene = self.extant_gene_map[attrib['id']]
+            if 'LOFT' in attrib:
+                gene.set_LOFT(attrib['LOFT'])
+            self.hog_stack[-1].add_child(gene)
+
+            # if the gene is contained within a paralogousGroup need to update its .arose_by_duplication flag.
+            if self.in_paralogGroup == len(self.hog_stack):
+                self.paralogyNode.add_child(gene)
+
+        elif tag == "{http://orthoXML.org/2011/}orthologGroup":
+            if len(self.hog_stack) == 0:
+                # a new roothog
+                self._family_index += 1
+                self.current_family_id = attrib.get("id") or "#{} (without id)".format(self._family_index)
+                if self.with_progress:
+                    self._update_progress_bar(tag)
+
+                if self.filterObj is not None and not attrib.get("id", None) in self.filterObj.hogsId:
+                    self.skip_this_hog = True
+            # create HOG and add it to the stack (or a dummy value if we skip this hog)
+            if self.skip_this_hog:
+                self.hog_stack.append(0)
+            else:
+                self._build_hog(attrib)
+
+        elif tag == "{http://orthoXML.org/2011/}property" and not self.skip_this_hog:
+            self.hog_stack[-1].add_property(attrib["name"], attrib["value"])
+
+        elif tag == "{http://orthoXML.org/2011/}score" and not self.skip_this_hog:
+            self.hog_stack[-1].score(attrib['id'], float(attrib['value']))
+
+        elif tag == "{http://orthoXML.org/2011/}taxon":
+            taxon = build_taxon_node(**attrib)
+            self.taxon_stack.append(taxon)
+            if len(self.taxon_stack) > 1:
+                self.taxon_stack[-2].add_child(child=taxon)
+
+        elif tag == "{http://orthoXML.org/2011/}groups":
+            if self.taxonomy is None:
+                raise ValueError("Cannot parse this OrthoXML file without an explicit taxonomy tree. Please provide one.")
+            if self.unmapped_species:
+                from .validation import species_not_in_tree_error
+                raise species_not_in_tree_error(self.unmapped_species, self.n_species, self.taxonomy)
+
+    def end(self, tag):
+
+        if tag == "{http://orthoXML.org/2011/}species":
+            if self.with_progress:
+                self._update_progress_bar(tag)
+            logger.debug(f"Species {self.current_species.name} created.")
+            self.current_species = None
+
+        elif tag == "{http://orthoXML.org/2011/}taxon":
+            taxonomy = self.taxon_stack.pop()
+            if len(self.taxon_stack) == 0:
+                # we are at the root of the taxonomy tree. if no taxonomy is provided upfront, we create it.
+                if self.taxonomy is None:
+                    self.taxonomy = Taxonomy(taxonomy)
+                    # add all extant genomes to the taxonomy tree
+                    for genome in self._genomes_to_add:
+                        self._add_extant_genome_to_taxonomy(genome)
+                    logger.info("Taxonomy tree created from OrthoXML taxonomy. All extant genomes added to the tree.")
+                else:
+                    logger.info("OrthoXML taxonomy not used, using provided taxonomy tree instead.")
+
+        elif tag == "{http://orthoXML.org/2011/}paralogGroup" and self.skip_this_hog is False:
+            ln = self.paralog_stack.pop()
+            ln['node'].set_MRCA()
+            if len(self.paralog_stack) > 0:
+                self.in_paralogGroup = self.paralog_stack[-1]['depth']
+                self.paralogyNode = self.paralog_stack[-1]['node']
+            else:
+                self.in_paralogGroup = None
+                self.paralogyNode = None
+
+        elif tag == "{http://orthoXML.org/2011/}orthologGroup":
+            # get the latest hog
+            hog = self.hog_stack.pop()
+            if self.skip_this_hog:
+                if len(self.hog_stack) == 0:
+                    self.skip_this_hog = False
+                    self.current_family_id = None
+                return
+
+            try:
+                # assign the ancestral genome to this hog
+                self._assign_ancestral_genome(hog)
+            except abstractgene.SameLevelHOGError as e:
+                # this hog has children at the same level as its parent. Usually an 'augmented' HOG.
+                logger.info(str(e))
+                self.collapsed_same_level_hogs.append(hog.hog_id)
+                # remove this hog structure.
+                for child in hog.children:
+                    child.parent = hog.parent
+                hog.parent.children.remove(hog)
+                hog.parent.children.extend(hog.children)
+                if hog.arose_by_duplication:
+                    dupl_node = hog.arose_by_duplication
+                    dupl_node.remove_child(hog)
+                    for child in hog.children:
+                        dupl_node.add_child(child)
+                return
+
+            # get all child clustered by dup if any (skip children already flagged as a taxonomic
+            # conflict -- their genome doesn't reliably sit below hog.genome in the given tree, so
+            # the MRCA-below-hog assertion below can't be trusted for them)
+            child_by_duplication = defaultdict(list)
+            for child in hog.children:
+                if child in self._conflicted_children:
+                    continue
+                if child.arose_by_duplication:
+                    child_by_duplication[child.arose_by_duplication].append(child)
+
+            # all duplications must have their MRCA equal to hog.genome or below
+            for duplication, children in child_by_duplication.items():
+                if not (duplication.MRCA == hog.genome or self.taxonomy.is_child_recursive(duplication.MRCA.taxon, hog.genome.taxon)):
+                    copies = ", ".join(f"{_label_and_species(c)[0]} at \"{short_level_name(c.genome)}\""
+                                       for c in children)
+                    raise abstractgene.EvolutionaryConceptError(
+                        f"HOG {_label_and_species(hog)[0]} at \"{short_level_name(hog.genome)}\" contains a "
+                        f"duplication whose copies ({copies}) split before \"{short_level_name(duplication.MRCA)}\", "
+                        f"above the level of the HOG itself, but a HOG can't contain copies of a duplication older "
+                        f"than itself. Usually the HOG's level was inferred from its member species because its "
+                        f"TaxRange/taxid isn't a node of the species tree, or the group nesting contradicts the tree.")
+
+
+            # For each duplication
+            for duplication, children in child_by_duplication.items():
+
+                # add hog at the level of the duplication MRCA if its missing
+                if duplication.MRCA != hog.genome and self.taxonomy.is_child_recursive(duplication.MRCA.taxon, hog.genome.taxon):
+                    # create the MRCA hog
+                    mrca_genome = duplication.MRCA
+                    mrca_hog_id = id_formats.make_missing_hog_id(self._id_schema, hog.hog_id, mrca_genome.taxon)
+                    mrca_hog = abstractgene.HOG(id=mrca_hog_id)
+                    setattr(mrca_hog, "_missing_in_xml", hog.genome)
+                    mrca_genome.add_gene(mrca_hog)
+
+                    # link it to the current hog as parent
+                    hog.add_child(mrca_hog)
+
+                    # add missing level down (from mrcaHOG to duplicated child)
+                    for child in children:
+                        # move child from hog to mrca_hog
+                        hog.remove_child(child)
+                        mrca_hog.add_child(child)
+
+                    # give any duplication branch lacking its own real id a proper,
+                    # distinguishing LOFT segment before filling any remaining gaps
+                    wrapped = self._assign_duplication_branch_ids(mrca_hog, duplication, children)
+                    remaining = [c for c in children if c not in wrapped]
+
+                    # add missing levels if any, grouping children that are
+                    # missing the same intermediate ancestor under one shared HOG
+                    self._resolve_missing_levels(mrca_hog, remaining)
+
+                    duplication.set_parent(mrca_hog)
+                    for x in list(duplication.children):
+                        duplication.remove_child(x)
+                    for y in list(mrca_hog.children):
+                        duplication.add_child(y)
+
+                # Otherwise simply add missing taxa between hog and duplicated child
+                else:
+                    duplication.MRCA = self.taxonomy.get_genome_from_taxnode(hog.genome.taxon)
+                    duplication.set_parent(hog)
+
+                    # give any duplication branch lacking its own real id a proper,
+                    # distinguishing LOFT segment before filling any remaining gaps
+                    wrapped = self._assign_duplication_branch_ids(hog, duplication, children)
+
+                    for child_direct in children:
+                        if child_direct in wrapped:
+                            continue
+                        new_direct_child = self._create_missing_hogs(child_direct, hog)
+                        duplication.remove_child(child_direct)
+                        duplication.add_child(new_direct_child)
+
+            # insert any remaining missing intermediate levels, grouping children
+            # that are missing the same intermediate ancestor under one shared HOG
+            self._resolve_missing_levels(hog, list(hog.children))
+
+            if len(self.hog_stack) == 0:
+                hog_id = hog.hog_id.split('_')[0]
+                self.toplevel_hogs[hog_id] = hog
+                self.current_family_id = None
+                self.cpt += 1
+                if self.cpt % 500 == 0:
+                    logger.debug("{} HOGs parsed. ".format(self.cpt))
+
+        elif tag == "{http://orthoXML.org/2011/}groups":
+            if self.with_progress:
+                self._update_progress_bar(tag)
+
+    def data(self, data):
+        # Ignore data inside nodes
+        pass
+
+    def close(self):
+        # Nothing special to do here
+        return
+
+
+class IDSchemeSniffer:
+    """Lightweight OrthoXML parser target used to sample HOG ids for id-scheme
+    auto-detection (see :obj:`pyham.id_formats.detect_id_scheme`), without paying for a
+    full parse of the file.
+
+    Collects the `id` attribute of every `orthologGroup` element seen, and flips `done`
+    to True once a bounded number of top-level families or ids has been sampled -- the
+    caller is expected to stop feeding lines once `done` is True.
+    """
+
+    MAX_FAMILIES = 3
+    MAX_IDS = 300
+    _TAG = "{http://orthoXML.org/2011/}orthologGroup"
+
+    def __init__(self):
+        self.samples = []
+        self.families_seen = 0
+        self._depth = 0
+        self.done = False
+
+    def start(self, tag, attrib):
+        if tag == self._TAG:
+            self._depth += 1
+            if 'id' in attrib:
+                self.samples.append(attrib['id'])
+                if len(self.samples) >= self.MAX_IDS:
+                    self.done = True
+
+    def end(self, tag):
+        if tag == self._TAG:
+            self._depth -= 1
+            if self._depth == 0:
+                self.families_seen += 1
+                if self.families_seen >= self.MAX_FAMILIES:
+                    self.done = True
+
+    def data(self, data):
+        pass
+
+    def close(self):
+        return
+
+
+def sniff_id_schema(file_object):
+    """Sample the first few families of an OrthoXML file object and return the detected HOG id scheme
+    (see :obj:`pyham.id_formats.detect_id_scheme`)."""
+    sniffer = IDSchemeSniffer()
+    sniff_parser = XMLParser(target=sniffer)
+    # <species>/<taxonomy> can dwarf <groups> in size (e.g. for large exports with
+    # a big embedded taxonomy); skip straight to <groups> via a cheap text search
+    # instead of XML-parsing that whole preamble just to sample a few HOG ids.
+    sniff_parser.feed('<orthoXML xmlns="http://orthoXML.org/2011/">')
+    seen_groups = False
+    for line in file_object:
+        if not seen_groups:
+            idx = line.find('<groups')
+            if idx == -1:
+                continue
+            line = line[idx:]
+            seen_groups = True
+        sniff_parser.feed(line)
+        if sniffer.done:
+            break
+    return id_formats.detect_id_scheme(sniffer.samples)
+
+
+class FilterOrthoXMLParser(object):
+    """
+    Custom OrthoXML parser use to read the orthoxml file and get required information base on specific query set. It use
+    the ParserFilter object as input to do the selection.
+
+    The parse goes through the whole XML and create on the fly the required Ham objects:
+        - In the Xref/header, the parser creates the ExtantGenome and Gene objects.
+        - In the Groups section, it creates the HOGs with their hierarchy (parent/children links) and their related
+        AncestralGenomes.
+
+
+    Attributes:
+        filterObj (:obj:`FilterParser`): Filter object with all the filtering information.
+
+        geneUniqueId (:obj:`list`): list of unique gene ids to store for the OrthoXMLParser.
+        hogsId (:obj:`list`): list of hogs ids to store for the OrthoXMLParser.
+
+        current_hog (:obj:`int`): Current hog id.
+        hog_stack (:obj:`list`): Stack of hogs currently parsed. Reset at each top level hog change.
+        hog_generef (:obj:`list`): list of unique gene ids contained into the current parsed hog.
+        add_this_hog (:obj:`Boolean`): Boolean to know if we keep the current hog for the OrthoXMLParser.
+    """
+
+    def __init__(self, filterO):
+
+        self.filterObj = filterO
+
+        self.geneUniqueId = []
+        self.hogsId = []
+
+        self.current_hog = None
+        self.hog_stack = []
+        self.hog_generef = []
+        self.add_this_hog = False
+
+    def start(self, tag, attrib):
+
+        if tag == "{http://orthoXML.org/2011/}gene":
+            if self.filterObj.GeneIntId_filter:
+                if attrib['id'] in self.filterObj.GeneIntId_filter:
+                    self.geneUniqueId.append(attrib['id'])
+
+            if self.filterObj.GeneExtId_filter:
+                for xtid in attrib.values():
+                    if xtid in self.filterObj.GeneExtId_filter:
+                        self.geneUniqueId.append(attrib['id'])
+
+        elif tag == "{http://orthoXML.org/2011/}geneRef":
+            self.hog_generef.append(attrib['id'])
+            if attrib['id'] in self.geneUniqueId:
+                self.add_this_hog = True
+
+        elif tag == "{http://orthoXML.org/2011/}orthologGroup":
+            if len(self.hog_stack) == 0:
+                self.current_hog = attrib["id"]
+                if self.current_hog in self.filterObj.HOGId_filter:
+                    self.add_this_hog = True
+
+            self.hog_stack.append(1)
+
+    def end(self, tag):
+
+        if tag == "{http://orthoXML.org/2011/}orthologGroup":
+
+            hog = self.hog_stack.pop()
+
+            if len(self.hog_stack) == 0:
+
+                if self.add_this_hog:
+                    self.geneUniqueId = self.geneUniqueId + self.hog_generef
+                    self.hogsId.append(self.current_hog)
+
+                self.current_hog = None
+                self.hog_generef = []
+                self.add_this_hog = False
+
+    def data(self, data):
+        # Ignore data inside nodes
+        pass
+
+    def close(self):
+        # Nothing special to do here
+        return
+
+
+
+
+class PhyloXMLToETE:
+    def __init__(self, node_factory=None):
+        self.root = None
+        self.stack = []
+        self.current_tag = None
+        self.buffer = ''
+        self.in_taxonomy = False
+        self.taxonomy_data = None
+        self.in_clade = False
+        from .taxonomy import Tree
+        self.node_factory = node_factory if node_factory else Tree
+
+    def start(self, tag, attrib):
+        tag = self._strip_ns(tag)
+        self.current_tag = tag
+
+        if tag == "clade":
+            dist = attrib.get("branch_length", None)
+            node = self.node_factory({"dist": dist})
+            if len(self.stack) > 0:
+                self.stack[-1].add_child(node)
+            else:
+                self.root = node
+            self.stack.append(node)
+
+        elif tag == "taxonomy":
+            self.in_taxonomy = True
+            self.taxonomy_data = {}
+
+    def end(self, tag):
+        tag = self._strip_ns(tag)
+
+        if tag == "name" and not self.in_taxonomy:
+            if len(self.stack) == 0:
+                self.buffer = ""
+                return
+            # name of clade. -> set the name of the last node in the stack
+            self.stack[-1].name = self.buffer.strip()
+
+        elif self.in_taxonomy:
+            if tag in ('scientific_name', 'id', 'code'):
+                self.taxonomy_data[tag] = self.buffer.strip()
+
+            elif tag == "taxonomy":
+                node = self.stack[-1]
+                sci_name = self.taxonomy_data.get('scientific_name')
+                if sci_name:
+                    node.add_prop('scientific_name', sci_name)
+                taxon_id = self.taxonomy_data.get('id')
+                if taxon_id:
+                    node.add_prop('taxon_id', taxon_id)
+                code = self.taxonomy_data.get('code')
+                if code:
+                    node.add_prop('code', code)
+                self.in_taxonomy = False
+                self.taxonomy_data = None
+
+        elif tag == "clade":
+            self.stack.pop()
+
+        self.buffer = ""
+        self.current_tag = None
+
+    def data(self, data):
+        if self.current_tag:
+            self.buffer += data
+
+    def close(self):
+        return self.root
+
+    def _strip_ns(self, tag):
+        return tag.split('}', 1)[-1] if '}' in tag else tag