hanlp 2.1.0__py3-none-any.whl

hanlp/__init__.py

@@ -0,0 +1,57 @@
+# -*- coding:utf-8 -*-
+# Author: hankcs
+# Date: 2019-06-13 18:05
+import hanlp.common
+import hanlp.components
+import hanlp.pretrained
+import hanlp.utils
+from hanlp.version import __version__
+
+hanlp.utils.ls_resource_in_module(hanlp.pretrained)
+
+
+def load(save_dir: str, verbose=None, **kwargs) -> hanlp.common.component.Component:
+    """Load a pretrained component from an identifier.
+
+    Args:
+      save_dir (str): The identifier to the saved component. It could be a remote URL or a local path.
+      verbose: ``True`` to print loading progress.
+      **kwargs: Arguments passed to :func:`hanlp.common.torch_component.TorchComponent.load`, e.g.,
+        ``devices`` is a useful argument to specify which GPU devices a PyTorch component will use.
+
+    Examples::
+
+        import hanlp
+        # Load component onto the 0-th GPU.
+        hanlp.load(..., devices=0)
+        # Load component onto the 0-th and 1-st GPUs using data parallelization.
+        hanlp.load(..., devices=[0, 1])
+
+    .. Note::
+        A component can have dependencies on other components or resources, which will be recursively loaded. So it's
+        common to see multiple downloading messages per single load.
+
+    Returns:
+      hanlp.common.component.Component: A pretrained component.
+
+    """
+    save_dir = hanlp.pretrained.ALL.get(save_dir, save_dir)
+    from hanlp.utils.component_util import load_from_meta_file
+    if verbose is None:
+        from hanlp_common.constant import HANLP_VERBOSE
+        verbose = HANLP_VERBOSE
+    return load_from_meta_file(save_dir, 'meta.json', verbose=verbose, **kwargs)
+
+
+def pipeline(*pipes) -> hanlp.components.pipeline.Pipeline:
+    """Creates a pipeline of components. It's made for bundling `KerasComponents`. For `TorchComponent`, use
+    :class:`~hanlp.components.mtl.multi_task_learning.MultiTaskLearning` instead.
+
+    Args:
+      *pipes: Components if pre-defined any.
+
+    Returns:
+      hanlp.components.pipeline.Pipeline: A pipeline, which is a list of components in order.
+
+    """
+    return hanlp.components.pipeline.Pipeline(*pipes)

hanlp/callbacks/__init__.py

@@ -0,0 +1,3 @@
+# -*- coding:utf-8 -*-
+# Author: hankcs
+# Date: 2019-12-05 02:10

hanlp/callbacks/fine_csv_logger.py

@@ -0,0 +1,71 @@
+# -*- coding:utf-8 -*-
+# Author: hankcs
+# Date: 2019-12-05 02:12
+import copy
+from io import TextIOWrapper
+from typing import List
+
+import numpy as np
+import tensorflow as tf
+
+
+class StreamTableFormatter(object):
+
+    def __init__(self) -> None:
+        super().__init__()
+        self.col_widths = None
+
+    def format_row(self, cells) -> List[str]:
+        if not isinstance(cells, list):
+            cells = list(cells)
+        if not self.col_widths:
+            self.col_widths = [0] * len([_ for _ in cells])
+        for i, c in enumerate(cells):
+            self.col_widths[i] = max(self.col_widths[i], len(self.format_cell(c, self.col_widths[i])))
+        return list(self.format_cell(cell, width) for cell, width in zip(cells, self.col_widths))
+
+    def format_cell(self, cell: str, min_width) -> str:
+        if isinstance(cell, (np.float32, np.float)):
+            return '{:>{}.4f}'.format(cell, min_width)
+        return '{:>{}}'.format(cell, min_width)
+
+
+class FineCSVLogger(tf.keras.callbacks.History):
+
+    def __init__(self, filename, separator=',', append=False):
+        super().__init__()
+        self.append = append
+        self.separator = separator
+        self.filename = filename
+        self.out: TextIOWrapper = None
+        self.keys = []
+        self.formatter = StreamTableFormatter()
+
+    def on_train_begin(self, logs=None):
+        super().on_train_begin(logs)
+        self.out = open(self.filename, 'a' if self.append else 'w')
+
+    def on_train_end(self, logs=None):
+        self.out.close()
+
+    def on_epoch_end(self, epoch, logs=None):
+        super().on_epoch_end(epoch, logs)
+        if not self.keys:
+            self.keys = sorted(logs.keys())
+
+            if getattr(self.model, 'stop_training', None):
+                # We set NA so that csv parsers do not fail for this last epoch.
+                logs = dict([(k, logs[k]) if k in logs else (k, 'NA') for k in self.keys])
+
+            # feed them twice to decide the actual width
+            values = self.formatter.format_row([epoch + 1] + [logs.get(k, 'NA') for k in self.keys])
+            headers = self.formatter.format_row(['epoch'] + self.keys)
+            # print headers and bars
+            self.out.write(self.separator.join(headers) + '\n')
+            # bars for markdown style
+            bars = [''.join(['-'] * width) for width in self.formatter.col_widths]
+            self.out.write(self.separator.join(bars) + '\n')
+
+        values = self.formatter.format_row([epoch + 1] + [logs.get(k, 'NA') for k in self.keys])
+        self.out.write(self.separator.join(values) + '\n')
+        self.out.flush()

hanlp/common/__init__.py

@@ -0,0 +1,3 @@
+# -*- coding:utf-8 -*-
+# Author: hankcs
+# Date: 2019-08-26 14:45

hanlp/common/component.py

@@ -0,0 +1,36 @@
+# -*- coding:utf-8 -*-
+# Author: hankcs
+# Date: 2019-08-26 14:45
+import inspect
+from abc import ABC, abstractmethod
+from typing import Any
+
+from hanlp_common.configurable import Configurable
+
+
+class Component(Configurable, ABC):
+    @abstractmethod
+    def predict(self, *args, **kwargs):
+        """Predict on data. This is the base class for all components, including rule based and statistical ones.
+
+        Args:
+          *args: Any type of data subject to sub-classes
+          **kwargs: Additional arguments
+
+        Returns: Any predicted annotations.
+
+        """
+        raise NotImplementedError('%s.%s()' % (self.__class__.__name__, inspect.stack()[0][3]))
+
+    def __call__(self, *args, **kwargs):
+        """
+        A shortcut for :func:`~hanlp.common.component.predict`.
+
+        Args:
+          *args: Any type of data subject to sub-classes
+          **kwargs: Additional arguments
+
+        Returns: Any predicted annotations.
+
+        """
+        return self.predict(*args, **kwargs)