bitswan 2024.4__tar.gz

bitswan-2024.4/.coveragerc

@@ -0,0 +1,5 @@
+[run]
+source=bspump/
+
+[report]
+include=bspump*

bitswan-2024.4/.github/workflows/docker-publish.yml

@@ -0,0 +1,31 @@
+name: Build and Push Docker Image
+
+on:
+  push:
+    branches: [ master ]
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Check out the repo
+      uses: actions/checkout@v2
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v1
+
+    - name: Log in to Docker Hub
+      uses: docker/login-action@v1
+      with:
+        username: ${{ secrets.DOCKERHUB_USERNAME }}
+        password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }}
+
+    - name: Get Year and Commit Hash
+      id: vars
+      run: |
+        echo "::set-output name=year::$(date +%Y)"
+        echo "::set-output name=commit_hash::$(git rev-parse --short HEAD)"
+
+    - name: Build and Push
+      run: bash build_and_push.sh

bitswan-2024.4/.github/workflows/test.yml

@@ -0,0 +1,36 @@
+name: Python 3.10 Test
+
+on: [push, pull_request]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    container:
+      image: python:3.10
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Install dependencies
+      run: |
+        apt-get update
+        apt-get -y install unixodbc-dev
+        git config --global --add safe.directory /__w/BitSwan/BitSwan
+        pip3 install --upgrade cython
+        pip3 install .[dev]
+
+    - name: Run tests
+      run: |
+        pytest bspump/file
+
+    - name: Run bitswan tests
+      run: |
+        bitswan-cli --test examples/Testing/main.ipynb
+        bitswan-cli --test examples/Testing/MultiplePipelines/main.ipynb
+        bitswan-cli --test examples/Testing/AutoPipeline/main.ipynb
+
+
+    - name: Run linter
+      run: |
+        black --check bspump
+        ruff check bspump

bitswan-2024.4/.gitignore

@@ -0,0 +1,118 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+*.dot
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# dotenv
+.env
+
+# virtualenv
+.venv
+venv/
+ENV/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+
+# idea
+.idea/
+
+# ignoring site configuration
+etc/site.conf
+etc/site.d/
+
+# ignoring example outputs
+examples/data/ignore_*
+
+doc/_build
+
+# VSCode
+.vscode/
+

bitswan-2024.4/.gitlab-ci.yml

@@ -0,0 +1,23 @@
+variables:
+  DOCKER_IMAGE: bitswan-pre
+
+stages:
+  - test
+  - build
+
+test-py310:
+  image: python:3.10
+  stage: test
+  coverage: '/\d+\%$/'
+  script:
+    - apt-get update
+    - apt-get -y install unixodbc-dev
+    - pip3 install --upgrade cython
+    - pip3 install .
+    - pip install -r requirements-dev.txt
+    - pytest bspump/file
+    - ruff bspump
+
+include:
+  - project: 'LibertyAces/Product/bitswanmonorepo'
+    file: 'cicd/docker-build-public.yml'

bitswan-2024.4/Dockerfile

@@ -0,0 +1,40 @@
+FROM python:3.10-slim-bullseye
+MAINTAINER LibertyAces Ltd
+LABEL "space.bitswan.pipeline.protocol-version"="2023.12-1"
+LABEL "space.bitswan.pipeline.ide"="Jupyter"
+LABEL src=https://github.com/bitswan-space/BitSwan
+ENV ASABFORCECONSOLE=1
+ENV PYTHONUNBUFFERED=1
+
+RUN set -ex \
+ && apt-get update \
+ && apt-get -y upgrade
+
+RUN apt-get -yqq install \
+ git \
+ gcc \
+ g++ \
+ libsnappy-dev \
+ autoconf \
+ automake \
+ libtool \
+ build-essential \
+ docker-compose
+
+COPY . /src/
+RUN cd /src/ ; pip3 install .
+
+COPY pre/ /opt/
+
+CMD ["sh", "/opt/entrypoint.sh"]
+
+# Setup bitswan user
+
+RUN useradd --uid 1000 --create-home bitswan
+ENV HOME=/home/bitswan
+USER bitswan
+
+USER root
+RUN mkdir -p /home/bitswan/work
+RUN chown -R bitswan:bitswan /home/bitswan/work
+

bitswan-2024.4/LICENSE

@@ -0,0 +1,29 @@
+BSD 3-Clause License
+
+Copyright (c) 2018-2019, LibertyAces Ltd 
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice,
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution.
+
+* Neither the name of the copyright holder nor the names of its
+  contributors may be used to endorse or promote products derived from
+  this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

bitswan-2024.4/MANIFEST.in

@@ -0,0 +1 @@
+include *.txt *.md

bitswan-2024.4/PKG-INFO

@@ -0,0 +1,148 @@
+Metadata-Version: 2.3
+Name: bitswan
+Version: 2024.4
+Summary: Bitswan is a framework for building automations and pipelines in Jupyter
+Project-URL: homepage, https://github.com/bitswan-space/BitSwan
+Author-email: LibertyAces Ltd <timothy.hobbs@libertyaces.com>
+License: BSD 3-Clause License
+        
+        Copyright (c) 2018-2019, LibertyAces Ltd 
+        All rights reserved.
+        
+        Redistribution and use in source and binary forms, with or without
+        modification, are permitted provided that the following conditions are met:
+        
+        * Redistributions of source code must retain the above copyright notice, this
+          list of conditions and the following disclaimer.
+        
+        * Redistributions in binary form must reproduce the above copyright notice,
+          this list of conditions and the following disclaimer in the documentation
+          and/or other materials provided with the distribution.
+        
+        * Neither the name of the copyright holder nor the names of its
+          contributors may be used to endorse or promote products derived from
+          this software without specific prior written permission.
+        
+        THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+        AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+        IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+        DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+        FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+        DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+        SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+        CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+        OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+        OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+Requires-Dist: aiohttp>=3.8.3
+Requires-Dist: aiomysql>=0.0.20
+Requires-Dist: aiosmtplib>=1.1.3
+Requires-Dist: aiozk>=0.25.0
+Requires-Dist: confluent-kafka>=1.8.2
+Requires-Dist: croniter>=1.4.1
+Requires-Dist: docker
+Requires-Dist: dockerfile-parse
+Requires-Dist: fastavro>=0.23.5
+Requires-Dist: fastjsonschema<3,>=2.16.2
+Requires-Dist: google-api-python-client>=1.7.10
+Requires-Dist: jupyter
+Requires-Dist: kazoo<3,>=2.9.0
+Requires-Dist: mongoquery>=1.3.6
+Requires-Dist: motor>=2.1.0
+Requires-Dist: mysql-replication>=0.21
+Requires-Dist: nest-asyncio==1.6.0
+Requires-Dist: netaddr>=0.7.20
+Requires-Dist: numpy>=1.19.0
+Requires-Dist: orjson
+Requires-Dist: paho-mqtt
+Requires-Dist: pandas>=0.24.2
+Requires-Dist: pika>=1.1.0
+Requires-Dist: pyasn1==0.4.8
+Requires-Dist: pybind11>=2.6.1
+Requires-Dist: pyjwt==v2.10.0
+Requires-Dist: pymongo>=3.10.1
+Requires-Dist: pymysql<=0.9.2,>=0.9.2
+Requires-Dist: python-dotenv==1.0.1
+Requires-Dist: pytz>=2020.1
+Requires-Dist: pyyaml>=5.4
+Requires-Dist: requests>=2.24.0
+Requires-Dist: xxhash>=1.4.4
+Provides-Extra: dev
+Requires-Dist: black==23.1.0; extra == 'dev'
+Requires-Dist: coverage==4.5.3; extra == 'dev'
+Requires-Dist: pytest; extra == 'dev'
+Requires-Dist: pytest-asyncio; extra == 'dev'
+Requires-Dist: ruff; extra == 'dev'
+Provides-Extra: pyjq-support
+Requires-Dist: pyjq>=2.6.0; extra == 'pyjq-support'
+Description-Content-Type: text/markdown
+
+Bitswan: A tool for building Pipelines & Automations in Jupyter
+===============================================
+
+You can find example pipelines in the [examples](./examples/) directory.
+
+Installation
+--------------
+
+```
+$ git clone git@github.com:bitswan-space/BitSwan.git
+$ cd BitSwan
+$ python3 -m venv venv
+$ source venv/bin/activate
+$ pip3 install -e ".[dev]"
+```
+
+Running pipelines
+--------------------
+
+You can run a pipeline with a simple command:
+
+```
+$ bitswan-cli examples/WebForms/main.ipynb
+```
+
+When developing web endpoints it can be helpful to instruct the pipeline to automatically restart if the source code changes.
+
+```
+$ bitswan-cli examples/WebForms/main.ipynb --watch
+```
+
+Running Tests
+----------------
+
+You can find examples for automatically testing pipelines in the [testing examples](./examples/Testing) directory.
+
+Run tests with the `--test` flag.
+
+```
+$ bitswan-cli examples/Testing/InspectError/main.ipynb --test
+
+Running tests for pipeline Kafka2KafkaPipeline.
+
+    ┌ Testing event:        b'foo'
+    └ Outputs:              [b'FOO'] ✔
+
+All tests passed for Kafka2KafkaPipeline.
+
+
+Running tests for pipeline auto_pipeline_1.
+
+    ┌ Testing event:        b'{"foo":"aaa"}'
+    └ Outputs:              [b'{"foo": "A   A   A"}'] ✔
+
+    ┌ Testing event:        b'{"foo":"aab"}'
+    │ Probing after-upper.
+    └ Outputs:              [b'{"foo": "B   A   A"}'] ✔
+
+    ┌ Testing event:        b'{"foo":"cab"}'
+    └ Outputs:              [b'{"foo": "B   A   C"}'] ✘
+```
+
+You can combine `--test` with `--watch` to automatically rerun tests whenever the source files change.
+
+
+Licence
+-------
+
+Bitswan is open-source software, available under BSD 3-Clause License.
+

bitswan-2024.4/README.md

@@ -0,0 +1,70 @@
+Bitswan: A tool for building Pipelines & Automations in Jupyter
+===============================================
+
+You can find example pipelines in the [examples](./examples/) directory.
+
+Installation
+--------------
+
+```
+$ git clone git@github.com:bitswan-space/BitSwan.git
+$ cd BitSwan
+$ python3 -m venv venv
+$ source venv/bin/activate
+$ pip3 install -e ".[dev]"
+```
+
+Running pipelines
+--------------------
+
+You can run a pipeline with a simple command:
+
+```
+$ bitswan-cli examples/WebForms/main.ipynb
+```
+
+When developing web endpoints it can be helpful to instruct the pipeline to automatically restart if the source code changes.
+
+```
+$ bitswan-cli examples/WebForms/main.ipynb --watch
+```
+
+Running Tests
+----------------
+
+You can find examples for automatically testing pipelines in the [testing examples](./examples/Testing) directory.
+
+Run tests with the `--test` flag.
+
+```
+$ bitswan-cli examples/Testing/InspectError/main.ipynb --test
+
+Running tests for pipeline Kafka2KafkaPipeline.
+
+    ┌ Testing event:        b'foo'
+    └ Outputs:              [b'FOO'] ✔
+
+All tests passed for Kafka2KafkaPipeline.
+
+
+Running tests for pipeline auto_pipeline_1.
+
+    ┌ Testing event:        b'{"foo":"aaa"}'
+    └ Outputs:              [b'{"foo": "A   A   A"}'] ✔
+
+    ┌ Testing event:        b'{"foo":"aab"}'
+    │ Probing after-upper.
+    └ Outputs:              [b'{"foo": "B   A   A"}'] ✔
+
+    ┌ Testing event:        b'{"foo":"cab"}'
+    └ Outputs:              [b'{"foo": "B   A   C"}'] ✘
+```
+
+You can combine `--test` with `--watch` to automatically rerun tests whenever the source files change.
+
+
+Licence
+-------
+
+Bitswan is open-source software, available under BSD 3-Clause License.
+

bitswan-2024.4/bitswan/__init__.py

@@ -0,0 +1,54 @@
+from .application import BSPumpApplication
+from .service import BSPumpService
+from .pipeline import Pipeline
+from .pumpbuilder import PumpBuilder
+from .abc.source import Source
+from .abc.source import TriggerSource
+from .abc.sink import Sink
+from .abc.processor import Processor
+from .abc.generator import Generator
+from .abc.connection import Connection
+
+from .exception import ProcessingError
+from .abc.lookup import Lookup
+from .abc.lookup import MappingLookup
+from .abc.lookup import DictionaryLookup
+from .fileloader import load_json_file
+
+from .analyzer.analyzer import Analyzer
+
+from .matrix.matrix import Matrix, PersistentMatrix
+from .matrix.namedmatrix import NamedMatrix, PersistentNamedMatrix
+from .model.model import Model
+
+from .abc.anomaly import Anomaly
+
+from .__version__ import __version__, __build__
+
+__all__ = (
+    "BSPumpApplication",
+    "BSPumpService",
+    "Pipeline",
+    "PumpBuilder",
+    "Source",
+    "TriggerSource",
+    "Sink",
+    "Processor",
+    "Generator",
+    "Connection",
+    "Analyzer",
+    "ProcessingError",
+    "Lookup",
+    "MappingLookup",
+    "DictionaryLookup",
+    "load_json_file",
+    "Matrix",
+    "PersistentMatrix",
+    "NamedMatrix",
+    "Model",
+    "PersistentNamedMatrix",
+    "Anomaly",
+    "__version__",
+    "__build__",
+    "step",
+)

bitswan-2024.4/bitswan/__main__.py

@@ -0,0 +1,4 @@
+from .application import BSPumpApplication
+
+app = BSPumpApplication()
+app.run()

bitswan-2024.4/bitswan/__version__.py

@@ -0,0 +1,12 @@
+# THIS-FILE-WILL-BE-REPLACED !!! DO NOT CHANGE OR MODIFY THIS FILE !!!
+
+# During `setup.py build_py`, this file is overwritten
+# __version__ = '[git describe --abbrev=7 --tags --dirty=+dirty --always]
+# __build__ = '[git rev-parse HEAD]'
+
+# PEP 440 -- Version Identification and Dependency Specification
+# https://www.python.org/dev/peps/pep-0440/
+
+
+__version__ = "devel"
+__build__ = "devel"

bitswan-2024.4/bitswan/abc/anomaly.py

@@ -0,0 +1,41 @@
+class Anomaly(dict):
+    """
+    Description: 	Anomaly is an abstract class to be overriden for a specific anomaly and its type.
+
+    :return:
+
+    Implement: TYPE, on_tick
+
+    |
+
+    """
+
+    TYPE = None
+
+    def is_closed(self):
+        """
+        Description:
+
+        :return: sets status to closed
+
+        |
+
+        """
+        return self.get("status") == "closed"
+
+    def close(self, current_time):
+        """
+        Description:
+
+        """
+        self["ts_end"] = current_time
+        self["D"] = self["ts_end"] - self["@timestamp"]
+        self["status"] = "closed"
+
+    async def on_tick(self, current_time):
+        """
+        Description:
+
+        :hint: Implement to perform operations on the anomaly, f. e. close.
+        """
+        raise NotImplementedError()

bitswan-2024.4/bitswan/abc/connection.py

@@ -0,0 +1,71 @@
+import abc
+from bspump.asab import Configurable
+
+
+class Connection(abc.ABC, Configurable):
+    """
+    Connection class is responsible for creating a connection between items or services within the infrastructure of BSPump.
+    Their main use is to create connection with the main components of BSPump: source, :meth:`processor <bspump.Processor()>` and sink.
+
+    |
+
+    """
+
+    def __init__(self, app, id=None, config=None):
+        """
+        Description:
+
+
+        **Parameters**
+
+        app : Application
+                        Specification of an `Application <https://asab.readthedocs.io/en/latest/asab/application.html#>`_.
+
+        id : default None
+
+        config : JSON or other compatible format, default None
+                        It contains important information and data responsible for creating a connection.
+
+        """
+
+        _id = id if id is not None else self.__class__.__name__
+        super().__init__("connection:{}".format(_id), config=config)
+
+        self.App = app
+        self.Loop = app.Loop
+
+        self.Id = _id
+
+    def time(self):
+        """
+        Returns accurate time of the asynchronous process.
+
+        :hint: More information in the ASAB documentation in `UTC Time <https://asab.readthedocs.io/en/latest/asab/application.html#utc-time>`_.
+
+        |
+
+        """
+        return self.App.time()
+
+    @classmethod
+    def construct(cls, app, definition: dict):
+        """
+        Creates a connection based on a specific definition. For example, a JSON file.
+
+        **Parameters**
+
+        app : Application
+                        ID of the `Application <https://asab.readthedocs.io/en/latest/asab/application.html#>_`.
+
+        definition : definition format
+                        Defines instructions for the method that can be used to create a connection.
+
+
+        :return: cls(app, newid, config)
+
+        |
+
+        """
+        newid = definition.get("id")
+        config = definition.get("config")
+        return cls(app, newid, config)