kosh 3.3.0__tar.gz → 3.4__tar.gz

{kosh-3.3.0/kosh.egg-info → kosh-3.4}/PKG-INFO

@@ -1,20 +1,18 @@
 Metadata-Version: 2.4
 Name: kosh
-Version: 3.3.0
+Version: 3.4
 Summary: Manages Data Store with External Bulk Data
-Home-page: https://github.com/LLNL/Kosh
 Author: Charles Doutriaux, Renee Olson
-Author-email: doutriaux1@llnl.gov
 License: MIT
-Classifier: Programming Language :: Python
-Classifier: License :: OSI Approved :: MIT License
-Classifier: Operating System :: OS Independent
+Project-URL: Homepage, https://github.com/LLNL/Kosh
+Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE
 License-File: NOTICE
-Requires-Dist: llnl-sina>=1.15.1
+Requires-Dist: llnl-sina>=1.16.0
 Requires-Dist: networkx>=2.6
 Requires-Dist: numpy>=1.20
+Requires-Dist: filelock
 Requires-Dist: scipy
 Requires-Dist: h5py>=3
 Requires-Dist: scikit-learn>=1.0.2
@@ -22,17 +20,7 @@ Requires-Dist: pandas
 Requires-Dist: hdbscan
 Requires-Dist: matplotlib
 Requires-Dist: tqdm
-Requires-Dist: filelock
-Dynamic: author
-Dynamic: author-email
-Dynamic: classifier
-Dynamic: description
-Dynamic: description-content-type
-Dynamic: home-page
-Dynamic: license
 Dynamic: license-file
-Dynamic: requires-dist
-Dynamic: summary
 
 ![Kosh Logo](share/icons/png/Kosh_Logo_Blue.png)
 # Overview
@@ -69,6 +57,8 @@ sys.path.append(path_to_kosh_repo)
 
 For more details look into the [installation doc](INSTALL.md)
 
+If you want to use Kosh through Codex, see [the Codex skill install guide](docs/codex_skill.md).
+
 # First steps
 
 See [this file](docs/source/users/index.md)

{kosh-3.3.0 → kosh-3.4}/README.md

@@ -33,6 +33,8 @@ sys.path.append(path_to_kosh_repo)
 
 For more details look into the [installation doc](INSTALL.md)
 
+If you want to use Kosh through Codex, see [the Codex skill install guide](docs/codex_skill.md).
+
 # First steps
 
 See [this file](docs/source/users/index.md)
@@ -54,4 +56,4 @@ Kosh is distributed under the terms of the MIT license; new contributions must b
 
 SPDX-License-Identifier: MIT
 
-``LLNL-CODE-814755``
+``LLNL-CODE-814755``

kosh-3.4/docs/gen_api_docs.py

@@ -0,0 +1,63 @@
+"""
+Helper to auto generate reference pages for the api docs.
+
+Recipe from mkdocstrings docs.
+"""
+
+from pathlib import Path
+import sys
+import mkdocs_gen_files
+
+nav = mkdocs_gen_files.Nav()
+
+src_path = Path(".")
+
+# Resolve whether in docs dir or project root (local vs readthedocs config)
+proj_toml = Path('.') / 'pyproject.toml'
+mkdocs_yml = Path('.') / 'mkdocs.yml'
+
+print(f"{proj_toml} exists: {proj_toml.exists()}")
+print(f"{mkdocs_yml} exists: {mkdocs_yml.exists()}")
+if proj_toml.exists():
+    src_path = Path('.') / 'kosh'
+elif mkdocs_yml.exists():
+    src_path = Path('.') / 'kosh'
+else:
+    mkdocs_gen_files.log.warning('Skipping API docs because "kosh" directory is missing.')
+    sys.exit()
+
+for path in sorted(src_path.rglob("*.py")):
+
+    module_path = path.relative_to(src_path).with_suffix("")
+    doc_path = path.relative_to(src_path).with_suffix(".md")
+    full_doc_path = Path("source/reference_guide/api_reference", doc_path)
+
+    parts = list(module_path.parts)
+
+    # Ensure __init__.py's get added/parsed properly and write contents to
+    # the current module's index.md
+    parts = ['kosh'] + parts  # add package prefix back on
+    if parts[-1] == "__init__":  # special handling for top level module files
+        parts = parts[:-1]
+        doc_path = doc_path.with_name("index.md")
+        full_doc_path = full_doc_path.with_name("index.md")
+        if not parts:
+            continue
+    elif parts[-1] == "__main__":
+        continue
+
+    with mkdocs_gen_files.open(full_doc_path, "w") as fd:  #
+        identifier = ".".join(parts)
+        # Set custom title for index pages to match nav entries
+        if doc_path.name == "index.md":
+            print(f"# {parts[-1].capitalize()}", file=fd)
+            print("---", file=fd)
+
+        # add the mkdoc strings hook
+        print("::: " + identifier, file=fd)
+
+    mkdocs_gen_files.set_edit_path(full_doc_path, path)  #
+
+# NOTE: SUMMARY.md has to be the name of the nav file
+with mkdocs_gen_files.open("sources/reference_guide/SUMMARY.md", "w") as nav_file:
+    nav_file.writelines(nav.build_literate_nav())

kosh-3.4/docs/source/conf.py

@@ -0,0 +1,452 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# Kosh documentation build configuration file, created by
+# sphinx-quickstart on Fri Oct 11 11:45:47 2019.
+#
+# This file is execfile()d with the current directory set to its
+# containing dir.
+#
+# Note that not all possible configuration values are present in this
+# autogenerated file.
+#
+# All configuration values have a default; values that are commented out
+# serve to show the default.
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+# Following changes to include AutoStructify to add tables to markdown + sphinx = recommonmark
+# https://github.com/readthedocs/recommonmark/blob/master/docs/conf.py
+import os
+from recommonmark.transform import AutoStructify
+# import sys
+# sys.path.insert(0, os.path.abspath('.'))
+
+# -- General configuration ------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#
+# needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+    'sphinx.ext.autodoc',
+    'sphinx.ext.doctest',
+    'sphinx.ext.viewcode',
+    'autoapi.extension',
+    "recommonmark",
+    "nbsphinx"
+]
+
+
+# NBSPHIX OPTIONS
+nbsphinx_allow_errors = True
+nbsphinx_execute = 'never'
+
+# AUTOAPI confi
+autoapi_type = "python"
+autoapi_dirs = ['../../kosh']
+
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix(es) of source filenames.
+# You can specify multiple suffix as a list of string:
+#
+# source_suffix = ['.rst', '.md']
+source_suffix = '.rst'
+
+# The encoding of source files.
+#
+# source_encoding = 'utf-8-sig'
+
+# The master toctree document.
+master_doc = 'index'
+
+# General information about the project.
+project = 'Kosh'
+copyright = '2019, Charles Doutriaux, Katie Lewis'
+author = 'Charles Doutriaux, Katie Lewis'
+
+# The version info for the project you're documenting, acts as replacement for
+# |version| and |release|, also used in various other places throughout the
+# built documents.
+#
+# The short X.Y version.
+version = '0'
+# The full version, including alpha/beta/rc tags.
+release = '1'
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#
+# This is also used if you do content translation via gettext catalogs.
+# Usually you set "language" from the command line for these cases.
+language = None
+
+# There are two options for replacing |today|: either, you set today to some
+# non-false value, then it is used:
+#
+# today = ''
+#
+# Else, today_fmt is used as the format for a strftime call.
+#
+# today_fmt = '%B %d, %Y'
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This patterns also effect to html_static_path and html_extra_path
+exclude_patterns = []
+
+# The reST default role (used for this markup: `text`) to use for all
+# documents.
+#
+# default_role = None
+
+# If true, '()' will be appended to :func: etc. cross-reference text.
+#
+# add_function_parentheses = True
+
+# If true, the current module name will be prepended to all description
+# unit titles (such as .. function::).
+#
+# add_module_names = True
+
+# If true, sectionauthor and moduleauthor directives will be shown in the
+# output. They are ignored by default.
+#
+# show_authors = False
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'sphinx'
+
+# A list of ignored prefixes for module index sorting.
+# modindex_common_prefix = []
+
+# If true, keep warnings as "system message" paragraphs in the built documents.
+# keep_warnings = False
+
+# If true, `todo` and `todoList` produce output, else they produce nothing.
+todo_include_todos = False
+
+
+# -- Options for HTML output ----------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+html_theme = 'alabaster'
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+#
+# html_theme_options = {}
+
+# Add any paths that contain custom themes here, relative to this directory.
+# html_theme_path = []
+
+# The name for this set of Sphinx documents.
+# "<project> v<release> documentation" by default.
+#
+# html_title = 'Kosh v1'
+
+# A shorter title for the navigation bar.  Default is the same as html_title.
+#
+# html_short_title = None
+
+# The name of an image file (relative to this directory) to place at the top
+# of the sidebar.
+#
+html_logo = os.path.join("..", "..", "share", "icons", "svg", "Kosh_Logo_Blue.svg")
+
+# The name of an image file (relative to this directory) to use as a favicon of
+# the docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
+# pixels large.
+#
+# html_favicon = None
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+# Add any extra paths that contain custom files (such as robots.txt or
+# .htaccess) here, relative to this directory. These files are copied
+# directly to the root of the documentation.
+#
+# html_extra_path = []
+
+# If not None, a 'Last updated on:' timestamp is inserted at every page
+# bottom, using the given strftime format.
+# The empty string is equivalent to '%b %d, %Y'.
+#
+# html_last_updated_fmt = None
+
+# If true, SmartyPants will be used to convert quotes and dashes to
+# typographically correct entities.
+#
+# html_use_smartypants = True
+
+# Custom sidebar templates, maps document names to template names.
+#
+# html_sidebars = {}
+
+# Additional templates that should be rendered to pages, maps page names to
+# template names.
+#
+# html_additional_pages = {}
+
+# If false, no module index is generated.
+#
+# html_domain_indices = True
+
+# If false, no index is generated.
+#
+# html_use_index = True
+
+# If true, the index is split into individual pages for each letter.
+#
+# html_split_index = False
+
+# If true, links to the reST sources are added to the pages.
+#
+# html_show_sourcelink = True
+
+# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
+#
+# html_show_sphinx = True
+
+# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
+#
+# html_show_copyright = True
+
+# If true, an OpenSearch description file will be output, and all pages will
+# contain a <link> tag referring to it.  The value of this option must be the
+# base URL from which the finished HTML is served.
+#
+# html_use_opensearch = ''
+
+# This is the file name suffix for HTML files (e.g. ".xhtml").
+# html_file_suffix = None
+
+# Language to be used for generating the HTML full-text search index.
+# Sphinx supports the following languages:
+#   'da', 'de', 'en', 'es', 'fi', 'fr', 'h', 'it', 'ja'
+#   'nl', 'no', 'pt', 'ro', 'r', 'sv', 'tr', 'zh'
+#
+# html_search_language = 'en'
+
+# A dictionary with options for the search language support, empty by default.
+# 'ja' uses this config value.
+# 'zh' user can custom change `jieba` dictionary path.
+#
+# html_search_options = {'type': 'default'}
+
+# The name of a javascript file (relative to the configuration directory) that
+# implements a search results scorer. If empty, the default will be used.
+#
+# html_search_scorer = 'scorer.js'
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'Koshdoc'
+
+# -- Options for LaTeX output ---------------------------------------------
+
+latex_elements = {
+     # The paper size ('letterpaper' or 'a4paper').
+     #
+     # 'papersize': 'letterpaper',
+
+     # The font size ('10pt', '11pt' or '12pt').
+     #
+     # 'pointsize': '10pt',
+
+     # Additional stuff for the LaTeX preamble.
+     #
+     # 'preamble': '',
+
+     # Latex figure (float) alignment
+     #
+     # 'figure_align': 'htbp',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+#  author, documentclass [howto, manual, or own class]).
+latex_documents = [
+    (master_doc, 'Kosh.tex', 'Kosh Documentation',
+     'Charles Doutriaux, Katie Lewis', 'manual'),
+]
+
+# The name of an image file (relative to this directory) to place at the top of
+# the title page.
+#
+# latex_logo = None
+
+# For "manual" documents, if this is true, then toplevel headings are parts,
+# not chapters.
+#
+# latex_use_parts = False
+
+# If true, show page references after internal links.
+#
+# latex_show_pagerefs = False
+
+# If true, show URL addresses after external links.
+#
+# latex_show_urls = False
+
+# Documents to append as an appendix to all manuals.
+#
+# latex_appendices = []
+
+# If false, no module index is generated.
+#
+# latex_domain_indices = True
+
+
+# -- Options for manual page output ---------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+    (master_doc, 'kosh', 'Kosh Documentation',
+     [author], 1)
+]
+
+# If true, show URL addresses after external links.
+#
+# man_show_urls = False
+
+
+# -- Options for Texinfo output -------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+#  dir menu entry, description, category)
+texinfo_documents = [
+    (master_doc, 'Kosh', 'Kosh Documentation',
+     author, 'Kosh', 'One line description of project.',
+     'Miscellaneous'),
+]
+
+# Documents to append as an appendix to all manuals.
+#
+# texinfo_appendices = []
+
+# If false, no module index is generated.
+#
+# texinfo_domain_indices = True
+
+# How to display URL addresses: 'footnote', 'no', or 'inline'.
+#
+# texinfo_show_urls = 'footnote'
+
+# If true, do not generate a @detailmenu in the "Top" node's menu.
+#
+# texinfo_no_detailmenu = False
+
+
+# -- Options for Epub output ----------------------------------------------
+
+# Bibliographic Dublin Core info.
+epub_title = project
+epub_author = author
+epub_publisher = author
+epub_copyright = copyright
+
+# The basename for the epub file. It defaults to the project name.
+# epub_basename = project
+
+# The HTML theme for the epub output. Since the default themes are not
+# optimized for small screen space, using the same theme for HTML and epub
+# output is usually not wise. This defaults to 'epub', a theme designed to save
+# visual space.
+#
+# epub_theme = 'epub'
+
+# The language of the text. It defaults to the language option
+# or 'en' if the language is not set.
+#
+# epub_language = ''
+
+# The scheme of the identifier. Typical schemes are ISBN or URL.
+# epub_scheme = ''
+
+# The unique identifier of the text. This can be a ISBN number
+# or the project homepage.
+#
+# epub_identifier = ''
+
+# A unique identification for the text.
+#
+# epub_uid = ''
+
+# A tuple containing the cover image and cover page html template filenames.
+#
+# epub_cover = ()
+
+# A sequence of (type, uri, title) tuples for the guide element of content.opf.
+#
+# epub_guide = ()
+
+# HTML files that should be inserted before the pages created by sphinx.
+# The format is a list of tuples containing the path and title.
+#
+# epub_pre_files = []
+
+# HTML files that should be inserted after the pages created by sphinx.
+# The format is a list of tuples containing the path and title.
+#
+# epub_post_files = []
+
+# A list of files that should not be packed into the epub file.
+epub_exclude_files = ['search.html']
+
+# The depth of the table of contents in toc.ncx.
+#
+# epub_tocdepth = 3
+
+# Allow duplicate toc entries.
+#
+# epub_tocdup = True
+
+# Choose between 'default' and 'includehidden'.
+#
+# epub_tocscope = 'default'
+
+# Fix unsupported image types using the Pillow.
+#
+# epub_fix_images = False
+
+# Scale large images.
+#
+# epub_max_image_width = 0
+
+# How to display URL addresses: 'footnote', 'no', or 'inline'.
+#
+# epub_show_urls = 'inline'
+
+# If false, no index is generated.
+#
+# epub_use_index = True
+
+# app setup hook
+
+
+def setup(app):
+    app.add_config_value('recommonmark_config', {
+        # 'url_resolver': lambda url: github_doc_root + url,
+        'auto_toc_tree_section': 'Contents',
+        'enable_math': False,
+        'enable_inline_math': False,
+        'enable_eval_rst': True,
+        'enable_auto_doc_ref': True,
+    }, True)
+    app.add_transform(AutoStructify)

kosh-3.4/examples/cluster.py

@@ -0,0 +1,70 @@
+# Inspired by:
+# https://scikit-learn.org/stable/modules/generated/sklearn.datasets.make_blobs.html#sklearn.datasets.make_blobs
+import kosh
+import matplotlib.pyplot as plt
+from sklearn import datasets
+
+
+class SKDatasetLoader(kosh.loaders.KoshLoader):
+    types = {"sk_dataset": ["numpy"]}
+
+    def extract(self):
+        args, kargs = self._user_passed_parameters
+        if self.feature == "moon":
+            return datasets.make_moons(random_state=8, **kargs)[0]
+        elif self.feature == "circle":
+            return datasets.make_circles(random_state=8, **kargs)[0]
+        elif self.feature == "blob":
+            return datasets.make_blobs(random_state=8, **kargs)[0]
+        else:
+            raise RuntimeError("not a feature")
+
+    def list_features(self):
+        return ["moon", "circle", "blob"]
+
+
+store = kosh.utils.create_new_db("crp.sql")
+
+store.add_loader(SKDatasetLoader)
+
+ds = store.create()
+ds.associate("/blah", mime_type="sk_dataset")
+print(ds._associated_data_)
+# ============
+# Generate datasets. We choose the size big enough to see the scalability
+# of the algorithms, but not too big to avoid too long running times
+# ============
+n_samples = 1500
+features = [
+    ("circle", {"factor": 0.5, "noise": 0.05}),
+    ("moon", {"noise": 0.05}),
+    ("blob", {}),
+]
+
+SC = kosh.transformers.StandardScaler()
+
+estimators = [
+    (kosh.transformers.KMeans, {"n_clusters": 3}),
+    (kosh.transformers.DBSCAN, {"eps": 0.3}),
+]
+colors = ["orange", "blue", "red", "green", "purple", "salmon", "pink", "grey", "brown", "beige"]
+
+f, axarr = plt.subplots(len(estimators), len(features))
+for i, (feature, args) in enumerate(features):
+    raw = ds.get(feature, n_samples=n_samples, transformers=[SC], **args)
+    # print(raw.shape)
+    for j, (est, kargs) in enumerate(estimators):
+        axarr[j, i].scatter(raw[:, 0], raw[:, 1], color="black", s=5)
+        E = est(**kargs)
+        estimator = ds.get(
+            feature, n_samples=n_samples, transformers=[SC, E], format="estimator", **args
+        )
+        # print(estimator)
+        E = est(n_samples=100., sampling_method="percent", **kargs)
+        labels, data = ds.get(feature, n_samples=n_samples, transformers=[SC, E], format="numpy", **args)
+        # print(len(labels))
+        for k, dat in enumerate(data):
+            print(dat.shape, k)
+            axarr[j, i].scatter(dat[:, 0], dat[:, 1], color=colors[k], s=5)
+
+plt.show()