npm - tranfi - Versions diffs - 0.0.2 → 0.1.2 - Mend

tranfi 0.0.2 → 0.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (108) hide show

package/README.md +395 -0
package/app/assets/index-6quYZ5Ap.css +5 -0
package/app/assets/index-pDFMluyz.js +160 -0
package/app/assets/materialdesignicons-webfont-B7mPwVP_.ttf +0 -0
package/app/assets/materialdesignicons-webfont-CSr8KVlo.eot +0 -0
package/app/assets/materialdesignicons-webfont-Dp5v-WZN.woff2 +0 -0
package/app/assets/materialdesignicons-webfont-PXm3-2wK.woff +0 -0
package/app/index.html +13 -0
package/binding.gyp +69 -0
package/csrc/arena.c +91 -0
package/csrc/batch.c +229 -0
package/csrc/buffer.c +78 -0
package/csrc/cJSON.c +3143 -0
package/csrc/cJSON.h +300 -0
package/csrc/codec_csv.c +1058 -0
package/csrc/codec_jsonl.c +374 -0
package/csrc/codec_table.c +218 -0
package/csrc/codec_text.c +229 -0
package/csrc/compiler.c +102 -0
package/csrc/date_utils.h +94 -0
package/csrc/dsl.c +1180 -0
package/csrc/dsl.h +22 -0
package/csrc/expr.c +1245 -0
package/csrc/expr.h +56 -0
package/csrc/internal.h +250 -0
package/csrc/ir.c +119 -0
package/csrc/ir.h +167 -0
package/csrc/ir_schema.c +60 -0
package/csrc/ir_serialize.c +104 -0
package/csrc/ir_sql.c +1211 -0
package/csrc/ir_validate.c +120 -0
package/csrc/main.c +392 -0
package/csrc/op_acf.c +133 -0
package/csrc/op_anomaly.c +120 -0
package/csrc/op_bin.c +109 -0
package/csrc/op_cast.c +195 -0
package/csrc/op_clip.c +88 -0
package/csrc/op_date_trunc.c +181 -0
package/csrc/op_datetime.c +212 -0
package/csrc/op_derive.c +248 -0
package/csrc/op_diff.c +134 -0
package/csrc/op_ewma.c +103 -0
package/csrc/op_explode.c +108 -0
package/csrc/op_fill_down.c +163 -0
package/csrc/op_fill_null.c +123 -0
package/csrc/op_filter.c +132 -0
package/csrc/op_frequency.c +193 -0
package/csrc/op_grep.c +163 -0
package/csrc/op_group_agg.c +285 -0
package/csrc/op_hash.c +126 -0
package/csrc/op_head.c +149 -0
package/csrc/op_interpolate.c +239 -0
package/csrc/op_join.c +384 -0
package/csrc/op_label_encode.c +144 -0
package/csrc/op_lead.c +190 -0
package/csrc/op_normalize.c +226 -0
package/csrc/op_onehot.c +185 -0
package/csrc/op_pivot.c +370 -0
package/csrc/op_registry.c +1148 -0
package/csrc/op_rename.c +138 -0
package/csrc/op_replace.c +202 -0
package/csrc/op_sample.c +101 -0
package/csrc/op_select.c +140 -0
package/csrc/op_skip.c +152 -0
package/csrc/op_sort.c +273 -0
package/csrc/op_split.c +114 -0
package/csrc/op_split_data.c +87 -0
package/csrc/op_stack.c +315 -0
package/csrc/op_stats.c +779 -0
package/csrc/op_step.c +171 -0
package/csrc/op_tail.c +96 -0
package/csrc/op_top.c +150 -0
package/csrc/op_trim.c +109 -0
package/csrc/op_unique.c +300 -0
package/csrc/op_unpivot.c +159 -0
package/csrc/op_validate.c +71 -0
package/csrc/op_window.c +150 -0
package/csrc/pipeline.c +315 -0
package/csrc/plan.c +206 -0
package/csrc/recipes.c +102 -0
package/csrc/recipes.h +27 -0
package/csrc/report.c +463 -0
package/csrc/report.h +22 -0
package/csrc/tranfi.h +123 -0
package/csrc/wasm_api.c +157 -0
package/napi_api.c +326 -0
package/package.json +46 -57
package/src/cli.js +193 -0
package/src/engines/duckdb.js +109 -0
package/src/index.js +306 -0
package/src/native.js +22 -0
package/src/pipeline.js +286 -0
package/src/server.js +277 -0
package/src/wasm.js +19 -0
package/wasm/index.js +244 -0
package/wasm/package.json +1 -0
package/wasm/tranfi_core.js +0 -0
package/LICENSE +0 -21
package/dist/bundle.js +0 -1
package/index.html +0 -18
package/src/app.css +0 -169
package/src/app.js +0 -203
package/src/app.vue +0 -250
package/src/bulma-input.vue +0 -110
package/src/common-inputs.js +0 -28
package/src/main.js +0 -20
package/src/transforms.js +0 -166
package/webpack.config.js +0 -108

package/index.html DELETED Viewed

@@ -1,18 +0,0 @@
-<!DOCTYPE html>
-<html>
-  <head>
-    <meta charset="utf-8" />
-    <meta name="viewport" content="width=device-width" />
-    <title>Tranfi</title>
-    <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bulma@0.9.3/css/bulma.min.css">
-    <link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Material+Icons|Material+Icons+Outlined">
-    <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@mdi/font@6.5.95/css/materialdesignicons.min.css">
-    <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@fortawesome/fontawesome-free@5.15.4/css/fontawesome.min.css">
-    <link href="https://fonts.googleapis.com/css2?family=Inter:wght@300;400;500;600;700;900&display=swap" rel="stylesheet">
-  </head>
-  <body>
-    <div id="progress"></div>
-    <div id="tranfi-container"></div>
-    <script src="dist/bundle.js"></script>
-  </body>
-</html>

package/src/app.css DELETED Viewed

@@ -1,169 +0,0 @@
-html, body {
-  font-family: sans-serif;
-  overflow-x: auto;
-  overflow-y: auto;
-}
-.navbar-brand {
-  align-items: center;
-  justify-content: space-between;
-}
-.logo {
-  color: white;
-  border-bottom: 1.5px solid white;
-  border-top: 1.5px solid white;
-  padding-left: 55px;
-  font-weight: 500;
-  text-transform: uppercase;
-}
-.pipeline {
-  min-height: calc(100vh - 77px);
-  background: #f2f2f2;
-}
-.pipeline .card.active .card-content, .card.active .card-footer {
-  display: flex;
-}
-.pipeline .card .card-content, .card .card-footer {
-  display: none;
-}
-.pipeline .card .card-header-title {
-    cursor: move;
-    cursor: grab;
-    cursor: -moz-grab;
-    cursor: -webkit-grab;
-}
-.pipeline .card .card-header-title:active {
-    cursor: grabbing;
-    cursor: -moz-grabbing;
-    cursor: -webkit-grabbing;
-}
-.columns {
-  margin: 0;
-}
-.columns:last-child {
-  margin-bottom: 0;
-}
-.column {
-  padding: 10px;
-}
-.it-drawer-body--left {
-  top: 100px !important;
-  height: calc(100% - 120px) !important;
-}
-.mainbar-left {
-  padding-top: 12px;
-  background-color: #222326;
-}
-.mainbar-right {
-  background-color: #2b2e35;
-  align-items: stretch;
-  display: flex;
-}
-.mainbar-right .navbar-start{
-  justify-content: flex-start;
-  margin-right: auto;
-  align-items: stretch;
-  display: flex;
-}
-.mainbar-right .navbar-item {
-  align-items: center;
-  display: flex;
-  color: white;
-}
-.mainbar-right input {
-  background: #222326;
-  border: none;
-  color: #c3c6ce;
-}
-.mainbar-right input::placeholder {
-  color: #6b6f79;
-}
-.mainbar-title {
-  font-weight: 500;
-  font-size: 16px;
-}
-.navbar-item {
-  padding: .5rem 0.4rem;
-}
-.breadcrumb {
-  margin: 6px auto 13px !important;
-  opacity: 0.5;
-  font-size: 12px;
-}
-.preview-panel {
-  max-width: 400px;
-  margin: 25px auto;
-}
-/*
-.mainbar-title:before {
-  content: '> ';
-  opacity: 0.4;
-  margin-right: 10px;
-  display: inline-block;
-  font-weight: 200;
-}
-*/
-hr {
-  height: 1px;
-  margin: 0.2rem 0;
-}
-.preview-button {
-  min-width: 100px;
-}
-.preview-content pre {
-  background: none;
-  font-size: 12px;
-}
-#progress {
-  background: #00d1b2;
-  position: fixed;
-  top: 0;
-  left: 0;
-  height: 3px;
-  z-index: 1000;
-}
-@media screen and (max-width:768px) {
-  .mainbar-left {
-    display: none;
-  }
-  .desktop-only {
-    display: none;
-  }
-  .preview-button {
-    min-width: 0px;
-  }
-  .button.mobile-flexible .icon {
-    margin-left: calc(-.5em - 1px) !important;
-    margin-right: calc(-.5em - 1px) !important;
-  }
-}

package/src/app.js DELETED Viewed

@@ -1,203 +0,0 @@
-const through2 = require('through2')
-const transforms = require('./transforms')
-async function run (pipeline, isBatch) {
-  let readStream
-  let writeStream
-  for (const transform of pipeline) {
-    // Convert inputs to params object
-    const params = {}
-    transform.inputs.forEach(i => {
-      params[i.name] = i.type === 'file' ? i.file : i.value
-    })
-    // Init stream
-    let nextStream = await transforms[transform.name].initStream(params)
-    if (typeof readStream === 'undefined') {
-      readStream = writeStream = nextStream
-      if ('File' in params) {
-        let size = params['File'].size
-        let progressBytes = 0
-        let progressBar = document.getElementById('progress')
-        progressBar.style.display = 'initial'
-        const progressStream = through2(function (chunk, enc, callback) {
-          progressBytes += chunk.length
-          let progressPercents = ((progressBytes / size) * 100).toFixed(1)
-          progressBar.style.width = progressPercents + '%'
-          this.push(chunk)
-          callback()
-        })
-        writeStream = writeStream.pipe(progressStream)
-      }
-    } else {
-      writeStream = writeStream.pipe(nextStream) //, {end: false})
-    }
-  }
-  writeStream.on('data', (piece) => {
-    if (isBatch) {
-      console.log('Pausing stream')
-      readStream.pause()
-    }
-  })
-  return {readStream, writeStream}
-}
-function resetInputs (inputs) {
-  inputs.forEach(input => {
-    if (input.default) {
-      input.value = input.default
-    } else {
-      switch (input.type) {
-        case 'int':
-        case 'float':
-        case 'number':
-          input.value = 0
-          break
-        case 'string':
-        case 'text':
-          input.value = ''
-          break
-        case 'color':
-          input.value = '#000000'
-          break
-        case 'categorical':
-        case 'select':
-          input.value = input.options ? input.options[0] : ''
-          break
-        case 'bool':
-        case 'checkbox':
-          input.value = false
-          break
-        case 'file':
-          input.file = null
-          input.value = ''
-          break
-        case 'group':
-          resetInputs(input.elements)
-          break
-        default:
-          input.value = ''
-      }
-    }
-  })
-}
-let readStream = null
-let writeStream = null
-export default {
-  components: {
-//    draggable,
-//    'vue-input': bulmaInput
-  },
-  data() {
-    return {
-      drag: false,
-      preview: false,
-      loadedAllData: true,
-      title: '',
-      showResults: false,
-      modalAddTransforms: false,
-      counter: 0,
-      transforms: transforms,
-      pipeline: []
-    }
-  },
-  methods: {
-    getTransformIndex (t) {
-      return this.pipeline.findIndex((el) => el.id === t.id)
-    },
-    cloneTransform (t) {
-      const i = this.getTransformIndex(t)
-      const newTransform = JSON.parse(JSON.stringify(t))
-      newTransform.id = this.counter
-      this.counter += 1
-      this.pipeline.splice(i + 1, 0, newTransform)
-    },
-    resetTransform (t) {
-      resetInputs(t.inputs)
-    },
-    deleteTransform (t) {
-      const i = this.getTransformIndex(t)
-      this.pipeline.splice(i, 1)
-    },
-    showLoading (p) {
-      this.$Loading.update(p)
-    },
-    addTransform (Transform) {
-      const newTransform = new Transform(this.counter)
-      this.resetTransform(newTransform)
-      this.pipeline.push(newTransform)
-      this.modalAddTransforms = false
-      this.counter += 1
-    },
-    showMessage (type) {
-      switch (type) {
-        case 'success':
-          this.$Message.success({ text: 'Success message!' })
-          break
-        case 'danger':
-          this.$Message.danger({ text: 'Danger message!' })
-          break
-        case 'warning':
-          this.$Message.warning({ text: 'Success message!' })
-          break
-        default:
-          this.$Message({ text: 'Primary message!' })
-          break
-      }
-    },
-    run (isBatch) {
-      run(this.pipeline, isBatch).then(streams => {
-        this.loadedAllData = false
-        readStream = streams.readStream
-        readStream.on('end', function () {
-          this.loadedAllData = true
-          console.log('End of stream')
-          let progressBar = document.getElementById('progress')
-          progressBar.style.display = 'none'
-        })
-      })
-    },
-    runMore () {
-      readStream.resume()
-    }
-  },
-  watch: {
-    // pipeline: {
-    //   handler (pipeline) {
-    //     this.env.data.inputs = pipeline.map((v, i) => ({type: 'int', name: v.name}))
-    //   },
-    //   deep: true
-    // }
-  },
-  computed: {
-    // a computed getter
-    displayInputs() {
-      const inputs = []
-      this.pipeline.forEach(transform => {
-        transform.displayInputs.forEach(i => {
-          inputs.push(transform.inputs[i])
-        })
-      })
-      return inputs
-    }
-  },
-  mounted () {
-    // const tranfi = this
-    // this.env = new JSEE({
-    //   model: {
-    //     code: function run (params) {
-    //       tranfi.run(params)
-    //     },
-    //     worker: false
-    //   },
-    //   design: {
-    //     grid: [6, 12]
-    //   },
-    //   inputs: []
-    // }, "#jsee-container")
-  }
-}

package/src/app.vue DELETED Viewed

@@ -1,250 +0,0 @@
-<template>
-  <div>
-  <it-drawer placement="left" v-model="modalAddTransforms">
-    <nav class="panel">
-      <!--
-      <p class="panel-tabs">
-        <a class="is-active">All</a>
-        <a>Public</a>
-        <a>Private</a>
-      </p>
-      -->
-      <a class="panel-block" v-for="(Transform, i) in transforms" :key="i" @click="addTransform(Transform)">
-        <span class="panel-icon">
-          <i class="mdi mdi-import" aria-hidden="true" v-if="Transform.type === 'loader'"></i>
-          <i class="mdi mdi-shape" aria-hidden="true" v-if="Transform.type === 'parser'"></i>
-          <i class="mdi mdi-play-box-outline" aria-hidden="true" v-if="Transform.type === 'output'"></i>
-        </span>
-        {{ Transform.name }}
-      </a>
-    </nav>
-  </it-drawer>
-  <nav class="mainbar" role="navigation" aria-label="main navigation" style="background: #e6e6e6; position: fixed; width: 100%; top: 0; left: 0; z-index: 100">
-    <div class="columns">
-      <div class="column is-3 mainbar-left" v-if="!preview">
-        <div class="navbar-brand">
-          <span class="logo">Tranfi</span>
-          <button class="button is-primary is-outlined" @click="modalAddTransforms = true">
-            <span class="icon is-small">
-              <i class="mdi mdi-playlist-plus" aria-hidden="true"></i>
-            </span>
-          </button>
-        </div>
-      </div>
-      <div class="column mainbar-right" v-bind:class="{ 'is-9': !preview, 'is-12': preview }">
-        <div class="navbar-start">
-          <div class="navbar-item mainbar-title">
-            <span v-if="preview">{{ title }}</span>
-            <input class="input" type="text" v-model="title" placeholder="Title..." v-else>
-          </div>
-        </div>
-        <div class="navbar-end">
-          <div class="navbar-item">
-            <div class="buttons">
-              <a class="button is-light is-outlined mobile-flexible" v-if="pipeline.length">
-                <span class="icon is-small">
-                  <i class="mdi mdi-content-save" aria-hidden="true"></i>
-                </span>
-                <span class="desktop-only">
-                  Save pipeline
-                </span>
-              </a>
-              <a class="button is-light is-outlined preview-button mobile-flexible" @click="preview=true" v-if="!preview">
-                <span class="icon is-small">
-                  <i class="mdi mdi-eye" aria-hidden="true"></i>
-                </span>
-                <span class="desktop-only">
-                  View
-                </span>
-              </a>
-              <a class="button is-light is-outlined preview-button mobile-flexible" @click="preview=false" v-if="preview">
-                <span class="icon is-small">
-                  <i class="mdi mdi-pencil" aria-hidden="true"></i>
-                </span>
-                <span class="desktop-only">
-                  Edit
-                </span>
-              </a>
-            </div>
-          </div>
-        </div>
-      </div>
-    </div>
-  </nav>
-  <div class="columns" style="margin-top: 77px">
-    <div class="pipeline column is-3" v-if="!preview">
-      <draggable
-        v-model="pipeline"
-        @start="drag=true"
-        @end="drag=false"
-        item-key="id"
-        handle=".card-header"
-      >
-        <template #item="{element}">
-          <div>
-            <div class="card" v-bind:class="{ active: element.expanded }">
-              <header class="card-header">
-                <p class="card-header-title">
-                  {{ element.name }}
-                </p>
-                <button class="card-header-icon" aria-label="more options" v-on:click="element.expanded = !element.expanded">
-                  <span class="icon">
-                    <i class="mdi mdi-chevron-up" aria-hidden="true" v-if="element.expanded"></i>
-                    <i class="mdi mdi-chevron-down" aria-hidden="true" v-else></i>
-                  </span>
-                </button>
-              </header>
-              <div class="card-content" >
-                <ul>
-                  <li v-for="(input, index) in element.inputs">
-                    <vue-input v-bind:input="input"></vue-input>
-                  </li>
-                  <li v-if="element.inputs.length">
-                    <div class="field">
-                      <label v-bind:for="'disp' + element.id" class="is-size-7">Display on preview:</label>
-                      <div class="control">
-                        <div class="select is-multiple">
-                          <select
-                            v-model="element.displayInputs"
-                            v-bind:id="'disp' + element.id"
-                            multiple
-                            size="3"
-                          >
-                            <option v-for="(input, index) in element.inputs" v-bind:value="index">
-                              {{ input.name }}
-                            </option>
-                          </select>
-                        </div>
-                      </div>
-                    </div>
-                  </li>
-                  <li>
-                    <pre style="font-size: 10px;">{{ element }}</pre>
-                  </li>
-                </ul>
-              </div>
-              <footer class="card-footer">
-                <a class="card-footer-item" @click="cloneTransform(element)">Clone</a>
-                <a class="card-footer-item" @click="resetTransform(element)">Reset</a>
-                <a class="card-footer-item" @click="deleteTransform(element)">Delete</a>
-              </footer>
-            </div>
-            <p style="text-align: center; font-size: 11px; color: #AAA;">
-              <span> ↓ </span>
-              <small>{{ element.outputType }}</small>
-            </p>
-          </div>
-        </template>
-      </draggable>
-      <button
-        @click="modalAddTransforms = true"
-        class="button is-primary is-outlined is-fullwidth"
-        style="margin-bottom: 10px; min-height: 45px;"
-      >
-        <span class="icon is-small">
-          <i class="mdi mdi-playlist-plus" aria-hidden="true"></i>
-        </span>
-        <span>
-          Add stream
-        </span>
-      </button>
-    </div>
-    <div class="column preview-content" v-bind:class="{ 'is-9': !preview, 'is-12': preview }">
-      <nav class="breadcrumb has-arrow-separator" aria-label="breadcrumbs">
-        <ul>
-          <li v-for="transform in pipeline">&nbsp;{{ transform.name }}&nbsp;</li>
-        </ul>
-      </nav>
-      <hr v-if="pipeline.length">
-      <nav class="panel preview-panel" v-if="pipeline.length">
-        <div class="panel-block" v-for="(input, index) in displayInputs">
-          <vue-input v-bind:input="input"></vue-input>
-        </div>
-        <div class="panel-block" v-if="pipeline.length">
-          <button class="button is-primary is-outlined is-fullwidth" @click="run(true)">
-            <span class="icon is-small">
-              <i class="mdi mdi-play-pause" aria-hidden="true"></i>
-            </span>
-            <span>
-              Load batch
-            </span>
-          </button>
-        </div>
-        <div class="panel-block" v-if="pipeline.length">
-          <button class="button is-primary is-large is-fullwidth" @click="run(false)">
-            <span class="icon is-small">
-              <i class="mdi mdi-play" aria-hidden="true"></i>
-            </span>
-            <span>
-              Load all
-            </span>
-          </button>
-        </div>
-      </nav>
-      <hr v-if="pipeline.length">
-      <pre id="text-output" style="display: none;"></pre>
-      <hr v-if="pipeline.length && (!loadedAllData)">
-      <nav class="panel preview-panel" v-if="pipeline.length && (!loadedAllData)">
-        <div class="panel-block" v-if="pipeline.length">
-          <button class="button is-primary is-outlined is-fullwidth" @click="runMore">
-            <span class="icon is-small">
-              <i class="mdi mdi-play-pause" aria-hidden="true"></i>
-            </span>
-            <span>
-              Load more
-            </span>
-          </button>
-        </div>
-      </nav>
-      <!--
-      <it-tag>Neutral</it-tag><it-tag>Neutral</it-tag><it-tag>Neutral</it-tag><it-tag>Neutral</it-tag>
-      <it-button @click="showMessage('Warning')" type="warning">Warning message</it-button>
-      <it-tabs box style="flex: 1">
-        <it-tab title="Tab 1">First tab</it-tab>
-        <it-tab title="Tab 2">Second tab</it-tab>
-        <it-tab title="Tab 3" :disabled="disabledTab">Third tab</it-tab>
-      </it-tabs>
-      -->
-    </div>
-  </div>
-  <!--
-  <nav class="bottombar" role="navigation" style="background: none; position: fixed; width: 100%; bottom: 0; left: 0; z-index: 100">
-    <div class="columns">
-      <div class="column is-3" v-if="!preview"></div>
-      <div class="column bottombar-right" v-bind:class="{ 'is-9': !preview, 'is-12': preview }" style="">
-        <div class="buttons" v-if="pipeline.length">
-          <button class="button is-primary" @click="run">
-            <span class="icon is-small">
-              <i class="mdi mdi-subdirectory-arrow-right" aria-hidden="true"></i>
-            </span>
-            <span>
-              Run
-            </span>
-          </button>
-        </div>
-      </div>
-    </div>
-  </nav>
-  -->
-  </div>
-</template>
-<style src="./app.css"></style>
-<script src="./app.js" scoped></script>