sec-edgar-api 0.2.1 → 0.2.2

package/README.md

@@ -24,14 +24,10 @@ Reports are all returned as a uniform interface:
 
 ```TS
 interface ReportTranslated {
-	cik: number
-	url: string | null
-	dateReport: string
+    dateReport: string
 	dateFiled: string
 	fiscalPeriod: FiscalPeriod
 	fiscalYear: number
-	splitDate: string | null
-	splitRatio: number | null
 
 	assetTotal: number | null
 	assetCurrent: number | null
@@ -96,6 +92,8 @@ interface ReportTranslated {
 	cashFlowDeferredTax: number | null
 
 	cashFlowWorkingCapitalNonCash: number | null
+
+	splitRatio: number | null
 }
 ```
 

package/build/services/DocumentParser/HtmlTableExtractor.d.ts

@@ -0,0 +1,41 @@
+export interface Cell {
+    attributes: string;
+    rowSpan: number;
+    colSpan: number;
+    rowIndex: number;
+    colIndex: number;
+    tableCellIndex: number;
+    html: string;
+    isHeaderRowCell: boolean;
+    isBodyTitleRowCell: boolean;
+    valueParsed: string | number | null;
+    headerCol: string | null;
+    headerRowIndex: number | null;
+}
+export interface TableHTMLData {
+    tableIndex: number;
+    parentTableIndex: number | null;
+    childTableIndexes: number[];
+    positionStart: number;
+    positionEnd: number;
+    htmlBefore: string;
+    html: string;
+    rows: Cell[][];
+}
+interface ParseOptions {
+    tagsToExclude?: string[];
+    tagsToInclude?: string[];
+    stripParenthesis?: boolean;
+    stripHtml?: boolean;
+}
+export default class HtmlTableExtractor {
+    extractTables(html: string, options?: ParseOptions): TableHTMLData[];
+    mergeHeaderRows(tables: TableHTMLData[]): void;
+    removeEmptyColumns(tables: TableHTMLData[]): TableHTMLData[];
+    private addTableCells;
+    private addMissingNameCol;
+    private addTableCellValues;
+    stripHtml(str: string, options?: Omit<ParseOptions, 'stripHtml' | 'stripParenthesis'>): string;
+    parseValue(str: string | number | null, options?: ParseOptions): string | number | null;
+}
+export {};

package/build/services/DocumentParser/HtmlTableExtractor.js

@@ -0,0 +1,408 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+var XMLParser_1 = require("./XMLParser");
+var HtmlTableExtractor = /** @class */ (function () {
+    function HtmlTableExtractor() {
+    }
+    HtmlTableExtractor.prototype.extractTables = function (html, options) {
+        var _a;
+        var tablesOpen = [];
+        var tableIndex = -1;
+        var htmlBefore = '';
+        var tablesData = [];
+        for (var i = 0; i < html.length; i++) {
+            var isTableStart = html.substring(i, i + 6).toLowerCase() === '<table';
+            var isTableEnd = html.substring(i - 7, i + 1).toLowerCase() === '</table>';
+            var parentTable = tablesOpen[tablesOpen.length - 1];
+            if (isTableStart) {
+                tableIndex++;
+                tablesOpen.push({
+                    tableIndex: tableIndex,
+                    parentTableIndex: (_a = parentTable === null || parentTable === void 0 ? void 0 : parentTable.tableIndex) !== null && _a !== void 0 ? _a : null,
+                    childTableIndexes: [],
+                    positionStart: i,
+                    positionEnd: -1,
+                    htmlBefore: htmlBefore,
+                    html: '',
+                    rows: [],
+                });
+                parentTable === null || parentTable === void 0 ? void 0 : parentTable.childTableIndexes.push(tableIndex);
+                htmlBefore = '';
+            }
+            if (tablesOpen.length === 0) {
+                htmlBefore += html[i];
+            }
+            for (var a = tablesOpen.length - 1; a >= 0; a--) {
+                tablesOpen[a].html += html[i];
+            }
+            if (isTableEnd && tablesOpen.length > 0) {
+                tablesOpen[tablesOpen.length - 1].positionEnd = i;
+                var tableData = tablesOpen.pop();
+                tablesData[tableData.tableIndex] = tableData;
+            }
+        }
+        this.addTableCells(tablesData);
+        this.addTableCellValues(tablesData, options);
+        this.addMissingNameCol(tablesData);
+        this.removeEmptyColumns(tablesData);
+        this.mergeHeaderRows(tablesData);
+        return tablesData.filter(Boolean);
+    };
+    HtmlTableExtractor.prototype.mergeHeaderRows = function (tables) {
+        var _a;
+        for (var _i = 0, tables_1 = tables; _i < tables_1.length; _i++) {
+            var table = tables_1[_i];
+            var bodyRowIndex = table.rows.findIndex(function (row) { return row.some(function (col) { return !col.isHeaderRowCell; }); });
+            var headerRowIndex = bodyRowIndex - 1;
+            var bodyRow = table.rows[bodyRowIndex];
+            var headerRow = table.rows[headerRowIndex];
+            if (!bodyRow || headerRowIndex < 0)
+                continue;
+            for (var i = 0; i < bodyRow.length; i++) {
+                var headerCol = headerRow[i];
+                var bodyCol = bodyRow[i];
+                headerCol.valueParsed = (_a = bodyCol.headerCol) !== null && _a !== void 0 ? _a : headerCol.valueParsed;
+            }
+        }
+    };
+    HtmlTableExtractor.prototype.removeEmptyColumns = function (tables) {
+        var _loop_1 = function (table) {
+            var emptyColumns = new Set();
+            var _loop_2 = function (c) {
+                var isAllEmpty = table.rows.every(function (row) { var _a, _b; return ((_a = row[c]) === null || _a === void 0 ? void 0 : _a.valueParsed) === null || ((_b = row[c]) === null || _b === void 0 ? void 0 : _b.isHeaderRowCell); });
+                if (isAllEmpty) {
+                    emptyColumns.add(c);
+                }
+            };
+            for (var c = 0; c < table.rows[0].length; c++) {
+                _loop_2(c);
+            }
+            for (var r = 0; r < table.rows.length; r++) {
+                var row = table.rows[r];
+                table.rows[r] = row.filter(function (_, i) { return !emptyColumns.has(i); });
+            }
+            table.rows = table.rows.filter(function (row) { return row.some(function (col) { return col.valueParsed !== null; }); });
+        };
+        for (var _i = 0, tables_2 = tables; _i < tables_2.length; _i++) {
+            var table = tables_2[_i];
+            _loop_1(table);
+        }
+        return tables;
+    };
+    HtmlTableExtractor.prototype.addTableCells = function (tables) {
+        var tablesByIndex = new Map(tables.map(function (t) { return [t.tableIndex, t]; }));
+        tablesByIndex.forEach(function (table) {
+            var _a, _b;
+            var skipIndexMap = new Map(table.childTableIndexes.map(function (childIndex) {
+                var _a, _b;
+                var child = tablesByIndex.get(childIndex);
+                return [
+                    ((_a = child === null || child === void 0 ? void 0 : child.positionStart) !== null && _a !== void 0 ? _a : 0) - table.positionStart,
+                    ((_b = child === null || child === void 0 ? void 0 : child.positionEnd) !== null && _b !== void 0 ? _b : 0) - table.positionStart,
+                ];
+            }));
+            var grid = [];
+            var isInCell = false;
+            var isInCellAtts = false;
+            var cellAtts = '';
+            var cellHTML = '';
+            var rowIndex = -1;
+            var tableCellIndex = -1;
+            var createCell = function (html, atts) {
+                var _a, _b, _c, _d, _e, _f;
+                var attributePairs = atts
+                    .toLowerCase()
+                    .split(' ')
+                    .map(function (att) { return att.split('='); });
+                var rowSpan = Number((_b = (_a = attributePairs.find(function (_a) {
+                    var key = _a[0];
+                    return key === 'rowspan';
+                })) === null || _a === void 0 ? void 0 : _a[1]) === null || _b === void 0 ? void 0 : _b.replace(/[^0-9]/g, '')) || 1;
+                var colSpan = Number((_d = (_c = attributePairs.find(function (_a) {
+                    var key = _a[0];
+                    return key === 'colspan';
+                })) === null || _c === void 0 ? void 0 : _c[1]) === null || _d === void 0 ? void 0 : _d.replace(/[^0-9]/g, '')) || 1;
+                var cell = {
+                    attributes: atts.length > 4 ? atts.substring(4, atts.length - 1) : '',
+                    html: html,
+                    colSpan: colSpan,
+                    rowSpan: rowSpan,
+                    tableCellIndex: tableCellIndex,
+                    rowIndex: rowIndex,
+                    colIndex: -1,
+                    isHeaderRowCell: false,
+                    isBodyTitleRowCell: false,
+                    valueParsed: null,
+                    headerCol: null,
+                    headerRowIndex: null,
+                };
+                // const hasCopies = cell.colSpan > 1 || cell.rowSpan > 1
+                var curRow = (_e = grid[rowIndex]) !== null && _e !== void 0 ? _e : [];
+                var nextEmptyCellIndex = curRow.findIndex(function (cell) { return !cell; });
+                var idxStart = nextEmptyCellIndex === -1 ? curRow.length : nextEmptyCellIndex;
+                for (var r = rowIndex; r < rowIndex + rowSpan; r++) {
+                    grid[r] = (_f = grid[r]) !== null && _f !== void 0 ? _f : [];
+                    for (var c = idxStart; c < idxStart + colSpan; c++) {
+                        cell.colIndex = cell.colIndex > -1 ? cell.colIndex : c;
+                        grid[r][c] = cell;
+                    }
+                }
+            };
+            for (var i = 0; i < table.html.length; i++) {
+                var skipIndex = (_a = skipIndexMap.get(i)) !== null && _a !== void 0 ? _a : null;
+                if (skipIndex) {
+                    cellHTML += table.html.substring(i, skipIndex + 1);
+                    i = skipIndex;
+                    continue;
+                }
+                var prev5Chars = table.html.substring(i - 4, i + 1).toLowerCase();
+                var next3Chars = table.html.substring(i, i + 3).toLowerCase();
+                var isCellAttsStart = ['<td', '<th'].includes(next3Chars);
+                var isSelfEnclosed = isInCellAtts && table.html[i - 1] === '/' && table.html[i] === '>';
+                var isCellAttsEnd = (isInCell && table.html[i] === '>') || isSelfEnclosed;
+                var isCellEnd = ['</td>', '</th>'].includes(prev5Chars);
+                var isRowStart = next3Chars === '<tr';
+                if (isRowStart) {
+                    rowIndex++;
+                    grid[rowIndex] = (_b = grid[rowIndex]) !== null && _b !== void 0 ? _b : [];
+                }
+                if (isCellAttsStart) {
+                    tableCellIndex++;
+                    isInCell = true;
+                    isInCellAtts = true;
+                }
+                if (isInCellAtts) {
+                    cellAtts += table.html[i];
+                }
+                if (isInCell) {
+                    cellHTML += table.html[i];
+                }
+                if (isCellAttsEnd) {
+                    isInCellAtts = false;
+                }
+                if (isCellEnd || isSelfEnclosed) {
+                    isInCell = false;
+                    isInCellAtts = false;
+                    createCell(cellHTML, cellAtts);
+                    cellHTML = '';
+                    cellAtts = '';
+                }
+            }
+            table.rows = grid;
+        });
+    };
+    HtmlTableExtractor.prototype.addMissingNameCol = function (tables) {
+        var _a, _b;
+        var _loop_3 = function (table) {
+            var bodyIndex = table.rows.findIndex(function (row) { return row.some(function (col) { return !col.isHeaderRowCell; }); });
+            // get the first column index that has a value
+            var firstPopulatedColIndex = Infinity;
+            for (var i = bodyIndex; i < table.rows.length; i++) {
+                var row = table.rows[i];
+                var populatedIndex = row.findIndex(function (col) { return col.valueParsed; });
+                var isFirstPopulatedIndex = populatedIndex > -1 && populatedIndex < firstPopulatedColIndex;
+                if (isFirstPopulatedIndex)
+                    firstPopulatedColIndex = populatedIndex;
+                if (firstPopulatedColIndex === 0)
+                    break;
+            }
+            var shouldAddName = table.rows.some(function (row) {
+                var firstCol = row[firstPopulatedColIndex];
+                var headerCol = firstCol === null || firstCol === void 0 ? void 0 : firstCol.headerCol;
+                // skip if the first column has a header col, or if there is no header row
+                if (!firstCol || headerCol || firstCol.headerRowIndex === null) {
+                    return false;
+                }
+                // if the first col is a string, assume it's a name
+                return typeof firstCol.valueParsed === 'string';
+            });
+            if (shouldAddName) {
+                for (var _c = 0, _d = table.rows; _c < _d.length; _c++) {
+                    var row = _d[_c];
+                    var col = row[firstPopulatedColIndex];
+                    if (!col)
+                        continue;
+                    var isEmptyRow = row.every(function (col) { return col.valueParsed === null; });
+                    // for header rows, add to valueParsed, body rows, set headerCol
+                    if (!isEmptyRow && col.isHeaderRowCell) {
+                        col.valueParsed = (_a = col.valueParsed) !== null && _a !== void 0 ? _a : '[name]';
+                    }
+                    else if (!col.isHeaderRowCell) {
+                        col.headerCol = (_b = col.headerCol) !== null && _b !== void 0 ? _b : '[name]';
+                    }
+                }
+            }
+        };
+        for (var _i = 0, tables_3 = tables; _i < tables_3.length; _i++) {
+            var table = tables_3[_i];
+            _loop_3(table);
+        }
+    };
+    HtmlTableExtractor.prototype.addTableCellValues = function (tables, options) {
+        var _this = this;
+        var _a;
+        var getHeaderRowIndex = function (rows) {
+            var bodyIndex = rows.findIndex(function (row, r) {
+                var _a;
+                var prevRow = (_a = rows[r - 1]) !== null && _a !== void 0 ? _a : [];
+                var hadUnderlines = prevRow.some(function (col) { return col.attributes.includes('border') && col.attributes.includes('bottom'); });
+                var hasUnderline = row.some(function (col) { return col.attributes.includes('border') && col.attributes.includes('bottom'); });
+                if (hadUnderlines && !hasUnderline) {
+                    return true;
+                }
+                return row.some(function (col) {
+                    var valueParsed = _this.parseValue(col.html, options);
+                    var isNumber = typeof valueParsed === 'number';
+                    var isYear = isNumber && valueParsed > 1900 && valueParsed < 2100;
+                    var isCol = isNumber && !isYear;
+                    return isCol;
+                });
+            });
+            return bodyIndex - 1;
+        };
+        var getNextCell = function (row, colIndex) {
+            var startingCol = row[colIndex];
+            for (var i = colIndex; i < row.length; i++) {
+                if (row[i].tableCellIndex !== (startingCol === null || startingCol === void 0 ? void 0 : startingCol.tableCellIndex)) {
+                    return row[i];
+                }
+            }
+            return null;
+        };
+        var completedCells = new Set();
+        var _loop_4 = function (table) {
+            var headerRowIndex = getHeaderRowIndex(table.rows);
+            var _loop_5 = function (rowIndex) {
+                var row = table.rows[rowIndex];
+                var countUniqueCells = new Set(row.map(function (c) { return c.tableCellIndex; })).size;
+                // skip titles in the middle of the body
+                var isBodyTitleRow = rowIndex > headerRowIndex && countUniqueCells === 1 && ((_a = row[0]) === null || _a === void 0 ? void 0 : _a.colSpan) > 0;
+                var isHeaderRow = rowIndex <= headerRowIndex;
+                var headerByIndex = new Map();
+                var getHeaderCol = function (c) {
+                    var _a, _b, _c;
+                    if (headerByIndex.has(c)) {
+                        return (_a = headerByIndex.get(c)) !== null && _a !== void 0 ? _a : null;
+                    }
+                    if (isHeaderRow) {
+                        return null;
+                    }
+                    for (var r = 0; r <= headerRowIndex; r++) {
+                        var row_1 = table.rows[r];
+                        for (var c_1 = 0; c_1 < row_1.length; c_1++) {
+                            var col = row_1[c_1];
+                            var headerCurrent = (_b = headerByIndex.get(c_1)) !== null && _b !== void 0 ? _b : '';
+                            var value = headerCurrent.endsWith("".concat(col.valueParsed || ''))
+                                ? headerCurrent
+                                : "".concat(headerCurrent, " ").concat(col.valueParsed || '').trim();
+                            headerByIndex.set(c_1, value);
+                        }
+                    }
+                    return (_c = headerByIndex.get(c)) !== null && _c !== void 0 ? _c : null;
+                };
+                for (var colIndex = 0; colIndex < row.length; colIndex++) {
+                    var cell = row[colIndex];
+                    if (completedCells.has(cell)) {
+                        continue;
+                    }
+                    cell.headerRowIndex = headerRowIndex > -1 ? headerRowIndex : null;
+                    cell.isBodyTitleRowCell = isBodyTitleRow;
+                    cell.isHeaderRowCell = isHeaderRow;
+                    // sometimes there is a rogue percent sign that is not in a column, so we need to check the next column
+                    var nextCell = getNextCell(row, colIndex);
+                    // const isMissingPercentSign =
+                    // 	nextCell?.html.includes('%') && this.parseValue(nextCell?.html) === null
+                    var isMissingParenthesis = (nextCell === null || nextCell === void 0 ? void 0 : nextCell.html.includes(')')) && cell.html.includes('(') && !cell.html.includes(')');
+                    var colValue = isMissingParenthesis ? "".concat(cell.html.trim(), ")") : cell.html.trim();
+                    // colValue = isMissingPercentSign ? `${colValue}` : colValue
+                    colValue = this_1.parseValue(colValue, options);
+                    colValue = typeof colValue === 'string' ? colValue.replace(/\s+/g, ' ') : colValue;
+                    // add parsed value
+                    cell.valueParsed = colValue;
+                    cell.headerCol = getHeaderCol(colIndex);
+                    completedCells.add(cell);
+                }
+            };
+            for (var rowIndex = 0; rowIndex < table.rows.length; rowIndex++) {
+                _loop_5(rowIndex);
+            }
+        };
+        var this_1 = this;
+        for (var _i = 0, tables_4 = tables; _i < tables_4.length; _i++) {
+            var table = tables_4[_i];
+            _loop_4(table);
+        }
+    };
+    HtmlTableExtractor.prototype.stripHtml = function (str, options) {
+        var _a = options !== null && options !== void 0 ? options : {}, _b = _a.tagsToExclude, tagsToExclude = _b === void 0 ? [] : _b, _c = _a.tagsToInclude, tagsToInclude = _c === void 0 ? [] : _c;
+        var strNew = '';
+        var iterator = new XMLParser_1.default();
+        var isAllowedPath = true;
+        var updateAllowedPath = function (data) {
+            var openTags = data.path.split('.');
+            if (tagsToInclude.length > 0) {
+                isAllowedPath = tagsToInclude.some(function (t) { return openTags.includes(t); });
+            }
+            else if (tagsToExclude.length > 0) {
+                isAllowedPath = !tagsToExclude.some(function (t) { return openTags.includes(t); });
+            }
+        };
+        iterator.iterateXML({
+            xml: str,
+            onOpenTag: function (data) {
+                updateAllowedPath(data);
+            },
+            onCloseTag: function (data) {
+                strNew += isAllowedPath && !strNew.endsWith('\n') ? '\n' : '';
+                updateAllowedPath(data);
+            },
+            onCharacter: function (data) {
+                if (isAllowedPath) {
+                    strNew += data.char;
+                }
+            },
+        });
+        return strNew;
+    };
+    HtmlTableExtractor.prototype.parseValue = function (str, options) {
+        var _a;
+        if (str === null)
+            return null;
+        if (typeof str === 'number')
+            return str;
+        var _b = options !== null && options !== void 0 ? options : {}, _c = _b.stripHtml, stripHtml = _c === void 0 ? true : _c, _d = _b.tagsToExclude, tagsToExclude = _d === void 0 ? [] : _d, _e = _b.tagsToInclude, tagsToInclude = _e === void 0 ? [] : _e, _f = _b.stripParenthesis, stripParenthesis = _f === void 0 ? false : _f;
+        var strNew = stripHtml ? this.stripHtml(str, { tagsToExclude: tagsToExclude, tagsToInclude: tagsToInclude }) : str;
+        var text = strNew
+            .replace(/&#160;|&nbsp;|\n/g, ' ')
+            .replace(/&#174;|&#9744;/g, '')
+            .replace(/&#8211;|&#8212;|&#x2014;|&#151;/g, '-')
+            .replace(/&#8217;|&#8220;|&#8221;|&rsquo;/g, "'");
+        if (stripParenthesis) {
+            text = text.replace(/\(.*?\)/g, '');
+        }
+        text = text
+            .replace(/\s+/, ' ')
+            .replace(/&([a-z0-9]+|#[0-9]{1,6}|#x[0-9a-fA-F]{1,6});/g, ' ')
+            .trim();
+        if (str.replace(/&#8211;|&#8212;|&#x2014;/g, '-') === '-')
+            return '-';
+        if (text === '')
+            return null;
+        var colNum = text.replace(/,|\(|\)|\%/g, '').trim();
+        if (colNum === '-' || colNum === '$')
+            return null;
+        colNum = colNum.replace(/\-|\$/g, '');
+        var hasNumBeforeParenthesis = Boolean(/\d+\s*(?=\()/.test(text));
+        colNum = hasNumBeforeParenthesis ? (_a = colNum.split(' ')[0]) === null || _a === void 0 ? void 0 : _a.trim() : colNum;
+        if (!isNaN(Number(colNum))) {
+            if (text.includes('%'))
+                return text.replace(/[^a-zA-Z\d\s:]/g, '') === '' ? null : text;
+            return (text.trim().includes('(') && !hasNumBeforeParenthesis) || text.includes('-')
+                ? Number(colNum) * -1
+                : Number(colNum);
+        }
+        return text;
+    };
+    return HtmlTableExtractor;
+}());
+exports.default = HtmlTableExtractor;

package/build/services/DocumentParser/XMLParser.d.ts

@@ -1,4 +1,3 @@
-import { DocumentNode } from './XMLNode/DocumentNode';
 interface OnCharacterData {
     char: string;
     index: number;
@@ -6,9 +5,7 @@ interface OnCharacterData {
     pathOccurrenceCount: number;
     attributesStr: string;
 }
-interface ParseTableNodesParams {
-    xml: string;
-}
+type ObjOrArr = Record<string | symbol | number, any> | any[];
 interface Parse2Params {
     xml: string;
     onCharacter?: (data: OnCharacterData) => void;
@@ -29,12 +26,30 @@ interface IterateTablesParams {
         textMap: Map<string, string>;
     }) => void;
 }
+interface Parser {
+    parse(xml: string): Record<string, any>;
+}
+interface XMLParserArgs {
+    parser: Parser;
+}
+interface Section {
+    text: string;
+    tables: string[][][][];
+}
 export default class XMLParser {
+    private readonly parser;
+    constructor(args?: XMLParserArgs);
+    extractJsonKey(json: Record<string, any>, extractKey: string): Record<string, any>;
+    parse(xml: string): any;
+    /**
+     * Return true in the callback if you want the loop to stop
+     */
+    iterateKeysDeep(obj: ObjOrArr, cb: (key: string, currentObj: ObjOrArr, originalObj: ObjOrArr) => void | boolean): void;
+    parseSectionTables(xml: string): Section[];
     iterateXML(params: Parse2Params): string[];
     /**
      * Returns text in each table cell mapped by `${table}.${row}.${col}`
      */
     getTableTextMap(params: IterateTablesParams): Map<string, string>;
-    getDocumentNode(params: ParseTableNodesParams): DocumentNode;
 }
 export {};