sec-edgar-api 0.0.2 → 0.0.5

package/README.md

@@ -3,6 +3,12 @@
 Fetch and parse earnings reports and other documents filed with the SEC using the EDGAR API.
 This package is focused on the earnings reports for stock analysis.
 
+## Installation
+
+```SH
+npm install sec-edgar-api
+```
+
 ## Report Interface
 
 Reports are all returned as a uniform interface:
@@ -100,10 +106,12 @@ const reports = await secEdgarApi.getReports({ symbol: 'AAPL' })
 or download all data from the SEC website and read directly from the files so you don't have to worry about rate limiting.
 
 ```TS
+import { factsDownloader } from 'sec-edgar-api/downloader'
+
 const downloadDirectory = './downloads/companyfacts'
 
 // Download companyfacts directory from sec.gov (over 15GB)
-await secEdgarApi.downloadCompanyFactsDirectory({
+await factsDownloader.downloadCompanyFactsDirectory({
     outputDirname: downloadDirectory,
     onDownloadComplete: () => process.stdout.write('\n'),
     onChunk: ({ percentComplete, stage }) => {

package/build/downloader.d.ts

@@ -0,0 +1,6 @@
+import FactsDownloader from './services/FactsDownloader';
+/**
+ * Downloads companyfacts.zip from sec.gov and extracts the directory
+ */
+declare const factsDownloader: FactsDownloader;
+export { factsDownloader };

package/build/downloader.js

@@ -0,0 +1,9 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.factsDownloader = void 0;
+var FactsDownloader_1 = require("./services/FactsDownloader");
+/**
+ * Downloads companyfacts.zip from sec.gov and extracts the directory
+ */
+var factsDownloader = new FactsDownloader_1.default();
+exports.factsDownloader = factsDownloader;

package/build/services/Client/Client.d.ts

@@ -0,0 +1,44 @@
+/// <reference types="node" />
+/// <reference types="node" />
+import { ClientRequest, IncomingMessage, RequestOptions } from 'http';
+type Primitive = string | number | boolean | null | undefined;
+interface HttpClient {
+    request: (options: string | URL | RequestOptions, callback?: (res: IncomingMessage) => void) => ClientRequest;
+}
+export interface OnChunkData {
+    percentComplete: number;
+    chunk: Buffer;
+}
+export interface ClientResponse {
+    statusCode: number;
+    message: string;
+    data: Buffer | null;
+}
+export interface RequestParams {
+    url: string;
+    headers?: Record<string, string>;
+    method?: 'GET' | 'POST' | 'PUT' | 'DELETE';
+    data?: string | Record<string, Primitive | object>;
+    timeout?: number;
+    onError?: (err: Error) => void;
+    onChunk?: (data: OnChunkData) => void;
+    onResponse?: (response: IncomingMessage) => void;
+    onSuccess?: (response: ClientResponse) => void;
+    resolveData?: boolean;
+}
+export interface ClientArgs {
+    httpClient?: HttpClient;
+    defaultHeaders?: Record<string, string>;
+}
+export interface IClient {
+    request(params: RequestParams): Promise<ClientResponse>;
+    setDefaultHeaders?(headers: Record<string, string>): void;
+}
+export default class Client implements IClient {
+    private readonly httpClient;
+    private defaultHeaders;
+    constructor(args?: ClientArgs);
+    setDefaultHeaders(headers: Record<string, string>): void;
+    request(params: RequestParams): Promise<ClientResponse>;
+}
+export {};

package/build/services/Client/Client.js

@@ -0,0 +1,104 @@
+"use strict";
+var __assign = (this && this.__assign) || function () {
+    __assign = Object.assign || function(t) {
+        for (var s, i = 1, n = arguments.length; i < n; i++) {
+            s = arguments[i];
+            for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p))
+                t[p] = s[p];
+        }
+        return t;
+    };
+    return __assign.apply(this, arguments);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+var https = require("https");
+var Client = /** @class */ (function () {
+    function Client(args) {
+        if (args === void 0) { args = {
+            httpClient: https,
+            defaultHeaders: {
+                // this can be any user agent, just not empty
+                'User-Agent': 'Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36 RuxitSynthetic/1.0 v1060014908909962014 t8797611264074608560 ath259cea6f altpriv cvcv=2 smf=0',
+            },
+        }; }
+        var httpClient = args.httpClient, defaultHeaders = args.defaultHeaders;
+        this.httpClient = httpClient !== null && httpClient !== void 0 ? httpClient : https;
+        this.defaultHeaders = defaultHeaders !== null && defaultHeaders !== void 0 ? defaultHeaders : {};
+    }
+    Client.prototype.setDefaultHeaders = function (headers) {
+        this.defaultHeaders = headers;
+    };
+    Client.prototype.request = function (params) {
+        var _this = this;
+        var url = params.url, data = params.data, headers = params.headers, onChunk = params.onChunk, onResponse = params.onResponse, onError = params.onError, onSuccess = params.onSuccess, _a = params.resolveData, resolveData = _a === void 0 ? true : _a, _b = params.method, method = _b === void 0 ? 'GET' : _b, _c = params.timeout, timeout = _c === void 0 ? 86400000 : _c;
+        var allHeaders = __assign(__assign({}, this.defaultHeaders), headers);
+        return new Promise(function (resolve, reject) {
+            var responseData = '';
+            var request = _this.httpClient.request(url, function (res) {
+                var _a, _b, _c;
+                var lengthTotal = parseInt((_a = res.headers['content-length']) !== null && _a !== void 0 ? _a : '0');
+                var lengthCurrent = 0;
+                onResponse === null || onResponse === void 0 ? void 0 : onResponse(res);
+                if (res.statusCode !== 200) {
+                    reject({
+                        statusCode: (_b = res.statusCode) !== null && _b !== void 0 ? _b : 400,
+                        message: (_c = res.statusMessage) !== null && _c !== void 0 ? _c : 'Bad Request',
+                        data: null,
+                    });
+                }
+                res.on('data', function (chunk) {
+                    lengthCurrent += chunk.length;
+                    if (resolveData) {
+                        responseData += chunk;
+                    }
+                    onChunk === null || onChunk === void 0 ? void 0 : onChunk({
+                        percentComplete: lengthCurrent / lengthTotal,
+                        chunk: Buffer.from(chunk),
+                    });
+                });
+                res.on('error', function (err) {
+                    var _a, _b;
+                    onError === null || onError === void 0 ? void 0 : onError(err);
+                    reject({
+                        statusCode: (_a = res.statusCode) !== null && _a !== void 0 ? _a : 400,
+                        message: (_b = res.statusMessage) !== null && _b !== void 0 ? _b : 'Bad Request',
+                        data: null,
+                    });
+                });
+                res.on('end', function () {
+                    var _a, _b;
+                    var buffer = Buffer.from(responseData);
+                    var clientResponse = {
+                        statusCode: (_a = res.statusCode) !== null && _a !== void 0 ? _a : 200,
+                        message: (_b = res.statusMessage) !== null && _b !== void 0 ? _b : 'OK',
+                        data: buffer,
+                    };
+                    onSuccess === null || onSuccess === void 0 ? void 0 : onSuccess(clientResponse);
+                    resolve(clientResponse);
+                });
+            });
+            request.on('timeout', function () {
+                request.destroy();
+                reject({
+                    statusCode: null,
+                    message: 'Connection Timeout',
+                    data: null,
+                });
+            });
+            request.setTimeout(timeout, function () { return reject("timeout after ".concat(timeout, "ms")); });
+            for (var key in allHeaders) {
+                request.setHeader(key, allHeaders[key]);
+            }
+            if (data) {
+                if (typeof data === 'string')
+                    request.write(data);
+                else
+                    request.write(JSON.stringify(data));
+            }
+            request.method = method;
+            request.end();
+        });
+    };
+    return Client;
+}());
+exports.default = Client;

package/build/services/Client/index.d.ts

@@ -0,0 +1,3 @@
+import Client from './Client';
+export default Client;
+export * from './Client';

package/build/services/Client/index.js

@@ -0,0 +1,19 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+var Client_1 = require("./Client");
+exports.default = Client_1.default;
+__exportStar(require("./Client"), exports);

package/build/services/FactsDownloader/Downloader.d.ts

@@ -0,0 +1,26 @@
+/// <reference types="node" />
+import { WriteStream } from 'fs';
+import { IClient, RequestParams } from '../Client';
+export interface DownloadParams extends RequestParams {
+    filename: string;
+    createDirIfNotExists?: boolean;
+}
+export interface IDownloader {
+    download(params: DownloadParams): Promise<boolean>;
+}
+interface FileManager {
+    createWriteStream(path: string): WriteStream;
+    existsSync(path: string): boolean;
+    mkdirSync(path: string): void;
+}
+interface DownloaderArgs {
+    client: IClient;
+    fileManager: FileManager;
+}
+export default class Downlaoder implements IDownloader {
+    private readonly client;
+    private readonly fileManager;
+    constructor(args?: DownloaderArgs);
+    download(params: DownloadParams): Promise<boolean>;
+}
+export {};

package/build/services/FactsDownloader/Downloader.js

@@ -0,0 +1,102 @@
+"use strict";
+var __assign = (this && this.__assign) || function () {
+    __assign = Object.assign || function(t) {
+        for (var s, i = 1, n = arguments.length; i < n; i++) {
+            s = arguments[i];
+            for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p))
+                t[p] = s[p];
+        }
+        return t;
+    };
+    return __assign.apply(this, arguments);
+};
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __generator = (this && this.__generator) || function (thisArg, body) {
+    var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
+    return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
+    function verb(n) { return function (v) { return step([n, v]); }; }
+    function step(op) {
+        if (f) throw new TypeError("Generator is already executing.");
+        while (g && (g = 0, op[0] && (_ = 0)), _) try {
+            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
+            if (y = 0, t) op = [op[0] & 2, t.value];
+            switch (op[0]) {
+                case 0: case 1: t = op; break;
+                case 4: _.label++; return { value: op[1], done: false };
+                case 5: _.label++; y = op[1]; op = [0]; continue;
+                case 7: op = _.ops.pop(); _.trys.pop(); continue;
+                default:
+                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
+                    if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
+                    if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
+                    if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
+                    if (t[2]) _.ops.pop();
+                    _.trys.pop(); continue;
+            }
+            op = body.call(thisArg, _);
+        } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
+        if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
+    }
+};
+var __rest = (this && this.__rest) || function (s, e) {
+    var t = {};
+    for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p) && e.indexOf(p) < 0)
+        t[p] = s[p];
+    if (s != null && typeof Object.getOwnPropertySymbols === "function")
+        for (var i = 0, p = Object.getOwnPropertySymbols(s); i < p.length; i++) {
+            if (e.indexOf(p[i]) < 0 && Object.prototype.propertyIsEnumerable.call(s, p[i]))
+                t[p[i]] = s[p[i]];
+        }
+    return t;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+var fs = require("fs");
+var Client_1 = require("../Client");
+var Downlaoder = /** @class */ (function () {
+    function Downlaoder(args) {
+        if (args === void 0) { args = { client: new Client_1.default(), fileManager: fs }; }
+        var client = args.client, fileManager = args.fileManager;
+        this.client = client;
+        this.fileManager = fileManager;
+    }
+    Downlaoder.prototype.download = function (params) {
+        var _this = this;
+        var filename = params.filename, onSuccess = params.onSuccess, onError = params.onError, _a = params.createDirIfNotExists, createDirIfNotExists = _a === void 0 ? false : _a, rest = __rest(params, ["filename", "onSuccess", "onError", "createDirIfNotExists"]);
+        return new Promise(function (resolve, reject) { return __awaiter(_this, void 0, void 0, function () {
+            var dir, file;
+            return __generator(this, function (_a) {
+                dir = filename.substring(0, filename.lastIndexOf('/'));
+                if (dir.length > 0 && createDirIfNotExists && !this.fileManager.existsSync(dir)) {
+                    this.fileManager.mkdirSync(dir);
+                }
+                file = this.fileManager.createWriteStream(filename);
+                file.on('close', function () { return resolve(true); });
+                file.on('finish', function () { return file.close(); });
+                file.on('error', function (err) {
+                    file.destroy();
+                    onError === null || onError === void 0 ? void 0 : onError(err);
+                    reject(false);
+                });
+                this.client
+                    .request(__assign(__assign({ onError: onError }, rest), { onResponse: function (res) {
+                        var _a;
+                        res.pipe(file);
+                        (_a = rest.onResponse) === null || _a === void 0 ? void 0 : _a.call(rest, res);
+                    } }))
+                    .then(function (res) { return onSuccess === null || onSuccess === void 0 ? void 0 : onSuccess(res); })
+                    .catch(function () { return reject(false); });
+                return [2 /*return*/];
+            });
+        }); });
+    };
+    return Downlaoder;
+}());
+exports.default = Downlaoder;

package/build/services/FactsDownloader/FactsDownloader.d.ts

@@ -0,0 +1,37 @@
+/// <reference types="node" />
+import { IDownloader } from './Downloader';
+import Unzipper from './Unzipper';
+interface FactsDownloaderArgs {
+    downloader: IDownloader;
+    unzipper: Unzipper;
+}
+interface OnChunkData {
+    percentComplete: number;
+    chunk: Buffer;
+    stage: 'download' | 'unzip';
+}
+export interface DownloadCompanyFactsDirectoryParams {
+    outputDirname: string;
+    unzip?: boolean;
+    onChunk?: (data: OnChunkData) => void;
+    onDownloadComplete?: () => void;
+    onComplete?: () => void;
+    onError?: (err: Error) => void;
+}
+export interface IFactsDownloader {
+    downloadCompanyFactsDirectory(params: DownloadCompanyFactsDirectoryParams): Promise<boolean>;
+}
+export default class FactsDownloader implements IFactsDownloader {
+    private readonly unzipper;
+    private readonly downloader;
+    constructor(args?: FactsDownloaderArgs);
+    /**
+     * Downloads the companyfacts.zip file and extracts the directory containing all company
+     * reports available from sec.gov. After downloading, you can use factFileReader and reportParser
+     * to get and read reports.
+     *
+     * Note: Over 15GB of data is downloaded and extracted.
+     */
+    downloadCompanyFactsDirectory(params: DownloadCompanyFactsDirectoryParams): Promise<boolean>;
+}
+export {};

package/build/services/FactsDownloader/FactsDownloader.js

@@ -0,0 +1,131 @@
+"use strict";
+var __assign = (this && this.__assign) || function () {
+    __assign = Object.assign || function(t) {
+        for (var s, i = 1, n = arguments.length; i < n; i++) {
+            s = arguments[i];
+            for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p))
+                t[p] = s[p];
+        }
+        return t;
+    };
+    return __assign.apply(this, arguments);
+};
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __generator = (this && this.__generator) || function (thisArg, body) {
+    var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
+    return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
+    function verb(n) { return function (v) { return step([n, v]); }; }
+    function step(op) {
+        if (f) throw new TypeError("Generator is already executing.");
+        while (g && (g = 0, op[0] && (_ = 0)), _) try {
+            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
+            if (y = 0, t) op = [op[0] & 2, t.value];
+            switch (op[0]) {
+                case 0: case 1: t = op; break;
+                case 4: _.label++; return { value: op[1], done: false };
+                case 5: _.label++; y = op[1]; op = [0]; continue;
+                case 7: op = _.ops.pop(); _.trys.pop(); continue;
+                default:
+                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
+                    if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
+                    if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
+                    if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
+                    if (t[2]) _.ops.pop();
+                    _.trys.pop(); continue;
+            }
+            op = body.call(thisArg, _);
+        } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
+        if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
+    }
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+var Downloader_1 = require("./Downloader");
+var Unzipper_1 = require("./Unzipper");
+var FactsDownloader = /** @class */ (function () {
+    function FactsDownloader(args) {
+        if (args === void 0) { args = {
+            downloader: new Downloader_1.default(),
+            unzipper: new Unzipper_1.default(),
+        }; }
+        var unzipper = args.unzipper, downloader = args.downloader;
+        this.unzipper = unzipper;
+        this.downloader = downloader;
+    }
+    /**
+     * Downloads the companyfacts.zip file and extracts the directory containing all company
+     * reports available from sec.gov. After downloading, you can use factFileReader and reportParser
+     * to get and read reports.
+     *
+     * Note: Over 15GB of data is downloaded and extracted.
+     */
+    FactsDownloader.prototype.downloadCompanyFactsDirectory = function (params) {
+        return __awaiter(this, void 0, void 0, function () {
+            var outputDirname, onChunk, onDownloadComplete, onError, onComplete, _a, unzip;
+            var _this = this;
+            return __generator(this, function (_b) {
+                outputDirname = params.outputDirname, onChunk = params.onChunk, onDownloadComplete = params.onDownloadComplete, onError = params.onError, onComplete = params.onComplete, _a = params.unzip, unzip = _a === void 0 ? true : _a;
+                return [2 /*return*/, new Promise(function (resolve, reject) { return __awaiter(_this, void 0, void 0, function () {
+                        var filename, e_1;
+                        return __generator(this, function (_a) {
+                            switch (_a.label) {
+                                case 0:
+                                    filename = "".concat(outputDirname, ".zip");
+                                    _a.label = 1;
+                                case 1:
+                                    _a.trys.push([1, 5, , 6]);
+                                    // download from sec
+                                    return [4 /*yield*/, this.downloader.download({
+                                            url: 'https://www.sec.gov/Archives/edgar/daily-index/xbrl/companyfacts.zip',
+                                            filename: filename,
+                                            createDirIfNotExists: true,
+                                            resolveData: false,
+                                            headers: {
+                                                'Accept-Encoding': 'gzip, deflate',
+                                            },
+                                            onError: function (err) { return reject(err); },
+                                            onChunk: onChunk ? function (data) { return onChunk === null || onChunk === void 0 ? void 0 : onChunk(__assign(__assign({}, data), { stage: 'download' })); } : undefined,
+                                        })];
+                                case 2:
+                                    // download from sec
+                                    _a.sent();
+                                    onDownloadComplete === null || onDownloadComplete === void 0 ? void 0 : onDownloadComplete();
+                                    if (!unzip) return [3 /*break*/, 4];
+                                    // unzip companyfacts.zip
+                                    return [4 /*yield*/, this.unzipper.unzip({
+                                            inputFilename: filename,
+                                            outputDirname: outputDirname,
+                                            deleteOriginal: true,
+                                            onError: function (err) { return reject(err); },
+                                            onChunk: onChunk ? function (data) { return onChunk === null || onChunk === void 0 ? void 0 : onChunk(__assign(__assign({}, data), { stage: 'unzip' })); } : undefined,
+                                        })];
+                                case 3:
+                                    // unzip companyfacts.zip
+                                    _a.sent();
+                                    _a.label = 4;
+                                case 4:
+                                    onComplete === null || onComplete === void 0 ? void 0 : onComplete();
+                                    resolve(true);
+                                    return [3 /*break*/, 6];
+                                case 5:
+                                    e_1 = _a.sent();
+                                    onError === null || onError === void 0 ? void 0 : onError(e_1);
+                                    reject(e_1);
+                                    return [3 /*break*/, 6];
+                                case 6: return [2 /*return*/];
+                            }
+                        });
+                    }); })];
+            });
+        });
+    };
+    return FactsDownloader;
+}());
+exports.default = FactsDownloader;

package/build/services/FactsDownloader/Unzipper.d.ts

@@ -0,0 +1,40 @@
+/// <reference types="node" />
+/// <reference types="node" />
+import * as fs from 'fs';
+import { ReadStream } from 'fs';
+import { ParseStream } from 'unzipper';
+interface IFileManager {
+    createReadStream(path: string): ReadStream;
+    unlinkSync(path: string): void;
+    statSync(path: string): fs.Stats;
+}
+interface IFileUnzipper {
+    Extract(options: {
+        path: string;
+    }): ParseStream;
+}
+interface UnzipperArgs {
+    unzipper: IFileUnzipper;
+    fileManager: IFileManager;
+}
+interface OnChunkData {
+    percentComplete: number;
+    chunk: Buffer;
+}
+interface UnzipParams {
+    inputFilename: string;
+    outputDirname: string;
+    onChunk?: (data: OnChunkData) => void;
+    onError?: (err: Error) => void;
+    deleteOriginal?: boolean;
+}
+export interface IUnzipper {
+    unzip(params: UnzipParams): Promise<any>;
+}
+export default class Unzipper implements IUnzipper {
+    private readonly fileManager;
+    private readonly unzipper;
+    constructor(args?: UnzipperArgs);
+    unzip(params: UnzipParams): Promise<unknown>;
+}
+export {};

package/build/services/FactsDownloader/Unzipper.js

@@ -0,0 +1,40 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+var fs = require("fs");
+var _unzipper = require("unzipper");
+var Unzipper = /** @class */ (function () {
+    function Unzipper(args) {
+        if (args === void 0) { args = { fileManager: fs, unzipper: _unzipper }; }
+        var fileManager = args.fileManager, unzipper = args.unzipper;
+        this.fileManager = fileManager;
+        this.unzipper = unzipper;
+    }
+    Unzipper.prototype.unzip = function (params) {
+        var _this = this;
+        var inputFilename = params.inputFilename, outputDirname = params.outputDirname, onChunk = params.onChunk, onError = params.onError, _a = params.deleteOriginal, deleteOriginal = _a === void 0 ? false : _a;
+        return new Promise(function (resolve, reject) {
+            var file = _this.fileManager.createReadStream(inputFilename);
+            var filesize = _this.fileManager.statSync(inputFilename).size;
+            var unzipped = _this.unzipper.Extract({ path: outputDirname });
+            var lengthCurrent = 0;
+            file.on('data', function (chunk) {
+                lengthCurrent += chunk.length;
+                var percentComplete = lengthCurrent / filesize;
+                onChunk === null || onChunk === void 0 ? void 0 : onChunk({ percentComplete: percentComplete, chunk: chunk });
+            });
+            file.pipe(unzipped);
+            file.on('end', function () {
+                if (deleteOriginal) {
+                    _this.fileManager.unlinkSync(inputFilename);
+                }
+                resolve(true);
+            });
+            file.on('error', function (err) {
+                onError === null || onError === void 0 ? void 0 : onError(err);
+                reject(false);
+            });
+        });
+    };
+    return Unzipper;
+}());
+exports.default = Unzipper;

package/build/services/FactsDownloader/index.d.ts

@@ -0,0 +1,2 @@
+import FactsDownloader from './FactsDownloader';
+export default FactsDownloader;

package/build/services/FactsDownloader/index.js

@@ -0,0 +1,4 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+var FactsDownloader_1 = require("./FactsDownloader");
+exports.default = FactsDownloader_1.default;

package/build/services/ReportParser/ReportParser.d.ts

@@ -35,6 +35,10 @@ export default class ReportParser {
      * @see https://www.sec.gov/edgar/sec-api-documentation
      */
     parseReportsRaw(companyFactListData: Pick<CompanyFactListData, 'facts'>, options?: ParseReportsOptions): ReportRaw[];
+    /**
+     * parseReportsRaw but removes meta data from the report
+     */
+    parseReportsRawNoMeta(companyFactListData: Pick<CompanyFactListData, 'facts'>, options?: ParseReportsOptions): Record<string, number>[];
     /**
      * Avoids deep nesting logic while iteratating through company facts
      *

package/build/services/ReportParser/ReportParser.js

@@ -53,6 +53,25 @@ var ReportParser = /** @class */ (function () {
     ReportParser.prototype.parseReportsRaw = function (companyFactListData, options) {
         return this.reportRawParser.parseReports(companyFactListData, options);
     };
+    /**
+     * parseReportsRaw but removes meta data from the report
+     */
+    ReportParser.prototype.parseReportsRawNoMeta = function (companyFactListData, options) {
+        var reportsRaw = this.parseReportsRaw(companyFactListData, options);
+        reportsRaw.forEach(function (reportRaw) {
+            var report = reportRaw;
+            delete report.dateFiled;
+            delete report.dateReport;
+            delete report.fiscalPeriod;
+            delete report.fiscalYear;
+            delete report.form;
+            delete report.frame;
+            delete report.isTTM;
+            delete report.reportType;
+            delete report.taxonomy;
+        });
+        return reportsRaw;
+    };
     /**
      * Avoids deep nesting logic while iteratating through company facts
      *

package/build/services/SecEdgarApi/FactsDownloader.js

@@ -65,7 +65,6 @@ var FactsDownloader = /** @class */ (function () {
             var _this = this;
             return __generator(this, function (_b) {
                 outputDirname = params.outputDirname, onChunk = params.onChunk, onDownloadComplete = params.onDownloadComplete, onError = params.onError, onComplete = params.onComplete, _a = params.unzip, unzip = _a === void 0 ? true : _a;
-                // writes download and unzip progress with percentage in terminal if useWriteProgressBar is true
                 return [2 /*return*/, new Promise(function (resolve, reject) { return __awaiter(_this, void 0, void 0, function () {
                         var filename, e_1;
                         return __generator(this, function (_a) {

package/build/services/SecEdgarApi/SecConnector.d.ts

@@ -1,6 +1,6 @@
 import { CompanyFactFrame, CompanyFactListData, MultiCompanyFactFrame } from '../../types';
 import { SubmissionList } from '../../types/submission.type';
-import { IClient } from './Client';
+import { IClient } from '../Client';
 import { IThrottler } from './Throttler';
 interface SecApiArgs {
     throttler: IThrottler;

package/build/services/SecEdgarApi/SecConnector.js

@@ -37,7 +37,7 @@ var __generator = (this && this.__generator) || function (thisArg, body) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 var cik_by_symbol_1 = require("../../util/cik-by-symbol");
-var Client_1 = require("./Client");
+var Client_1 = require("../Client");
 var Throttler_1 = require("./Throttler");
 /**
  * Gets reports from companies filed with the SEC

package/build/services/SecEdgarApi/SecEdgarApi.d.ts

@@ -2,12 +2,10 @@ import { CompanyFactFrame, CompanyFactListData, MultiCompanyFactFrame, ReportRaw
 import ReportParser from '../ReportParser';
 import { ParseReportsOptions } from '../ReportParser/ReportRawParser';
 import ReportWrapper from '../ReportParser/ReportWrapper';
-import { DownloadCompanyFactsDirectoryParams, IFactsDownloader } from './FactsDownloader';
 import { GetFactFrameParams, GetFactParams, GetSymbolParams, ISecConnector } from './SecConnector';
 interface SecEdgarApiArgs {
     secConnector: ISecConnector;
     reportParser: ReportParser;
-    factsDownloader: IFactsDownloader;
 }
 /**
  * Gets company reports and filings from the SEC EDGAR API
@@ -19,7 +17,6 @@ interface SecEdgarApiArgs {
 export default class SecEdgarApi {
     private readonly reportParser;
     private readonly secConnector;
-    private readonly factsDownloader;
     constructor(args?: SecEdgarApiArgs);
     /**
      * endpoint: /submissions/CIK${cik}.json
@@ -70,11 +67,14 @@ export default class SecEdgarApi {
      */
     getFactFrame(params: GetFactFrameParams): Promise<MultiCompanyFactFrame>;
     /**
+     * Note: Properties that are not provied from report are calculated an may not be accurate,
+     * verify results finance.yahoo.com (ex: https://finance.yahoo.com/quote/AAPL/financials)
+     *
+     * Please contribute to improve resolving report properties: https://github.com/andyevers/sec-edgar-api
+     *
      * Parses reports from company facts. Calculates missing properties and uses a single interface
      * for all reports. This includes only 10-K and 10-Q annual and quarterly reports. To include
-     * all reports, use getReportsRaw
-     *
-     * Note that calculated properties are estimated if they are not available in the company facts.
+     * all reports, use getReportsRaw.
      */
     getReports(params: GetSymbolParams): Promise<ReportWrapper[]>;
     /**
@@ -82,11 +82,5 @@ export default class SecEdgarApi {
      * for all reports.
      */
     getReportsRaw(params: GetSymbolParams & ParseReportsOptions): Promise<ReportRaw[]>;
-    /**
-     * Downloads the companyfacts.zip file and extracts the directory containing all company
-     * reports available from sec.gov. After downloading, you can use factFileReader and reportParser
-     * to get and read reports.
-     */
-    downloadCompanyFactsDirectory(params: DownloadCompanyFactsDirectoryParams): Promise<boolean>;
 }
 export {};

package/build/services/SecEdgarApi/SecEdgarApi.js

@@ -37,7 +37,6 @@ var __generator = (this && this.__generator) || function (thisArg, body) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 var ReportParser_1 = require("../ReportParser");
-var FactsDownloader_1 = require("./FactsDownloader");
 var SecConnector_1 = require("./SecConnector");
 /**
  * Gets company reports and filings from the SEC EDGAR API
@@ -51,12 +50,10 @@ var SecEdgarApi = /** @class */ (function () {
         if (args === void 0) { args = {
             secConnector: new SecConnector_1.default(),
             reportParser: new ReportParser_1.default(),
-            factsDownloader: new FactsDownloader_1.default(),
         }; }
-        var secConnector = args.secConnector, reportParser = args.reportParser, factsDownloader = args.factsDownloader;
+        var secConnector = args.secConnector, reportParser = args.reportParser;
         this.secConnector = secConnector;
         this.reportParser = reportParser;
-        this.factsDownloader = factsDownloader;
     }
     /**
      * endpoint: /submissions/CIK${cik}.json
@@ -131,11 +128,14 @@ var SecEdgarApi = /** @class */ (function () {
         });
     };
     /**
+     * Note: Properties that are not provied from report are calculated an may not be accurate,
+     * verify results finance.yahoo.com (ex: https://finance.yahoo.com/quote/AAPL/financials)
+     *
+     * Please contribute to improve resolving report properties: https://github.com/andyevers/sec-edgar-api
+     *
      * Parses reports from company facts. Calculates missing properties and uses a single interface
      * for all reports. This includes only 10-K and 10-Q annual and quarterly reports. To include
-     * all reports, use getReportsRaw
-     *
-     * Note that calculated properties are estimated if they are not available in the company facts.
+     * all reports, use getReportsRaw.
      */
     SecEdgarApi.prototype.getReports = function (params) {
         return __awaiter(this, void 0, void 0, function () {
@@ -167,18 +167,6 @@ var SecEdgarApi = /** @class */ (function () {
             });
         });
     };
-    /**
-     * Downloads the companyfacts.zip file and extracts the directory containing all company
-     * reports available from sec.gov. After downloading, you can use factFileReader and reportParser
-     * to get and read reports.
-     */
-    SecEdgarApi.prototype.downloadCompanyFactsDirectory = function (params) {
-        return __awaiter(this, void 0, void 0, function () {
-            return __generator(this, function (_a) {
-                return [2 /*return*/, this.factsDownloader.downloadCompanyFactsDirectory(params)];
-            });
-        });
-    };
     return SecEdgarApi;
 }());
 exports.default = SecEdgarApi;

package/build/services/SecEdgarApi/Throttler.d.ts

@@ -15,10 +15,10 @@ export interface IThrottler {
     add: (fn: () => Promise<any>) => void;
 }
 export default class Throttler implements IThrottler {
+    private readonly decrementTimeouts;
     private readonly queue;
     private readonly results;
     private readonly errors;
-    private countRunning;
     private maxConcurrent;
     private delayMs;
     onProgress?: (data: OnProgressData) => void;
@@ -26,7 +26,6 @@ export default class Throttler implements IThrottler {
     onError?: (err: any) => void;
     onEnd?: (results: any[], errors: any[]) => void;
     constructor(args?: ThrottlerArgs);
-    setMaxConcurrent(maxConcurrent: number): void;
     setDelayMs(delayMs: number): void;
     add(fn: () => Promise<any>): void;
     private run;

package/build/services/SecEdgarApi/Throttler.js

@@ -39,10 +39,10 @@ Object.defineProperty(exports, "__esModule", { value: true });
 var Throttler = /** @class */ (function () {
     function Throttler(args) {
         if (args === void 0) { args = {}; }
-        var _a = args.maxConcurrent, maxConcurrent = _a === void 0 ? 1 : _a, _b = args.delayMs, delayMs = _b === void 0 ? 120 : _b, onProgress = args.onProgress, onResult = args.onResult, onError = args.onError, onEnd = args.onEnd;
+        var _a = args.maxConcurrent, maxConcurrent = _a === void 0 ? 10 : _a, _b = args.delayMs, delayMs = _b === void 0 ? 1100 : _b, onProgress = args.onProgress, onResult = args.onResult, onError = args.onError, onEnd = args.onEnd;
         this.maxConcurrent = maxConcurrent;
         this.delayMs = delayMs;
-        this.countRunning = 0;
+        this.decrementTimeouts = new Set();
         this.queue = [];
         this.results = [];
         this.errors = [];
@@ -51,9 +51,6 @@ var Throttler = /** @class */ (function () {
         this.onError = onError;
         this.onEnd = onEnd;
     }
-    Throttler.prototype.setMaxConcurrent = function (maxConcurrent) {
-        this.maxConcurrent = maxConcurrent;
-    };
     Throttler.prototype.setDelayMs = function (delayMs) {
         this.delayMs = delayMs;
     };
@@ -64,22 +61,27 @@ var Throttler = /** @class */ (function () {
     Throttler.prototype.run = function () {
         var _a, _b, _c, _d;
         return __awaiter(this, void 0, void 0, function () {
-            var fn, result, err_1;
+            var countRunning, fn, decrementTimeout, result, err_1;
             var _this = this;
             return __generator(this, function (_e) {
                 switch (_e.label) {
                     case 0:
-                        if (this.countRunning >= this.maxConcurrent) {
+                        countRunning = this.decrementTimeouts.size;
+                        if (countRunning >= this.maxConcurrent) {
                             return [2 /*return*/];
                         }
                         if (this.queue.length === 0) {
-                            if (this.countRunning === 0) {
+                            if (countRunning === 0) {
                                 (_a = this.onEnd) === null || _a === void 0 ? void 0 : _a.call(this, this.results, this.errors);
                             }
                             return [2 /*return*/];
                         }
-                        this.countRunning++;
                         fn = this.queue.shift();
+                        decrementTimeout = setTimeout(function () {
+                            _this.decrementTimeouts.delete(decrementTimeout);
+                            _this.run();
+                        }, this.delayMs);
+                        this.decrementTimeouts.add(decrementTimeout);
                         _e.label = 1;
                     case 1:
                         _e.trys.push([1, 3, , 4]);
@@ -95,12 +97,10 @@ var Throttler = /** @class */ (function () {
                         (_c = this.onError) === null || _c === void 0 ? void 0 : _c.call(this, err_1);
                         return [3 /*break*/, 4];
                     case 4:
-                        this.countRunning--;
                         (_d = this.onProgress) === null || _d === void 0 ? void 0 : _d.call(this, {
                             queueLength: this.queue.length,
-                            countRunning: this.countRunning,
+                            countRunning: this.decrementTimeouts.size,
                         });
-                        setTimeout(function () { return _this.run(); }, this.delayMs);
                         return [2 /*return*/];
                 }
             });

package/package.json

@@ -1,21 +1,19 @@
 {
     "name": "sec-edgar-api",
-    "version": "0.0.2",
+    "version": "0.0.5",
     "description": "Fetch and parse SEC earnings reports and other filings. Useful for financial analysis.",
     "main": "build/index.js",
     "author": "Andrew Evers (https://github.com/andyevers)",
     "homepage": "https://github.com/andyevers/sec-edgar-api#readme",
     "license": "ISC",
-    "dependencies": {
-        "unzipper": "^0.10.14"
-    },
     "repository": {
         "type": "git",
         "url": "git+https://github.com/andyevers/sec-edgar-api.git"
     },
     "scripts": {
         "test": "jest test",
-        "build": "tsc"
+        "build": "tsc",
+        "publish": "npm run build && npm publish"
     },
     "keywords": [
         "sec",
@@ -27,6 +25,9 @@
         "earnings",
         "reports"
     ],
+    "dependencies": {
+        "unzipper": "^0.10.14"
+    },
     "devDependencies": {
         "@types/jest": "^29.5.2",
         "@types/unzipper": "^0.10.6",