together-ai 0.8.0 → 0.9.0

package/CHANGELOG.md

@@ -1,5 +1,17 @@
 # Changelog
 
+## 0.9.0 (2024-11-05)
+
+Full Changelog: [v0.8.0...v0.9.0](https://github.com/togethercomputer/together-typescript/compare/v0.8.0...v0.9.0)
+
+### Features
+
+* adding upload API as a helper function. Need to figure out how to make it part of Together.files package ([59efebe](https://github.com/togethercomputer/together-typescript/commit/59efebeea30bd389ecfc8dad484ce085e1efe412))
+* fixed linting error ([045f8dd](https://github.com/togethercomputer/together-typescript/commit/045f8dd9bdfd447d8663a5691096fea1e27adc5c))
+* removed ([d588157](https://github.com/togethercomputer/together-typescript/commit/d588157eba3f77dc68313b17aba3a3eefa2457ca))
+* updated yarn.lock ([6487474](https://github.com/togethercomputer/together-typescript/commit/6487474d24a1bc13416269bef0b36c11b424b9ef))
+* yarn lock change ([b8ce4ae](https://github.com/togethercomputer/together-typescript/commit/b8ce4ae8312789037122580f5408a72f21189d24))
+
 ## 0.8.0 (2024-10-30)
 
 Full Changelog: [v0.7.0...v0.8.0](https://github.com/togethercomputer/together-typescript/compare/v0.7.0...v0.8.0)

package/lib/upload.d.ts

@@ -0,0 +1,22 @@
+export interface FileResponse {
+    id: string;
+    object: string;
+    type: 'jsonl' | 'parquet';
+    purpose: 'fine-tune';
+    filename: string;
+    bytes: number;
+    line_count: number;
+    processed: boolean;
+}
+export interface ErrorResponse {
+    message: string;
+}
+export interface CheckFileResponse {
+    success: boolean;
+    message?: string;
+}
+export declare function check_file(fileName: string): Promise<CheckFileResponse>;
+export declare function check_parquet(fileName: string): Promise<string | undefined>;
+export declare function check_jsonl(fileName: string): Promise<string | undefined>;
+export declare function upload(fileName: string, check?: boolean): Promise<FileResponse | ErrorResponse>;
+//# sourceMappingURL=upload.d.ts.map

package/lib/upload.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"upload.d.ts","sourceRoot":"","sources":["../src/lib/upload.ts"],"names":[],"mappings":"AAYA,MAAM,WAAW,YAAY;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,OAAO,GAAG,SAAS,CAAC;IAC1B,OAAO,EAAE,WAAW,CAAC;IACrB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,OAAO,CAAC;CACpB;AAED,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,MAAM,CAAC;CACjB;AAYD,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AACD,wBAAsB,UAAU,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAiC7E;AAED,wBAAsB,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CA8BjF;AAGD,wBAAsB,WAAW,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CAkD/E;AAED,wBAAsB,MAAM,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,GAAE,OAAc,GAAG,OAAO,CAAC,YAAY,GAAG,aAAa,CAAC,CAsH3G"}

package/lib/upload.js

@@ -0,0 +1,271 @@
+"use strict";
+// Upload file to server using /files API
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.upload = exports.check_jsonl = exports.check_parquet = exports.check_file = void 0;
+const core = __importStar(require("../core.js"));
+const axios_1 = require("axios");
+const fs_1 = __importDefault(require("fs"));
+const node_fetch_1 = __importDefault(require("node-fetch"));
+const path = __importStar(require("path"));
+const progress_stream_1 = __importDefault(require("progress-stream"));
+const readline_1 = __importDefault(require("readline"));
+const parquetjs_1 = __importDefault(require("parquetjs"));
+const { ParquetReader } = parquetjs_1.default;
+const failedUploadMessage = {
+    message: 'failed to upload file',
+};
+const baseURL = 'https://api.together.xyz/v1';
+const MAX_FILE_SIZE = 4.8; // GB
+const BYTES_PER_GB = 1024 * 1024 * 1024;
+const MIN_SAMPLES = 1;
+const PARQUET_EXPECTED_COLUMNS = ['input_ids', 'attention_mask', 'labels'];
+async function check_file(fileName) {
+    const stat = fs_1.default.statSync(fileName);
+    if (stat.size == 0) {
+        return { success: false, message: `File is empty` };
+    }
+    if (stat.size > MAX_FILE_SIZE * BYTES_PER_GB) {
+        return { success: false, message: `File size exceeds the limit of ${MAX_FILE_SIZE} GB` };
+    }
+    const fileType = path.extname(fileName);
+    if (fileType !== '.jsonl' && fileType !== '.parquet') {
+        return {
+            success: false,
+            message: 'File type must be either .jsonl or .parquet',
+        };
+    }
+    if (fileType == '.jsonl') {
+        const jsonlCheck = await check_jsonl(fileName);
+        if (jsonlCheck) {
+            return { success: false, message: jsonlCheck };
+        }
+    }
+    if (fileType == '.parquet') {
+        const parquetCheck = await check_parquet(fileName);
+        if (parquetCheck) {
+            return { success: false, message: parquetCheck };
+        }
+    }
+    return { success: true };
+}
+exports.check_file = check_file;
+async function check_parquet(fileName) {
+    try {
+        const reader = await ParquetReader.openFile(fileName);
+        const cursor = reader.getCursor();
+        let record = null;
+        const fieldNames = Object.keys(reader.schema.fields);
+        if (!('input_ids' in fieldNames)) {
+            return `Parquet file ${fileName} does not contain the 'input_ids' column.`;
+        }
+        for (const fieldName in fieldNames) {
+            if (!PARQUET_EXPECTED_COLUMNS.includes(fieldName)) {
+                return `Parquet file ${fileName} contains unexpected column ${fieldName}. Only ${PARQUET_EXPECTED_COLUMNS.join(', ')} are supported`;
+            }
+        }
+        const numRows = reader.getRowCount();
+        if (numRows < MIN_SAMPLES) {
+            return `Parquet file ${fileName} contains only ${numRows} samples. Minimum of ${MIN_SAMPLES} samples are required`;
+        }
+        await reader.close();
+    }
+    catch (err) {
+        return `failed to read parquet file ${fileName}`;
+    }
+    return undefined;
+}
+exports.check_parquet = check_parquet;
+// return undefined if the file is valid, otherwise return an error message
+async function check_jsonl(fileName) {
+    const fileStream = fs_1.default.createReadStream(fileName);
+    const rl = readline_1.default.createInterface({
+        input: fileStream,
+        crlfDelay: Infinity,
+    });
+    let errors = [];
+    let lineNumber = 1;
+    for await (const line of rl) {
+        try {
+            // do not proceed if there are too many errors
+            if (errors.length > 20) {
+                break;
+            }
+            const parsedLine = JSON.parse(line);
+            if (typeof parsedLine !== 'object') {
+                errors.push(`Line number ${lineNumber} is not a valid JSON object`);
+                continue;
+            }
+            if (!('text' in parsedLine)) {
+                errors.push(`Missing 'text' field was found on line ${lineNumber} of the the input file. Expected format: {'text': 'my sample string'}.`);
+                continue;
+            }
+            if (typeof parsedLine['text'] !== 'string') {
+                errors.push(`'Invalid value type for "text" key on line ${lineNumber}. Expected string`);
+                continue;
+            }
+        }
+        catch (error) {
+            errors.push(`Error parsing line number ${lineNumber}`);
+        }
+        lineNumber += 1;
+    }
+    lineNumber -= 1;
+    if (lineNumber < MIN_SAMPLES) {
+        errors.push(`Processing ${fileName} resulted in only ${lineNumber - 1} samples.`);
+    }
+    if (errors.length > 0) {
+        return errors.join('\n');
+    }
+    return undefined;
+}
+exports.check_jsonl = check_jsonl;
+async function upload(fileName, check = true) {
+    let purpose = 'fine-tune';
+    if (!fs_1.default.existsSync(fileName)) {
+        return {
+            message: 'File does not exists',
+        };
+    }
+    const fileType = path.extname(fileName);
+    if (fileType !== '.jsonl' && fileType !== '.parquet') {
+        return {
+            message: 'File type must be either .jsonl or .parquet',
+        };
+    }
+    if (check) {
+        const checkFile = await check_file(fileName);
+        if (!checkFile.success) {
+            return {
+                message: checkFile.message || `verification of ${fileName} failed with some unknown reason`,
+            };
+        }
+    }
+    // steps to do
+    // 1. check if file exists
+    // 2. get signed upload url
+    // 3. upload file
+    const baseUrl = core.readEnv('TOGETHER_API_BASE_URL') || 'https://api.together.ai/v1';
+    const apiKey = core.readEnv('TOGETHER_API_KEY');
+    if (!apiKey) {
+        return {
+            message: 'API key is required',
+        };
+    }
+    const getSigned = baseURL + '/files';
+    try {
+        const params = new URLSearchParams({
+            file_name: fileName,
+            purpose: purpose,
+        });
+        const fullUrl = `${getSigned}?${params}`;
+        const r = await (0, node_fetch_1.default)(fullUrl, {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/x-www-form-urlencoded',
+                Authorization: `Bearer ${apiKey}`,
+            },
+            redirect: 'manual',
+            body: params.toString(),
+        });
+        if (r.status !== 302) {
+            return failedUploadMessage;
+        }
+        const uploadUrl = r.headers.get('location') || '';
+        if (!uploadUrl || uploadUrl === '') {
+            return failedUploadMessage;
+        }
+        const fileId = r.headers.get('x-together-file-id') || '';
+        if (!fileId || fileId === '') {
+            return failedUploadMessage;
+        }
+        const fileStream = fs_1.default.createReadStream(fileName);
+        const fileSize = fs_1.default.statSync(fileName).size;
+        const progressStream = (0, progress_stream_1.default)({
+            length: fileSize,
+            time: 100, // Emit progress events every 100ms
+        });
+        // Listen to progress events and log them
+        progressStream.on('progress', (progress) => {
+            displayProgress(progress.percentage);
+        });
+        let uploadedBytes = 0;
+        // upload the file to uploadUrl
+        const uploadResponse = await (0, node_fetch_1.default)(uploadUrl, {
+            method: 'PUT',
+            headers: {
+                'Content-Type': 'application/octet-stream',
+            },
+            body: fileStream.pipe(progressStream),
+        });
+        displayProgress(100);
+        process.stdout.write('\n');
+        return {
+            id: fileId,
+            object: 'file',
+            type: 'jsonl',
+            purpose: 'fine-tune',
+            filename: fileName,
+            bytes: fileSize,
+            line_count: 0,
+            processed: true,
+        };
+    }
+    catch (error) {
+        if ((0, axios_1.isAxiosError)(error)) {
+            // handle axios error here
+            if (error.status) {
+                return {
+                    message: `failed to upload file with status ${error.status}`,
+                };
+            }
+        }
+        return {
+            message: 'failed to upload file',
+        };
+    }
+}
+exports.upload = upload;
+async function displayProgress(progress) {
+    const barWidth = 40; // Number of characters for the progress bar
+    const completedBars = Math.round((progress / 100) * barWidth);
+    let remainingBars = barWidth - completedBars;
+    if (remainingBars < 0) {
+        remainingBars = 0;
+    }
+    const progressBar = `[${'='.repeat(completedBars)}${' '.repeat(remainingBars)}] ${progress.toFixed(2)}%`;
+    // Clear the line and write progress
+    //process.stdout.clearLine(0); //clean entire line
+    process.stdout.cursorTo(0);
+    process.stdout.write(progressBar, () => { });
+    await sleep(2000);
+}
+async function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+//# sourceMappingURL=upload.js.map

package/lib/upload.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"upload.js","sourceRoot":"","sources":["../src/lib/upload.ts"],"names":[],"mappings":";AAAA,yCAAyC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEzC,iDAAgC;AAChC,iCAAqC;AACrC,4CAAoB;AACpB,4DAA+B;AAC/B,2CAA6B;AAC7B,sEAAuC;AACvC,wDAAgC;AAChC,0DAA4B;AAC5B,MAAM,EAAE,aAAa,EAAE,GAAG,mBAAG,CAAC;AAiB9B,MAAM,mBAAmB,GAAG;IAC1B,OAAO,EAAE,uBAAuB;CACjC,CAAC;AAEF,MAAM,OAAO,GAAG,6BAA6B,CAAC;AAC9C,MAAM,aAAa,GAAG,GAAG,CAAC,CAAC,KAAK;AAChC,MAAM,YAAY,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI,CAAC;AACxC,MAAM,WAAW,GAAG,CAAC,CAAC;AACtB,MAAM,wBAAwB,GAAG,CAAC,WAAW,EAAE,gBAAgB,EAAE,QAAQ,CAAC,CAAC;AAMpE,KAAK,UAAU,UAAU,CAAC,QAAgB;IAC/C,MAAM,IAAI,GAAG,YAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACnC,IAAI,IAAI,CAAC,IAAI,IAAI,CAAC,EAAE;QAClB,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,eAAe,EAAE,CAAC;KACrD;IAED,IAAI,IAAI,CAAC,IAAI,GAAG,aAAa,GAAG,YAAY,EAAE;QAC5C,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,kCAAkC,aAAa,KAAK,EAAE,CAAC;KAC1F;IAED,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IACxC,IAAI,QAAQ,KAAK,QAAQ,IAAI,QAAQ,KAAK,UAAU,EAAE;QACpD,OAAO;YACL,OAAO,EAAE,KAAK;YACd,OAAO,EAAE,6CAA6C;SACvD,CAAC;KACH;IAED,IAAI,QAAQ,IAAI,QAAQ,EAAE;QACxB,MAAM,UAAU,GAAG,MAAM,WAAW,CAAC,QAAQ,CAAC,CAAC;QAC/C,IAAI,UAAU,EAAE;YACd,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC;SAChD;KACF;IAED,IAAI,QAAQ,IAAI,UAAU,EAAE;QAC1B,MAAM,YAAY,GAAG,MAAM,aAAa,CAAC,QAAQ,CAAC,CAAC;QACnD,IAAI,YAAY,EAAE;YAChB,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,YAAY,EAAE,CAAC;SAClD;KACF;IAED,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;AAC3B,CAAC;AAjCD,gCAiCC;AAEM,KAAK,UAAU,aAAa,CAAC,QAAgB;IAClD,IAAI;QACF,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QACtD,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;QAClC,IAAI,MAAM,GAAG,IAAI,CAAC;QAElB,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACrD,IAAI,CAAC,CAAC,WAAW,IAAI,UAAU,CAAC,EAAE;YAChC,OAAO,gBAAgB,QAAQ,2CAA2C,CAAC;SAC5E;QAED,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE;YAClC,IAAI,CAAC,wBAAwB,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE;gBACjD,OAAO,gBAAgB,QAAQ,+BAA+B,SAAS,UAAU,wBAAwB,CAAC,IAAI,CAC5G,IAAI,CACL,gBAAgB,CAAC;aACnB;SACF;QAED,MAAM,OAAO,GAAG,MAAM,CAAC,WAAW,EAAuB,CAAC;QAC1D,IAAI,OAAO,GAAG,WAAW,EAAE;YACzB,OAAO,gBAAgB,QAAQ,kBAAkB,OAAO,wBAAwB,WAAW,uBAAuB,CAAC;SACpH;QAED,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;KACtB;IAAC,OAAO,GAAG,EAAE;QACZ,OAAO,+BAA+B,QAAQ,EAAE,CAAC;KAClD;IAED,OAAO,SAAS,CAAC;AACnB,CAAC;AA9BD,sCA8BC;AAED,2EAA2E;AACpE,KAAK,UAAU,WAAW,CAAC,QAAgB;IAChD,MAAM,UAAU,GAAG,YAAE,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC;IAEjD,MAAM,EAAE,GAAG,kBAAQ,CAAC,eAAe,CAAC;QAClC,KAAK,EAAE,UAAU;QACjB,SAAS,EAAE,QAAQ;KACpB,CAAC,CAAC;IAEH,IAAI,MAAM,GAAa,EAAE,CAAC;IAC1B,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,EAAE,EAAE;QAC3B,IAAI;YACF,8CAA8C;YAC9C,IAAI,MAAM,CAAC,MAAM,GAAG,EAAE,EAAE;gBACtB,MAAM;aACP;YACD,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAEpC,IAAI,OAAO,UAAU,KAAK,QAAQ,EAAE;gBAClC,MAAM,CAAC,IAAI,CAAC,eAAe,UAAU,6BAA6B,CAAC,CAAC;gBACpE,SAAS;aACV;YAED,IAAI,CAAC,CAAC,MAAM,IAAI,UAAU,CAAC,EAAE;gBAC3B,MAAM,CAAC,IAAI,CACT,0CAA0C,UAAU,wEAAwE,CAC7H,CAAC;gBACF,SAAS;aACV;YAED,IAAI,OAAO,UAAU,CAAC,MAAM,CAAC,KAAK,QAAQ,EAAE;gBAC1C,MAAM,CAAC,IAAI,CAAC,8CAA8C,UAAU,mBAAmB,CAAC,CAAC;gBACzF,SAAS;aACV;SACF;QAAC,OAAO,KAAK,EAAE;YACd,MAAM,CAAC,IAAI,CAAC,6BAA6B,UAAU,EAAE,CAAC,CAAC;SACxD;QACD,UAAU,IAAI,CAAC,CAAC;KACjB;IAED,UAAU,IAAI,CAAC,CAAC;IAChB,IAAI,UAAU,GAAG,WAAW,EAAE;QAC5B,MAAM,CAAC,IAAI,CAAC,cAAc,QAAQ,qBAAqB,UAAU,GAAG,CAAC,WAAW,CAAC,CAAC;KACnF;IAED,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE;QACrB,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;KAC1B;IAED,OAAO,SAAS,CAAC;AACnB,CAAC;AAlDD,kCAkDC;AAEM,KAAK,UAAU,MAAM,CAAC,QAAgB,EAAE,QAAiB,IAAI;IAClE,IAAI,OAAO,GAAG,WAAW,CAAC;IAC1B,IAAI,CAAC,YAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE;QAC5B,OAAO;YACL,OAAO,EAAE,sBAAsB;SAChC,CAAC;KACH;IAED,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IACxC,IAAI,QAAQ,KAAK,QAAQ,IAAI,QAAQ,KAAK,UAAU,EAAE;QACpD,OAAO;YACL,OAAO,EAAE,6CAA6C;SACvD,CAAC;KACH;IAED,IAAI,KAAK,EAAE;QACT,MAAM,SAAS,GAAG,MAAM,UAAU,CAAC,QAAQ,CAAC,CAAC;QAC7C,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE;YACtB,OAAO;gBACL,OAAO,EAAE,SAAS,CAAC,OAAO,IAAI,mBAAmB,QAAQ,kCAAkC;aAC5F,CAAC;SACH;KACF;IAED,cAAc;IACd,0BAA0B;IAC1B,2BAA2B;IAC3B,iBAAiB;IACjB,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,uBAAuB,CAAC,IAAI,4BAA4B,CAAC;IACtF,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC;IAEhD,IAAI,CAAC,MAAM,EAAE;QACX,OAAO;YACL,OAAO,EAAE,qBAAqB;SAC/B,CAAC;KACH;IAED,MAAM,SAAS,GAAG,OAAO,GAAG,QAAQ,CAAC;IAErC,IAAI;QACF,MAAM,MAAM,GAAG,IAAI,eAAe,CAAC;YACjC,SAAS,EAAE,QAAQ;YACnB,OAAO,EAAE,OAAO;SACjB,CAAC,CAAC;QACH,MAAM,OAAO,GAAG,GAAG,SAAS,IAAI,MAAM,EAAE,CAAC;QACzC,MAAM,CAAC,GAAG,MAAM,IAAA,oBAAK,EAAC,OAAO,EAAE;YAC7B,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,cAAc,EAAE,mCAAmC;gBACnD,aAAa,EAAE,UAAU,MAAM,EAAE;aAClC;YACD,QAAQ,EAAE,QAAQ;YAClB,IAAI,EAAE,MAAM,CAAC,QAAQ,EAAE;SACxB,CAAC,CAAC;QAEH,IAAI,CAAC,CAAC,MAAM,KAAK,GAAG,EAAE;YACpB,OAAO,mBAAmB,CAAC;SAC5B;QAED,MAAM,SAAS,GAAG,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;QAClD,IAAI,CAAC,SAAS,IAAI,SAAS,KAAK,EAAE,EAAE;YAClC,OAAO,mBAAmB,CAAC;SAC5B;QACD,MAAM,MAAM,GAAG,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,IAAI,EAAE,CAAC;QACzD,IAAI,CAAC,MAAM,IAAI,MAAM,KAAK,EAAE,EAAE;YAC5B,OAAO,mBAAmB,CAAC;SAC5B;QAED,MAAM,UAAU,GAAG,YAAE,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC;QACjD,MAAM,QAAQ,GAAG,YAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC;QAE5C,MAAM,cAAc,GAAG,IAAA,yBAAQ,EAAC;YAC9B,MAAM,EAAE,QAAQ;YAChB,IAAI,EAAE,GAAG,EAAE,mCAAmC;SAC/C,CAAC,CAAC;QAEH,yCAAyC;QACzC,cAAc,CAAC,EAAE,CAAC,UAAU,EAAE,CAAC,QAAQ,EAAE,EAAE;YACzC,eAAe,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;QACvC,CAAC,CAAC,CAAC;QAEH,IAAI,aAAa,GAAG,CAAC,CAAC;QACtB,+BAA+B;QAC/B,MAAM,cAAc,GAAG,MAAM,IAAA,oBAAK,EAAC,SAAS,EAAE;YAC5C,MAAM,EAAE,KAAK;YACb,OAAO,EAAE;gBACP,cAAc,EAAE,0BAA0B;aAC3C;YACD,IAAI,EAAE,UAAU,CAAC,IAAI,CAAC,cAAc,CAAC;SACtC,CAAC,CAAC;QAEH,eAAe,CAAC,GAAG,CAAC,CAAC;QACrB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAE3B,OAAO;YACL,EAAE,EAAE,MAAM;YACV,MAAM,EAAE,MAAM;YACd,IAAI,EAAE,OAAO;YACb,OAAO,EAAE,WAAW;YACpB,QAAQ,EAAE,QAAQ;YAClB,KAAK,EAAE,QAAQ;YACf,UAAU,EAAE,CAAC;YACb,SAAS,EAAE,IAAI;SAChB,CAAC;KACH;IAAC,OAAO,KAAK,EAAE;QACd,IAAI,IAAA,oBAAY,EAAC,KAAK,CAAC,EAAE;YACvB,0BAA0B;YAC1B,IAAI,KAAK,CAAC,MAAM,EAAE;gBAChB,OAAO;oBACL,OAAO,EAAE,qCAAqC,KAAK,CAAC,MAAM,EAAE;iBAC7D,CAAC;aACH;SACF;QAED,OAAO;YACL,OAAO,EAAE,uBAAuB;SACjC,CAAC;KACH;AACH,CAAC;AAtHD,wBAsHC;AAED,KAAK,UAAU,eAAe,CAAC,QAAgB;IAC7C,MAAM,QAAQ,GAAG,EAAE,CAAC,CAAC,4CAA4C;IACjE,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,GAAG,GAAG,CAAC,GAAG,QAAQ,CAAC,CAAC;IAC9D,IAAI,aAAa,GAAG,QAAQ,GAAG,aAAa,CAAC;IAC7C,IAAI,aAAa,GAAG,CAAC,EAAE;QACrB,aAAa,GAAG,CAAC,CAAC;KACnB;IACD,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,aAAa,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;IAEzG,oCAAoC;IACpC,kDAAkD;IAClD,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;IAC3B,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,WAAW,EAAE,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IAC5C,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC;AACpB,CAAC;AAED,KAAK,UAAU,KAAK,CAAC,EAAU;IAC7B,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAC3D,CAAC"}

package/lib/upload.mjs

@@ -0,0 +1,238 @@
+// Upload file to server using /files API
+import * as core from "../core.mjs";
+import { isAxiosError } from 'axios';
+import fs from 'fs';
+import fetch from 'node-fetch';
+import * as path from 'path';
+import progress from 'progress-stream';
+import readline from 'readline';
+import pkg from 'parquetjs';
+const { ParquetReader } = pkg;
+const failedUploadMessage = {
+    message: 'failed to upload file',
+};
+const baseURL = 'https://api.together.xyz/v1';
+const MAX_FILE_SIZE = 4.8; // GB
+const BYTES_PER_GB = 1024 * 1024 * 1024;
+const MIN_SAMPLES = 1;
+const PARQUET_EXPECTED_COLUMNS = ['input_ids', 'attention_mask', 'labels'];
+export async function check_file(fileName) {
+    const stat = fs.statSync(fileName);
+    if (stat.size == 0) {
+        return { success: false, message: `File is empty` };
+    }
+    if (stat.size > MAX_FILE_SIZE * BYTES_PER_GB) {
+        return { success: false, message: `File size exceeds the limit of ${MAX_FILE_SIZE} GB` };
+    }
+    const fileType = path.extname(fileName);
+    if (fileType !== '.jsonl' && fileType !== '.parquet') {
+        return {
+            success: false,
+            message: 'File type must be either .jsonl or .parquet',
+        };
+    }
+    if (fileType == '.jsonl') {
+        const jsonlCheck = await check_jsonl(fileName);
+        if (jsonlCheck) {
+            return { success: false, message: jsonlCheck };
+        }
+    }
+    if (fileType == '.parquet') {
+        const parquetCheck = await check_parquet(fileName);
+        if (parquetCheck) {
+            return { success: false, message: parquetCheck };
+        }
+    }
+    return { success: true };
+}
+export async function check_parquet(fileName) {
+    try {
+        const reader = await ParquetReader.openFile(fileName);
+        const cursor = reader.getCursor();
+        let record = null;
+        const fieldNames = Object.keys(reader.schema.fields);
+        if (!('input_ids' in fieldNames)) {
+            return `Parquet file ${fileName} does not contain the 'input_ids' column.`;
+        }
+        for (const fieldName in fieldNames) {
+            if (!PARQUET_EXPECTED_COLUMNS.includes(fieldName)) {
+                return `Parquet file ${fileName} contains unexpected column ${fieldName}. Only ${PARQUET_EXPECTED_COLUMNS.join(', ')} are supported`;
+            }
+        }
+        const numRows = reader.getRowCount();
+        if (numRows < MIN_SAMPLES) {
+            return `Parquet file ${fileName} contains only ${numRows} samples. Minimum of ${MIN_SAMPLES} samples are required`;
+        }
+        await reader.close();
+    }
+    catch (err) {
+        return `failed to read parquet file ${fileName}`;
+    }
+    return undefined;
+}
+// return undefined if the file is valid, otherwise return an error message
+export async function check_jsonl(fileName) {
+    const fileStream = fs.createReadStream(fileName);
+    const rl = readline.createInterface({
+        input: fileStream,
+        crlfDelay: Infinity,
+    });
+    let errors = [];
+    let lineNumber = 1;
+    for await (const line of rl) {
+        try {
+            // do not proceed if there are too many errors
+            if (errors.length > 20) {
+                break;
+            }
+            const parsedLine = JSON.parse(line);
+            if (typeof parsedLine !== 'object') {
+                errors.push(`Line number ${lineNumber} is not a valid JSON object`);
+                continue;
+            }
+            if (!('text' in parsedLine)) {
+                errors.push(`Missing 'text' field was found on line ${lineNumber} of the the input file. Expected format: {'text': 'my sample string'}.`);
+                continue;
+            }
+            if (typeof parsedLine['text'] !== 'string') {
+                errors.push(`'Invalid value type for "text" key on line ${lineNumber}. Expected string`);
+                continue;
+            }
+        }
+        catch (error) {
+            errors.push(`Error parsing line number ${lineNumber}`);
+        }
+        lineNumber += 1;
+    }
+    lineNumber -= 1;
+    if (lineNumber < MIN_SAMPLES) {
+        errors.push(`Processing ${fileName} resulted in only ${lineNumber - 1} samples.`);
+    }
+    if (errors.length > 0) {
+        return errors.join('\n');
+    }
+    return undefined;
+}
+export async function upload(fileName, check = true) {
+    let purpose = 'fine-tune';
+    if (!fs.existsSync(fileName)) {
+        return {
+            message: 'File does not exists',
+        };
+    }
+    const fileType = path.extname(fileName);
+    if (fileType !== '.jsonl' && fileType !== '.parquet') {
+        return {
+            message: 'File type must be either .jsonl or .parquet',
+        };
+    }
+    if (check) {
+        const checkFile = await check_file(fileName);
+        if (!checkFile.success) {
+            return {
+                message: checkFile.message || `verification of ${fileName} failed with some unknown reason`,
+            };
+        }
+    }
+    // steps to do
+    // 1. check if file exists
+    // 2. get signed upload url
+    // 3. upload file
+    const baseUrl = core.readEnv('TOGETHER_API_BASE_URL') || 'https://api.together.ai/v1';
+    const apiKey = core.readEnv('TOGETHER_API_KEY');
+    if (!apiKey) {
+        return {
+            message: 'API key is required',
+        };
+    }
+    const getSigned = baseURL + '/files';
+    try {
+        const params = new URLSearchParams({
+            file_name: fileName,
+            purpose: purpose,
+        });
+        const fullUrl = `${getSigned}?${params}`;
+        const r = await fetch(fullUrl, {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/x-www-form-urlencoded',
+                Authorization: `Bearer ${apiKey}`,
+            },
+            redirect: 'manual',
+            body: params.toString(),
+        });
+        if (r.status !== 302) {
+            return failedUploadMessage;
+        }
+        const uploadUrl = r.headers.get('location') || '';
+        if (!uploadUrl || uploadUrl === '') {
+            return failedUploadMessage;
+        }
+        const fileId = r.headers.get('x-together-file-id') || '';
+        if (!fileId || fileId === '') {
+            return failedUploadMessage;
+        }
+        const fileStream = fs.createReadStream(fileName);
+        const fileSize = fs.statSync(fileName).size;
+        const progressStream = progress({
+            length: fileSize,
+            time: 100, // Emit progress events every 100ms
+        });
+        // Listen to progress events and log them
+        progressStream.on('progress', (progress) => {
+            displayProgress(progress.percentage);
+        });
+        let uploadedBytes = 0;
+        // upload the file to uploadUrl
+        const uploadResponse = await fetch(uploadUrl, {
+            method: 'PUT',
+            headers: {
+                'Content-Type': 'application/octet-stream',
+            },
+            body: fileStream.pipe(progressStream),
+        });
+        displayProgress(100);
+        process.stdout.write('\n');
+        return {
+            id: fileId,
+            object: 'file',
+            type: 'jsonl',
+            purpose: 'fine-tune',
+            filename: fileName,
+            bytes: fileSize,
+            line_count: 0,
+            processed: true,
+        };
+    }
+    catch (error) {
+        if (isAxiosError(error)) {
+            // handle axios error here
+            if (error.status) {
+                return {
+                    message: `failed to upload file with status ${error.status}`,
+                };
+            }
+        }
+        return {
+            message: 'failed to upload file',
+        };
+    }
+}
+async function displayProgress(progress) {
+    const barWidth = 40; // Number of characters for the progress bar
+    const completedBars = Math.round((progress / 100) * barWidth);
+    let remainingBars = barWidth - completedBars;
+    if (remainingBars < 0) {
+        remainingBars = 0;
+    }
+    const progressBar = `[${'='.repeat(completedBars)}${' '.repeat(remainingBars)}] ${progress.toFixed(2)}%`;
+    // Clear the line and write progress
+    //process.stdout.clearLine(0); //clean entire line
+    process.stdout.cursorTo(0);
+    process.stdout.write(progressBar, () => { });
+    await sleep(2000);
+}
+async function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+//# sourceMappingURL=upload.mjs.map

package/lib/upload.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"upload.mjs","sourceRoot":"","sources":["../src/lib/upload.ts"],"names":[],"mappings":"AAAA,yCAAyC;OAElC,KAAK,IAAI;OACT,EAAE,YAAY,EAAE,MAAM,OAAO;OAC7B,EAAE,MAAM,IAAI;OACZ,KAAK,MAAM,YAAY;OACvB,KAAK,IAAI,MAAM,MAAM;OACrB,QAAQ,MAAM,iBAAiB;OAC/B,QAAQ,MAAM,UAAU;OACxB,GAAG,MAAM,WAAW;AAC3B,MAAM,EAAE,aAAa,EAAE,GAAG,GAAG,CAAC;AAiB9B,MAAM,mBAAmB,GAAG;IAC1B,OAAO,EAAE,uBAAuB;CACjC,CAAC;AAEF,MAAM,OAAO,GAAG,6BAA6B,CAAC;AAC9C,MAAM,aAAa,GAAG,GAAG,CAAC,CAAC,KAAK;AAChC,MAAM,YAAY,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI,CAAC;AACxC,MAAM,WAAW,GAAG,CAAC,CAAC;AACtB,MAAM,wBAAwB,GAAG,CAAC,WAAW,EAAE,gBAAgB,EAAE,QAAQ,CAAC,CAAC;AAM3E,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,QAAgB;IAC/C,MAAM,IAAI,GAAG,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACnC,IAAI,IAAI,CAAC,IAAI,IAAI,CAAC,EAAE;QAClB,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,eAAe,EAAE,CAAC;KACrD;IAED,IAAI,IAAI,CAAC,IAAI,GAAG,aAAa,GAAG,YAAY,EAAE;QAC5C,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,kCAAkC,aAAa,KAAK,EAAE,CAAC;KAC1F;IAED,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IACxC,IAAI,QAAQ,KAAK,QAAQ,IAAI,QAAQ,KAAK,UAAU,EAAE;QACpD,OAAO;YACL,OAAO,EAAE,KAAK;YACd,OAAO,EAAE,6CAA6C;SACvD,CAAC;KACH;IAED,IAAI,QAAQ,IAAI,QAAQ,EAAE;QACxB,MAAM,UAAU,GAAG,MAAM,WAAW,CAAC,QAAQ,CAAC,CAAC;QAC/C,IAAI,UAAU,EAAE;YACd,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC;SAChD;KACF;IAED,IAAI,QAAQ,IAAI,UAAU,EAAE;QAC1B,MAAM,YAAY,GAAG,MAAM,aAAa,CAAC,QAAQ,CAAC,CAAC;QACnD,IAAI,YAAY,EAAE;YAChB,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,YAAY,EAAE,CAAC;SAClD;KACF;IAED,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;AAC3B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,QAAgB;IAClD,IAAI;QACF,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QACtD,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;QAClC,IAAI,MAAM,GAAG,IAAI,CAAC;QAElB,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACrD,IAAI,CAAC,CAAC,WAAW,IAAI,UAAU,CAAC,EAAE;YAChC,OAAO,gBAAgB,QAAQ,2CAA2C,CAAC;SAC5E;QAED,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE;YAClC,IAAI,CAAC,wBAAwB,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE;gBACjD,OAAO,gBAAgB,QAAQ,+BAA+B,SAAS,UAAU,wBAAwB,CAAC,IAAI,CAC5G,IAAI,CACL,gBAAgB,CAAC;aACnB;SACF;QAED,MAAM,OAAO,GAAG,MAAM,CAAC,WAAW,EAAuB,CAAC;QAC1D,IAAI,OAAO,GAAG,WAAW,EAAE;YACzB,OAAO,gBAAgB,QAAQ,kBAAkB,OAAO,wBAAwB,WAAW,uBAAuB,CAAC;SACpH;QAED,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;KACtB;IAAC,OAAO,GAAG,EAAE;QACZ,OAAO,+BAA+B,QAAQ,EAAE,CAAC;KAClD;IAED,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,2EAA2E;AAC3E,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,QAAgB;IAChD,MAAM,UAAU,GAAG,EAAE,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC;IAEjD,MAAM,EAAE,GAAG,QAAQ,CAAC,eAAe,CAAC;QAClC,KAAK,EAAE,UAAU;QACjB,SAAS,EAAE,QAAQ;KACpB,CAAC,CAAC;IAEH,IAAI,MAAM,GAAa,EAAE,CAAC;IAC1B,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,EAAE,EAAE;QAC3B,IAAI;YACF,8CAA8C;YAC9C,IAAI,MAAM,CAAC,MAAM,GAAG,EAAE,EAAE;gBACtB,MAAM;aACP;YACD,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAEpC,IAAI,OAAO,UAAU,KAAK,QAAQ,EAAE;gBAClC,MAAM,CAAC,IAAI,CAAC,eAAe,UAAU,6BAA6B,CAAC,CAAC;gBACpE,SAAS;aACV;YAED,IAAI,CAAC,CAAC,MAAM,IAAI,UAAU,CAAC,EAAE;gBAC3B,MAAM,CAAC,IAAI,CACT,0CAA0C,UAAU,wEAAwE,CAC7H,CAAC;gBACF,SAAS;aACV;YAED,IAAI,OAAO,UAAU,CAAC,MAAM,CAAC,KAAK,QAAQ,EAAE;gBAC1C,MAAM,CAAC,IAAI,CAAC,8CAA8C,UAAU,mBAAmB,CAAC,CAAC;gBACzF,SAAS;aACV;SACF;QAAC,OAAO,KAAK,EAAE;YACd,MAAM,CAAC,IAAI,CAAC,6BAA6B,UAAU,EAAE,CAAC,CAAC;SACxD;QACD,UAAU,IAAI,CAAC,CAAC;KACjB;IAED,UAAU,IAAI,CAAC,CAAC;IAChB,IAAI,UAAU,GAAG,WAAW,EAAE;QAC5B,MAAM,CAAC,IAAI,CAAC,cAAc,QAAQ,qBAAqB,UAAU,GAAG,CAAC,WAAW,CAAC,CAAC;KACnF;IAED,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE;QACrB,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;KAC1B;IAED,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,MAAM,CAAC,QAAgB,EAAE,QAAiB,IAAI;IAClE,IAAI,OAAO,GAAG,WAAW,CAAC;IAC1B,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE;QAC5B,OAAO;YACL,OAAO,EAAE,sBAAsB;SAChC,CAAC;KACH;IAED,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IACxC,IAAI,QAAQ,KAAK,QAAQ,IAAI,QAAQ,KAAK,UAAU,EAAE;QACpD,OAAO;YACL,OAAO,EAAE,6CAA6C;SACvD,CAAC;KACH;IAED,IAAI,KAAK,EAAE;QACT,MAAM,SAAS,GAAG,MAAM,UAAU,CAAC,QAAQ,CAAC,CAAC;QAC7C,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE;YACtB,OAAO;gBACL,OAAO,EAAE,SAAS,CAAC,OAAO,IAAI,mBAAmB,QAAQ,kCAAkC;aAC5F,CAAC;SACH;KACF;IAED,cAAc;IACd,0BAA0B;IAC1B,2BAA2B;IAC3B,iBAAiB;IACjB,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,uBAAuB,CAAC,IAAI,4BAA4B,CAAC;IACtF,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC;IAEhD,IAAI,CAAC,MAAM,EAAE;QACX,OAAO;YACL,OAAO,EAAE,qBAAqB;SAC/B,CAAC;KACH;IAED,MAAM,SAAS,GAAG,OAAO,GAAG,QAAQ,CAAC;IAErC,IAAI;QACF,MAAM,MAAM,GAAG,IAAI,eAAe,CAAC;YACjC,SAAS,EAAE,QAAQ;YACnB,OAAO,EAAE,OAAO;SACjB,CAAC,CAAC;QACH,MAAM,OAAO,GAAG,GAAG,SAAS,IAAI,MAAM,EAAE,CAAC;QACzC,MAAM,CAAC,GAAG,MAAM,KAAK,CAAC,OAAO,EAAE;YAC7B,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,cAAc,EAAE,mCAAmC;gBACnD,aAAa,EAAE,UAAU,MAAM,EAAE;aAClC;YACD,QAAQ,EAAE,QAAQ;YAClB,IAAI,EAAE,MAAM,CAAC,QAAQ,EAAE;SACxB,CAAC,CAAC;QAEH,IAAI,CAAC,CAAC,MAAM,KAAK,GAAG,EAAE;YACpB,OAAO,mBAAmB,CAAC;SAC5B;QAED,MAAM,SAAS,GAAG,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;QAClD,IAAI,CAAC,SAAS,IAAI,SAAS,KAAK,EAAE,EAAE;YAClC,OAAO,mBAAmB,CAAC;SAC5B;QACD,MAAM,MAAM,GAAG,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,IAAI,EAAE,CAAC;QACzD,IAAI,CAAC,MAAM,IAAI,MAAM,KAAK,EAAE,EAAE;YAC5B,OAAO,mBAAmB,CAAC;SAC5B;QAED,MAAM,UAAU,GAAG,EAAE,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC;QACjD,MAAM,QAAQ,GAAG,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC;QAE5C,MAAM,cAAc,GAAG,QAAQ,CAAC;YAC9B,MAAM,EAAE,QAAQ;YAChB,IAAI,EAAE,GAAG,EAAE,mCAAmC;SAC/C,CAAC,CAAC;QAEH,yCAAyC;QACzC,cAAc,CAAC,EAAE,CAAC,UAAU,EAAE,CAAC,QAAQ,EAAE,EAAE;YACzC,eAAe,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;QACvC,CAAC,CAAC,CAAC;QAEH,IAAI,aAAa,GAAG,CAAC,CAAC;QACtB,+BAA+B;QAC/B,MAAM,cAAc,GAAG,MAAM,KAAK,CAAC,SAAS,EAAE;YAC5C,MAAM,EAAE,KAAK;YACb,OAAO,EAAE;gBACP,cAAc,EAAE,0BAA0B;aAC3C;YACD,IAAI,EAAE,UAAU,CAAC,IAAI,CAAC,cAAc,CAAC;SACtC,CAAC,CAAC;QAEH,eAAe,CAAC,GAAG,CAAC,CAAC;QACrB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAE3B,OAAO;YACL,EAAE,EAAE,MAAM;YACV,MAAM,EAAE,MAAM;YACd,IAAI,EAAE,OAAO;YACb,OAAO,EAAE,WAAW;YACpB,QAAQ,EAAE,QAAQ;YAClB,KAAK,EAAE,QAAQ;YACf,UAAU,EAAE,CAAC;YACb,SAAS,EAAE,IAAI;SAChB,CAAC;KACH;IAAC,OAAO,KAAK,EAAE;QACd,IAAI,YAAY,CAAC,KAAK,CAAC,EAAE;YACvB,0BAA0B;YAC1B,IAAI,KAAK,CAAC,MAAM,EAAE;gBAChB,OAAO;oBACL,OAAO,EAAE,qCAAqC,KAAK,CAAC,MAAM,EAAE;iBAC7D,CAAC;aACH;SACF;QAED,OAAO;YACL,OAAO,EAAE,uBAAuB;SACjC,CAAC;KACH;AACH,CAAC;AAED,KAAK,UAAU,eAAe,CAAC,QAAgB;IAC7C,MAAM,QAAQ,GAAG,EAAE,CAAC,CAAC,4CAA4C;IACjE,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,GAAG,GAAG,CAAC,GAAG,QAAQ,CAAC,CAAC;IAC9D,IAAI,aAAa,GAAG,QAAQ,GAAG,aAAa,CAAC;IAC7C,IAAI,aAAa,GAAG,CAAC,EAAE;QACrB,aAAa,GAAG,CAAC,CAAC;KACnB;IACD,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,aAAa,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;IAEzG,oCAAoC;IACpC,kDAAkD;IAClD,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;IAC3B,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,WAAW,EAAE,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IAC5C,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC;AACpB,CAAC;AAED,KAAK,UAAU,KAAK,CAAC,EAAU;IAC7B,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAC3D,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "together-ai",
-  "version": "0.8.0",
+  "version": "0.9.0",
   "description": "The official TypeScript library for the Together API",
   "author": "Together <dev-feedback@TogetherAI.com>",
   "types": "./index.d.ts",
@@ -24,11 +24,16 @@
   "dependencies": {
     "@types/node": "^18.11.18",
     "@types/node-fetch": "^2.6.4",
+    "@types/parquetjs": "^0.10.6",
+    "@types/progress-stream": "^2.0.5",
     "abort-controller": "^3.0.0",
     "agentkeepalive": "^4.2.1",
+    "axios": "^1.7.7",
     "form-data-encoder": "1.7.2",
     "formdata-node": "^4.3.2",
-    "node-fetch": "^2.6.7"
+    "node-fetch": "^2.6.7",
+    "parquetjs": "^0.11.2",
+    "progress-stream": "^2.0.0"
   },
   "sideEffects": [
     "./_shims/index.js",

package/resources/files.d.ts

@@ -19,6 +19,10 @@ export declare class Files extends APIResource {
      * Get the contents of a single uploaded data file.
      */
     content(id: string, options?: Core.RequestOptions): Core.APIPromise<Response>;
+    /**
+     * Upload a file.
+     */
+    upload(_: string): Promise<void>;
 }
 export interface FileObject {
     id?: string;

package/resources/files.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"files.d.ts","sourceRoot":"","sources":["../src/resources/files.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,KAAK,IAAI,MAAM,SAAS,CAAC;AAChC,OAAO,KAAK,QAAQ,MAAM,SAAS,CAAC;AACpC,OAAO,EAAE,KAAK,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAEhD,qBAAa,KAAM,SAAQ,WAAW;IACpC;;OAEG;IACH,QAAQ,CAAC,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,UAAU,CAAC,oBAAoB,CAAC;IAI1F;;OAEG;IACH,IAAI,CAAC,OAAO,CAAC,EAAE,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,UAAU,CAAC,gBAAgB,CAAC;IAItE;;OAEG;IACH,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,UAAU,CAAC,kBAAkB,CAAC;IAItF;;OAEG;IACH,OAAO,CAAC,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC;CAG9E;AAED,MAAM,WAAW,UAAU;IACzB,EAAE,CAAC,EAAE,MAAM,CAAC;IAEZ,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,oBAAoB;IACnC,EAAE,EAAE,MAAM,CAAC;IAEX,KAAK,EAAE,MAAM,CAAC;IAEd,UAAU,EAAE,MAAM,CAAC;IAEnB,QAAQ,EAAE,MAAM,CAAC;IAEjB,QAAQ,EAAE,OAAO,GAAG,SAAS,CAAC;IAE9B,SAAS,EAAE,MAAM,CAAC;IAElB,MAAM,EAAE,MAAM,CAAC;IAEf,SAAS,EAAE,OAAO,CAAC;IAEnB,OAAO,EAAE,WAAW,CAAC;CACtB;AAED,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,KAAK,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC;CACpC;AAED,yBAAiB,gBAAgB,CAAC;IAChC,UAAiB,IAAI;QACnB,EAAE,EAAE,MAAM,CAAC;QAEX,KAAK,EAAE,MAAM,CAAC;QAEd,UAAU,EAAE,MAAM,CAAC;QAEnB,QAAQ,EAAE,MAAM,CAAC;QAEjB,QAAQ,EAAE,OAAO,GAAG,SAAS,CAAC;QAE9B,SAAS,EAAE,MAAM,CAAC;QAElB,MAAM,EAAE,MAAM,CAAC;QAEf,SAAS,EAAE,OAAO,CAAC;QAEnB,OAAO,EAAE,WAAW,CAAC;KACtB;CACF;AAED,MAAM,WAAW,kBAAkB;IACjC,EAAE,CAAC,EAAE,MAAM,CAAC;IAEZ,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAED,yBAAiB,KAAK,CAAC;IACrB,MAAM,QAAQ,UAAU,GAAG,QAAQ,CAAC,UAAU,CAAC;IAC/C,MAAM,QAAQ,oBAAoB,GAAG,QAAQ,CAAC,oBAAoB,CAAC;IACnE,MAAM,QAAQ,gBAAgB,GAAG,QAAQ,CAAC,gBAAgB,CAAC;IAC3D,MAAM,QAAQ,kBAAkB,GAAG,QAAQ,CAAC,kBAAkB,CAAC;CAChE"}
+{"version":3,"file":"files.d.ts","sourceRoot":"","sources":["../src/resources/files.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,KAAK,IAAI,MAAM,SAAS,CAAC;AAChC,OAAO,KAAK,QAAQ,MAAM,SAAS,CAAC;AACpC,OAAO,EAAE,KAAK,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAEhD,qBAAa,KAAM,SAAQ,WAAW;IACpC;;OAEG;IACH,QAAQ,CAAC,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,UAAU,CAAC,oBAAoB,CAAC;IAI1F;;OAEG;IACH,IAAI,CAAC,OAAO,CAAC,EAAE,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,UAAU,CAAC,gBAAgB,CAAC;IAItE;;OAEG;IACH,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,UAAU,CAAC,kBAAkB,CAAC;IAItF;;OAEG;IACH,OAAO,CAAC,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC;IAI7E;;OAEG;IACH,MAAM,CAAC,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;CAGjC;AAED,MAAM,WAAW,UAAU;IACzB,EAAE,CAAC,EAAE,MAAM,CAAC;IAEZ,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,oBAAoB;IACnC,EAAE,EAAE,MAAM,CAAC;IAEX,KAAK,EAAE,MAAM,CAAC;IAEd,UAAU,EAAE,MAAM,CAAC;IAEnB,QAAQ,EAAE,MAAM,CAAC;IAEjB,QAAQ,EAAE,OAAO,GAAG,SAAS,CAAC;IAE9B,SAAS,EAAE,MAAM,CAAC;IAElB,MAAM,EAAE,MAAM,CAAC;IAEf,SAAS,EAAE,OAAO,CAAC;IAEnB,OAAO,EAAE,WAAW,CAAC;CACtB;AAED,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,KAAK,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC;CACpC;AAED,yBAAiB,gBAAgB,CAAC;IAChC,UAAiB,IAAI;QACnB,EAAE,EAAE,MAAM,CAAC;QAEX,KAAK,EAAE,MAAM,CAAC;QAEd,UAAU,EAAE,MAAM,CAAC;QAEnB,QAAQ,EAAE,MAAM,CAAC;QAEjB,QAAQ,EAAE,OAAO,GAAG,SAAS,CAAC;QAE9B,SAAS,EAAE,MAAM,CAAC;QAElB,MAAM,EAAE,MAAM,CAAC;QAEf,SAAS,EAAE,OAAO,CAAC;QAEnB,OAAO,EAAE,WAAW,CAAC;KACtB;CACF;AAED,MAAM,WAAW,kBAAkB;IACjC,EAAE,CAAC,EAAE,MAAM,CAAC;IAEZ,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAED,yBAAiB,KAAK,CAAC;IACrB,MAAM,QAAQ,UAAU,GAAG,QAAQ,CAAC,UAAU,CAAC;IAC/C,MAAM,QAAQ,oBAAoB,GAAG,QAAQ,CAAC,oBAAoB,CAAC;IACnE,MAAM,QAAQ,gBAAgB,GAAG,QAAQ,CAAC,gBAAgB,CAAC;IAC3D,MAAM,QAAQ,kBAAkB,GAAG,QAAQ,CAAC,kBAAkB,CAAC;CAChE"}

package/resources/files.js

@@ -28,6 +28,12 @@ class Files extends resource_1.APIResource {
     content(id, options) {
         return this._client.get(`/files/${id}/content`, { ...options, __binaryResponse: true });
     }
+    /**
+     * Upload a file.
+     */
+    upload(_) {
+        throw 'please use together-ai/lib/upload';
+    }
 }
 exports.Files = Files;
 (function (Files) {

package/resources/files.js.map

@@ -1 +1 @@
-{"version":3,"file":"files.js","sourceRoot":"","sources":["../src/resources/files.ts"],"names":[],"mappings":";AAAA,sFAAsF;;;AAEtF,6CAA0C;AAK1C,MAAa,KAAM,SAAQ,sBAAW;IACpC;;OAEG;IACH,QAAQ,CAAC,EAAU,EAAE,OAA6B;QAChD,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,EAAE,EAAE,EAAE,OAAO,CAAC,CAAC;IACnD,CAAC;IAED;;OAEG;IACH,IAAI,CAAC,OAA6B;QAChC,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC7C,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,EAAU,EAAE,OAA6B;QAC9C,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,EAAE,EAAE,EAAE,OAAO,CAAC,CAAC;IACtD,CAAC;IAED;;OAEG;IACH,OAAO,CAAC,EAAU,EAAE,OAA6B;QAC/C,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,EAAE,UAAU,EAAE,EAAE,GAAG,OAAO,EAAE,gBAAgB,EAAE,IAAI,EAAE,CAAC,CAAC;IAC1F,CAAC;CACF;AA5BD,sBA4BC;AAgED,WAAiB,KAAK;AAKtB,CAAC,EALgB,KAAK,GAAL,aAAK,KAAL,aAAK,QAKrB"}
+{"version":3,"file":"files.js","sourceRoot":"","sources":["../src/resources/files.ts"],"names":[],"mappings":";AAAA,sFAAsF;;;AAEtF,6CAA0C;AAK1C,MAAa,KAAM,SAAQ,sBAAW;IACpC;;OAEG;IACH,QAAQ,CAAC,EAAU,EAAE,OAA6B;QAChD,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,EAAE,EAAE,EAAE,OAAO,CAAC,CAAC;IACnD,CAAC;IAED;;OAEG;IACH,IAAI,CAAC,OAA6B;QAChC,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC7C,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,EAAU,EAAE,OAA6B;QAC9C,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,EAAE,EAAE,EAAE,OAAO,CAAC,CAAC;IACtD,CAAC;IAED;;OAEG;IACH,OAAO,CAAC,EAAU,EAAE,OAA6B;QAC/C,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,EAAE,UAAU,EAAE,EAAE,GAAG,OAAO,EAAE,gBAAgB,EAAE,IAAI,EAAE,CAAC,CAAC;IAC1F,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,CAAS;QACd,MAAM,mCAAmC,CAAC;IAC5C,CAAC;CACF;AAnCD,sBAmCC;AAgED,WAAiB,KAAK;AAKtB,CAAC,EALgB,KAAK,GAAL,aAAK,KAAL,aAAK,QAKrB"}

package/resources/files.mjs

@@ -25,6 +25,12 @@ export class Files extends APIResource {
     content(id, options) {
         return this._client.get(`/files/${id}/content`, { ...options, __binaryResponse: true });
     }
+    /**
+     * Upload a file.
+     */
+    upload(_) {
+        throw 'please use together-ai/lib/upload';
+    }
 }
 (function (Files) {
 })(Files || (Files = {}));

package/resources/files.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"files.mjs","sourceRoot":"","sources":["../src/resources/files.ts"],"names":[],"mappings":"AAAA,sFAAsF;OAE/E,EAAE,WAAW,EAAE;AAKtB,MAAM,OAAO,KAAM,SAAQ,WAAW;IACpC;;OAEG;IACH,QAAQ,CAAC,EAAU,EAAE,OAA6B;QAChD,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,EAAE,EAAE,EAAE,OAAO,CAAC,CAAC;IACnD,CAAC;IAED;;OAEG;IACH,IAAI,CAAC,OAA6B;QAChC,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC7C,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,EAAU,EAAE,OAA6B;QAC9C,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,EAAE,EAAE,EAAE,OAAO,CAAC,CAAC;IACtD,CAAC;IAED;;OAEG;IACH,OAAO,CAAC,EAAU,EAAE,OAA6B;QAC/C,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,EAAE,UAAU,EAAE,EAAE,GAAG,OAAO,EAAE,gBAAgB,EAAE,IAAI,EAAE,CAAC,CAAC;IAC1F,CAAC;CACF;AAgED,WAAiB,KAAK;AAKtB,CAAC,EALgB,KAAK,KAAL,KAAK,QAKrB"}
+{"version":3,"file":"files.mjs","sourceRoot":"","sources":["../src/resources/files.ts"],"names":[],"mappings":"AAAA,sFAAsF;OAE/E,EAAE,WAAW,EAAE;AAKtB,MAAM,OAAO,KAAM,SAAQ,WAAW;IACpC;;OAEG;IACH,QAAQ,CAAC,EAAU,EAAE,OAA6B;QAChD,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,EAAE,EAAE,EAAE,OAAO,CAAC,CAAC;IACnD,CAAC;IAED;;OAEG;IACH,IAAI,CAAC,OAA6B;QAChC,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC7C,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,EAAU,EAAE,OAA6B;QAC9C,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,EAAE,EAAE,EAAE,OAAO,CAAC,CAAC;IACtD,CAAC;IAED;;OAEG;IACH,OAAO,CAAC,EAAU,EAAE,OAA6B;QAC/C,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,EAAE,UAAU,EAAE,EAAE,GAAG,OAAO,EAAE,gBAAgB,EAAE,IAAI,EAAE,CAAC,CAAC;IAC1F,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,CAAS;QACd,MAAM,mCAAmC,CAAC;IAC5C,CAAC;CACF;AAgED,WAAiB,KAAK;AAKtB,CAAC,EALgB,KAAK,KAAL,KAAK,QAKrB"}

package/src/lib/upload.ts

@@ -0,0 +1,300 @@
+// Upload file to server using /files API
+
+import * as core from '../core';
+import { isAxiosError } from 'axios';
+import fs from 'fs';
+import fetch from 'node-fetch';
+import * as path from 'path';
+import progress from 'progress-stream';
+import readline from 'readline';
+import pkg from 'parquetjs';
+const { ParquetReader } = pkg;
+
+export interface FileResponse {
+  id: string;
+  object: string;
+  type: 'jsonl' | 'parquet';
+  purpose: 'fine-tune';
+  filename: string;
+  bytes: number;
+  line_count: number;
+  processed: boolean;
+}
+
+export interface ErrorResponse {
+  message: string;
+}
+
+const failedUploadMessage = {
+  message: 'failed to upload file',
+};
+
+const baseURL = 'https://api.together.xyz/v1';
+const MAX_FILE_SIZE = 4.8; // GB
+const BYTES_PER_GB = 1024 * 1024 * 1024;
+const MIN_SAMPLES = 1;
+const PARQUET_EXPECTED_COLUMNS = ['input_ids', 'attention_mask', 'labels'];
+
+export interface CheckFileResponse {
+  success: boolean;
+  message?: string;
+}
+export async function check_file(fileName: string): Promise<CheckFileResponse> {
+  const stat = fs.statSync(fileName);
+  if (stat.size == 0) {
+    return { success: false, message: `File is empty` };
+  }
+
+  if (stat.size > MAX_FILE_SIZE * BYTES_PER_GB) {
+    return { success: false, message: `File size exceeds the limit of ${MAX_FILE_SIZE} GB` };
+  }
+
+  const fileType = path.extname(fileName);
+  if (fileType !== '.jsonl' && fileType !== '.parquet') {
+    return {
+      success: false,
+      message: 'File type must be either .jsonl or .parquet',
+    };
+  }
+
+  if (fileType == '.jsonl') {
+    const jsonlCheck = await check_jsonl(fileName);
+    if (jsonlCheck) {
+      return { success: false, message: jsonlCheck };
+    }
+  }
+
+  if (fileType == '.parquet') {
+    const parquetCheck = await check_parquet(fileName);
+    if (parquetCheck) {
+      return { success: false, message: parquetCheck };
+    }
+  }
+
+  return { success: true };
+}
+
+export async function check_parquet(fileName: string): Promise<string | undefined> {
+  try {
+    const reader = await ParquetReader.openFile(fileName);
+    const cursor = reader.getCursor();
+    let record = null;
+
+    const fieldNames = Object.keys(reader.schema.fields);
+    if (!('input_ids' in fieldNames)) {
+      return `Parquet file ${fileName} does not contain the 'input_ids' column.`;
+    }
+
+    for (const fieldName in fieldNames) {
+      if (!PARQUET_EXPECTED_COLUMNS.includes(fieldName)) {
+        return `Parquet file ${fileName} contains unexpected column ${fieldName}. Only ${PARQUET_EXPECTED_COLUMNS.join(
+          ', ',
+        )} are supported`;
+      }
+    }
+
+    const numRows = reader.getRowCount() as unknown as number;
+    if (numRows < MIN_SAMPLES) {
+      return `Parquet file ${fileName} contains only ${numRows} samples. Minimum of ${MIN_SAMPLES} samples are required`;
+    }
+
+    await reader.close();
+  } catch (err) {
+    return `failed to read parquet file ${fileName}`;
+  }
+
+  return undefined;
+}
+
+// return undefined if the file is valid, otherwise return an error message
+export async function check_jsonl(fileName: string): Promise<string | undefined> {
+  const fileStream = fs.createReadStream(fileName);
+
+  const rl = readline.createInterface({
+    input: fileStream,
+    crlfDelay: Infinity,
+  });
+
+  let errors: string[] = [];
+  let lineNumber = 1;
+  for await (const line of rl) {
+    try {
+      // do not proceed if there are too many errors
+      if (errors.length > 20) {
+        break;
+      }
+      const parsedLine = JSON.parse(line);
+
+      if (typeof parsedLine !== 'object') {
+        errors.push(`Line number ${lineNumber} is not a valid JSON object`);
+        continue;
+      }
+
+      if (!('text' in parsedLine)) {
+        errors.push(
+          `Missing 'text' field was found on line ${lineNumber} of the the input file. Expected format: {'text': 'my sample string'}.`,
+        );
+        continue;
+      }
+
+      if (typeof parsedLine['text'] !== 'string') {
+        errors.push(`'Invalid value type for "text" key on line ${lineNumber}. Expected string`);
+        continue;
+      }
+    } catch (error) {
+      errors.push(`Error parsing line number ${lineNumber}`);
+    }
+    lineNumber += 1;
+  }
+
+  lineNumber -= 1;
+  if (lineNumber < MIN_SAMPLES) {
+    errors.push(`Processing ${fileName} resulted in only ${lineNumber - 1} samples.`);
+  }
+
+  if (errors.length > 0) {
+    return errors.join('\n');
+  }
+
+  return undefined;
+}
+
+export async function upload(fileName: string, check: boolean = true): Promise<FileResponse | ErrorResponse> {
+  let purpose = 'fine-tune';
+  if (!fs.existsSync(fileName)) {
+    return {
+      message: 'File does not exists',
+    };
+  }
+
+  const fileType = path.extname(fileName);
+  if (fileType !== '.jsonl' && fileType !== '.parquet') {
+    return {
+      message: 'File type must be either .jsonl or .parquet',
+    };
+  }
+
+  if (check) {
+    const checkFile = await check_file(fileName);
+    if (!checkFile.success) {
+      return {
+        message: checkFile.message || `verification of ${fileName} failed with some unknown reason`,
+      };
+    }
+  }
+
+  // steps to do
+  // 1. check if file exists
+  // 2. get signed upload url
+  // 3. upload file
+  const baseUrl = core.readEnv('TOGETHER_API_BASE_URL') || 'https://api.together.ai/v1';
+  const apiKey = core.readEnv('TOGETHER_API_KEY');
+
+  if (!apiKey) {
+    return {
+      message: 'API key is required',
+    };
+  }
+
+  const getSigned = baseURL + '/files';
+
+  try {
+    const params = new URLSearchParams({
+      file_name: fileName,
+      purpose: purpose,
+    });
+    const fullUrl = `${getSigned}?${params}`;
+    const r = await fetch(fullUrl, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/x-www-form-urlencoded',
+        Authorization: `Bearer ${apiKey}`,
+      },
+      redirect: 'manual',
+      body: params.toString(),
+    });
+
+    if (r.status !== 302) {
+      return failedUploadMessage;
+    }
+
+    const uploadUrl = r.headers.get('location') || '';
+    if (!uploadUrl || uploadUrl === '') {
+      return failedUploadMessage;
+    }
+    const fileId = r.headers.get('x-together-file-id') || '';
+    if (!fileId || fileId === '') {
+      return failedUploadMessage;
+    }
+
+    const fileStream = fs.createReadStream(fileName);
+    const fileSize = fs.statSync(fileName).size;
+
+    const progressStream = progress({
+      length: fileSize,
+      time: 100, // Emit progress events every 100ms
+    });
+
+    // Listen to progress events and log them
+    progressStream.on('progress', (progress) => {
+      displayProgress(progress.percentage);
+    });
+
+    let uploadedBytes = 0;
+    // upload the file to uploadUrl
+    const uploadResponse = await fetch(uploadUrl, {
+      method: 'PUT',
+      headers: {
+        'Content-Type': 'application/octet-stream',
+      },
+      body: fileStream.pipe(progressStream),
+    });
+
+    displayProgress(100);
+    process.stdout.write('\n');
+
+    return {
+      id: fileId,
+      object: 'file',
+      type: 'jsonl',
+      purpose: 'fine-tune',
+      filename: fileName,
+      bytes: fileSize,
+      line_count: 0,
+      processed: true,
+    };
+  } catch (error) {
+    if (isAxiosError(error)) {
+      // handle axios error here
+      if (error.status) {
+        return {
+          message: `failed to upload file with status ${error.status}`,
+        };
+      }
+    }
+
+    return {
+      message: 'failed to upload file',
+    };
+  }
+}
+
+async function displayProgress(progress: number) {
+  const barWidth = 40; // Number of characters for the progress bar
+  const completedBars = Math.round((progress / 100) * barWidth);
+  let remainingBars = barWidth - completedBars;
+  if (remainingBars < 0) {
+    remainingBars = 0;
+  }
+  const progressBar = `[${'='.repeat(completedBars)}${' '.repeat(remainingBars)}] ${progress.toFixed(2)}%`;
+
+  // Clear the line and write progress
+  //process.stdout.clearLine(0); //clean entire line
+  process.stdout.cursorTo(0);
+  process.stdout.write(progressBar, () => {});
+  await sleep(2000);
+}
+
+async function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}

package/src/resources/files.ts

@@ -33,6 +33,13 @@ export class Files extends APIResource {
   content(id: string, options?: Core.RequestOptions): Core.APIPromise<Response> {
     return this._client.get(`/files/${id}/content`, { ...options, __binaryResponse: true });
   }
+
+  /**
+   * Upload a file.
+   */
+  upload(_: string): Promise<void> {
+    throw 'please use together-ai/lib/upload';
+  }
 }
 
 export interface FileObject {

package/src/version.ts

@@ -1 +1 @@
-export const VERSION = '0.8.0'; // x-release-please-version
+export const VERSION = '0.9.0'; // x-release-please-version

package/version.d.ts

@@ -1,2 +1,2 @@
-export declare const VERSION = "0.8.0";
+export declare const VERSION = "0.9.0";
 //# sourceMappingURL=version.d.ts.map

package/version.js

@@ -1,5 +1,5 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.VERSION = void 0;
-exports.VERSION = '0.8.0'; // x-release-please-version
+exports.VERSION = '0.9.0'; // x-release-please-version
 //# sourceMappingURL=version.js.map

package/version.mjs

@@ -1,2 +1,2 @@
-export const VERSION = '0.8.0'; // x-release-please-version
+export const VERSION = '0.9.0'; // x-release-please-version
 //# sourceMappingURL=version.mjs.map