telo 0.0.0 → 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -0
- package/dist/index.d.ts +9 -0
- package/dist/index.js +21 -0
- package/dist/index.test.d.mts +1 -0
- package/dist/index.test.mjs +21 -0
- package/dist/types.d.ts +11 -0
- package/dist/types.js +12 -0
- package/package.json +37 -4
- package/index.js +0 -1
package/README.md
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
```js
|
|
2
|
+
const telo = new Telo();
|
|
3
|
+
|
|
4
|
+
const record = await warc.createRecord(
|
|
5
|
+
TeloTypes.RESPONSE,
|
|
6
|
+
'http://example.com',
|
|
7
|
+
'<body />',
|
|
8
|
+
{
|
|
9
|
+
[Headers.CONTENT_TYPE]: 'text/html; charset=UTF-8'
|
|
10
|
+
},
|
|
11
|
+
{
|
|
12
|
+
Browser: 'Firefox'
|
|
13
|
+
}
|
|
14
|
+
);
|
|
15
|
+
|
|
16
|
+
const uncompressed = gunzipSync(record);
|
|
17
|
+
const text = uncompressed.toString('utf-8');
|
|
18
|
+
```
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { TeloTypes } from '#types.ts';
|
|
2
|
+
import { type WARCType } from 'warcio';
|
|
3
|
+
type Info = Record<string, any>;
|
|
4
|
+
declare class Telo {
|
|
5
|
+
private content;
|
|
6
|
+
static serializeInfo(filename: string, info: Record<string, string>): Promise<Uint8Array>;
|
|
7
|
+
createRecord(type: WARCType | undefined, url: string, page: string, httpHeaders?: HeadersInit, info?: Info): Promise<Buffer>;
|
|
8
|
+
}
|
|
9
|
+
export { Telo, TeloTypes };
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { gzipSync } from 'node:zlib';
|
|
2
|
+
import { TeloTypes } from '#types.ts';
|
|
3
|
+
import { WARC_1_1, WARCRecord, WARCSerializer } from 'warcio';
|
|
4
|
+
class Telo {
|
|
5
|
+
async *content(page) {
|
|
6
|
+
const textEncoder = new TextEncoder();
|
|
7
|
+
yield textEncoder.encode(page);
|
|
8
|
+
}
|
|
9
|
+
static async serializeInfo(filename, info) {
|
|
10
|
+
const warcInfo = await WARCRecord.createWARCInfo({ warcVersion: WARC_1_1, filename }, info);
|
|
11
|
+
return WARCSerializer.serialize(warcInfo);
|
|
12
|
+
}
|
|
13
|
+
async createRecord(type = TeloTypes.RESPONSE, url, page, httpHeaders = {}, info = {}) {
|
|
14
|
+
const record = await WARCRecord.create({ url, httpHeaders, type }, this.content(page));
|
|
15
|
+
const serializedRecord = await WARCSerializer.serialize(record);
|
|
16
|
+
const serializedInfo = await Telo.serializeInfo(url, info);
|
|
17
|
+
const buffer = Buffer.concat([serializedInfo, serializedRecord]);
|
|
18
|
+
return gzipSync(buffer);
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
export { Telo, TeloTypes };
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { gunzipSync } from 'node:zlib';
|
|
2
|
+
import { Headers } from 'http-directives';
|
|
3
|
+
import { describe, it, expect } from 'vitest';
|
|
4
|
+
import { Telo, TeloTypes } from '#index.ts';
|
|
5
|
+
describe('Telo', () => {
|
|
6
|
+
it('Should create a valid gzip WARC record', async () => {
|
|
7
|
+
const warc = new Telo();
|
|
8
|
+
const record = await warc.createRecord(TeloTypes.RESPONSE, 'http://example.com', '<body />', {
|
|
9
|
+
[Headers.CONTENT_TYPE]: 'text/html; charset=UTF-8'
|
|
10
|
+
}, {
|
|
11
|
+
Browser: 'Firefox'
|
|
12
|
+
});
|
|
13
|
+
const uncompressed = gunzipSync(record);
|
|
14
|
+
const text = uncompressed.toString('utf-8');
|
|
15
|
+
expect(text).toContain('WARC/1.1');
|
|
16
|
+
expect(text).toContain('http://example.com');
|
|
17
|
+
expect(text).toContain('Content-Type: text/html; charset=UTF-8');
|
|
18
|
+
expect(text).toContain('Browser: Firefox');
|
|
19
|
+
expect(text).toContain('<body />');
|
|
20
|
+
});
|
|
21
|
+
});
|
package/dist/types.d.ts
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
declare enum TeloTypes {
|
|
2
|
+
WARCINFO = "warcinfo",
|
|
3
|
+
RESPONSE = "response",
|
|
4
|
+
RESOURCE = "resource",
|
|
5
|
+
REQUEST = "request",
|
|
6
|
+
METADATA = "metadata",
|
|
7
|
+
REVISIT = "revisit",
|
|
8
|
+
CONVERSION = "conversion",
|
|
9
|
+
CONTINUATION = "continuation"
|
|
10
|
+
}
|
|
11
|
+
export { TeloTypes };
|
package/dist/types.js
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
var TeloTypes;
|
|
2
|
+
(function (TeloTypes) {
|
|
3
|
+
TeloTypes["WARCINFO"] = "warcinfo";
|
|
4
|
+
TeloTypes["RESPONSE"] = "response";
|
|
5
|
+
TeloTypes["RESOURCE"] = "resource";
|
|
6
|
+
TeloTypes["REQUEST"] = "request";
|
|
7
|
+
TeloTypes["METADATA"] = "metadata";
|
|
8
|
+
TeloTypes["REVISIT"] = "revisit";
|
|
9
|
+
TeloTypes["CONVERSION"] = "conversion";
|
|
10
|
+
TeloTypes["CONTINUATION"] = "continuation";
|
|
11
|
+
})(TeloTypes || (TeloTypes = {}));
|
|
12
|
+
export { TeloTypes };
|
package/package.json
CHANGED
|
@@ -1,6 +1,39 @@
|
|
|
1
1
|
{
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
2
|
+
"name": "telo",
|
|
3
|
+
"version": "0.0.1",
|
|
4
|
+
"type": "module",
|
|
5
|
+
"author": "Telo, Inc.",
|
|
6
|
+
"description": "",
|
|
7
|
+
"files": [
|
|
8
|
+
"dist"
|
|
9
|
+
],
|
|
10
|
+
"scripts": {
|
|
11
|
+
"build": "tsc",
|
|
12
|
+
"prepublishOnly": "npm run build && npm test",
|
|
13
|
+
"test": "vitest run"
|
|
14
|
+
},
|
|
15
|
+
"imports": {
|
|
16
|
+
"#*.ts": {
|
|
17
|
+
"telo": "./src/*.ts",
|
|
18
|
+
"default": "./dist/*.js"
|
|
19
|
+
}
|
|
20
|
+
},
|
|
21
|
+
"exports": {
|
|
22
|
+
".": "./dist/index.js",
|
|
23
|
+
"./*": "./dist/*"
|
|
24
|
+
},
|
|
25
|
+
"simple-git-hooks": {
|
|
26
|
+
"pre-commit": "npm test"
|
|
27
|
+
},
|
|
28
|
+
"dependencies": {
|
|
29
|
+
"warcio": "2.4.7"
|
|
30
|
+
},
|
|
31
|
+
"devDependencies": {
|
|
32
|
+
"@types/cli-progress": "3.11.6",
|
|
33
|
+
"@types/node": "24.2.1",
|
|
34
|
+
"http-directives": "^1.0.6",
|
|
35
|
+
"simple-git-hooks": "2.13.1",
|
|
36
|
+
"typescript": "5.9.2",
|
|
37
|
+
"vitest": "3.2.4"
|
|
38
|
+
}
|
|
6
39
|
}
|
package/index.js
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
|