shamela 1.2.3 → 1.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE.MD +1 -1
- package/README.md +262 -6
- package/dist/index.d.ts +208 -115
- package/dist/index.js +8 -7
- package/dist/index.js.map +1 -1
- package/package.json +22 -13
package/dist/index.d.ts
CHANGED
|
@@ -1,92 +1,95 @@
|
|
|
1
|
+
//#region src/db/types.d.ts
|
|
2
|
+
|
|
1
3
|
/**
|
|
2
4
|
* A record that can be deleted by patches.
|
|
3
5
|
*/
|
|
4
6
|
type Deletable = {
|
|
5
|
-
|
|
6
|
-
|
|
7
|
+
/** Indicates if it was deleted in the patch if it is set to '1 */
|
|
8
|
+
is_deleted?: string;
|
|
7
9
|
};
|
|
8
10
|
type Unique = {
|
|
9
|
-
|
|
10
|
-
|
|
11
|
+
/** Unique identifier */
|
|
12
|
+
id: number;
|
|
11
13
|
};
|
|
12
14
|
/**
|
|
13
15
|
* Database row structure for the author table.
|
|
14
16
|
*/
|
|
15
17
|
type AuthorRow = Deletable & Unique & {
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
18
|
+
/** Author biography */
|
|
19
|
+
biography: string;
|
|
20
|
+
/** Death year */
|
|
21
|
+
death_number: string;
|
|
22
|
+
/** The death year as a text */
|
|
23
|
+
death_text: string;
|
|
24
|
+
/** Author name */
|
|
25
|
+
name: string;
|
|
24
26
|
};
|
|
25
27
|
/**
|
|
26
28
|
* Database row structure for the book table.
|
|
27
29
|
*/
|
|
28
30
|
type BookRow = Deletable & Unique & {
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
31
|
+
/** Serialized author ID(s) "2747, 3147" or "513" */
|
|
32
|
+
author: string;
|
|
33
|
+
/** Bibliography information */
|
|
34
|
+
bibliography: string;
|
|
35
|
+
/** Category ID */
|
|
36
|
+
category: string;
|
|
37
|
+
/** Publication date (or 99999 for unavailable) */
|
|
38
|
+
date: string;
|
|
39
|
+
/** Hint or description */
|
|
40
|
+
hint: string;
|
|
41
|
+
/** Major version */
|
|
42
|
+
major_release: string;
|
|
43
|
+
/** Serialized metadata */
|
|
44
|
+
metadata: string;
|
|
45
|
+
/** Minor version */
|
|
46
|
+
minor_release: string;
|
|
47
|
+
/** Book name */
|
|
48
|
+
name: string;
|
|
49
|
+
/** Serialized PDF links */
|
|
50
|
+
pdf_links: string;
|
|
51
|
+
/** Printed flag */
|
|
52
|
+
printed: string;
|
|
53
|
+
/** Book type */
|
|
54
|
+
type: string;
|
|
53
55
|
};
|
|
54
56
|
/**
|
|
55
57
|
* Database row structure for the category table.
|
|
56
58
|
*/
|
|
57
59
|
type CategoryRow = Deletable & Unique & {
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
60
|
+
/** Category name */
|
|
61
|
+
name: string;
|
|
62
|
+
/** Category order in the list to show. */
|
|
63
|
+
order: string;
|
|
62
64
|
};
|
|
63
65
|
/**
|
|
64
66
|
* Database row structure for the page table.
|
|
65
67
|
*/
|
|
66
68
|
type PageRow = Deletable & Unique & {
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
69
|
+
/** Page content */
|
|
70
|
+
content: string;
|
|
71
|
+
/** Page number */
|
|
72
|
+
number: string | null;
|
|
73
|
+
/** Page reference */
|
|
74
|
+
page: string | null;
|
|
75
|
+
/** Part number */
|
|
76
|
+
part: string | null;
|
|
77
|
+
/** Additional metadata */
|
|
78
|
+
services: string | null;
|
|
77
79
|
};
|
|
78
80
|
/**
|
|
79
81
|
* Database row structure for the title table.
|
|
80
82
|
*/
|
|
81
83
|
type TitleRow = Deletable & Unique & {
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
84
|
+
/** Title content */
|
|
85
|
+
content: string;
|
|
86
|
+
/** Page number */
|
|
87
|
+
page: string;
|
|
88
|
+
/** Parent title ID */
|
|
89
|
+
parent: string | null;
|
|
88
90
|
};
|
|
89
|
-
|
|
91
|
+
//#endregion
|
|
92
|
+
//#region src/types.d.ts
|
|
90
93
|
/**
|
|
91
94
|
* Represents an author entity.
|
|
92
95
|
*/
|
|
@@ -103,94 +106,123 @@ type Category = CategoryRow;
|
|
|
103
106
|
* A page in a book.
|
|
104
107
|
*/
|
|
105
108
|
type Page = Pick<PageRow, 'id' | 'content'> & {
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
+
page?: number;
|
|
110
|
+
part?: string;
|
|
111
|
+
number?: string;
|
|
109
112
|
};
|
|
110
113
|
/**
|
|
111
114
|
* A title heading in a book.
|
|
112
115
|
*/
|
|
113
116
|
type Title = Pick<TitleRow, 'id' | 'content'> & {
|
|
114
|
-
|
|
115
|
-
|
|
117
|
+
page: number;
|
|
118
|
+
parent?: number;
|
|
116
119
|
};
|
|
117
120
|
/**
|
|
118
121
|
* Represents book content data.
|
|
119
122
|
*/
|
|
120
123
|
type BookData = {
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
124
|
+
/** Array of pages in the book */
|
|
125
|
+
pages: Page[];
|
|
126
|
+
/** Array of titles/chapters */
|
|
127
|
+
titles: Title[];
|
|
125
128
|
};
|
|
126
129
|
/**
|
|
127
130
|
* Master data structure containing all core entities.
|
|
128
131
|
*/
|
|
129
132
|
type MasterData = {
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
133
|
+
/** Array of all authors */
|
|
134
|
+
authors: Author[];
|
|
135
|
+
/** Array of all books */
|
|
136
|
+
books: Book[];
|
|
137
|
+
/** Array of all categories */
|
|
138
|
+
categories: Category[];
|
|
139
|
+
/** Version number for the downloaded master database */
|
|
140
|
+
version: number;
|
|
136
141
|
};
|
|
137
142
|
/**
|
|
138
143
|
* Options for downloading a book.
|
|
139
144
|
*/
|
|
140
145
|
type DownloadBookOptions = {
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
146
|
+
/** Optional book metadata */
|
|
147
|
+
bookMetadata?: GetBookMetadataResponsePayload;
|
|
148
|
+
/** Output file configuration */
|
|
149
|
+
outputFile: OutputOptions;
|
|
145
150
|
};
|
|
146
151
|
/**
|
|
147
152
|
* Options for downloading master data.
|
|
148
153
|
*/
|
|
149
154
|
type DownloadMasterOptions = {
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
155
|
+
/** Optional master metadata */
|
|
156
|
+
masterMetadata?: GetMasterMetadataResponsePayload;
|
|
157
|
+
/** Output file configuration */
|
|
158
|
+
outputFile: OutputOptions;
|
|
154
159
|
};
|
|
155
160
|
/**
|
|
156
161
|
* Options for getting book metadata.
|
|
157
162
|
*/
|
|
158
163
|
type GetBookMetadataOptions = {
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
164
|
+
/** Major version number */
|
|
165
|
+
majorVersion: number;
|
|
166
|
+
/** Minor version number */
|
|
167
|
+
minorVersion: number;
|
|
163
168
|
};
|
|
164
169
|
/**
|
|
165
170
|
* Response payload for book metadata requests.
|
|
166
171
|
*/
|
|
167
172
|
type GetBookMetadataResponsePayload = {
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
173
|
+
/** Major release version */
|
|
174
|
+
majorRelease: number;
|
|
175
|
+
/** URL for major release download */
|
|
176
|
+
majorReleaseUrl: string;
|
|
177
|
+
/** Optional minor release version */
|
|
178
|
+
minorRelease?: number;
|
|
179
|
+
/** Optional URL for minor release download */
|
|
180
|
+
minorReleaseUrl?: string;
|
|
176
181
|
};
|
|
177
182
|
/**
|
|
178
183
|
* Response payload for master metadata requests.
|
|
179
184
|
*/
|
|
180
185
|
type GetMasterMetadataResponsePayload = {
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
186
|
+
/** Download URL */
|
|
187
|
+
url: string;
|
|
188
|
+
/** Version number */
|
|
189
|
+
version: number;
|
|
190
|
+
};
|
|
191
|
+
type NodeJSOutput = {
|
|
192
|
+
/** Output file path (Node.js only) */
|
|
193
|
+
path: string;
|
|
194
|
+
writer?: never;
|
|
195
|
+
};
|
|
196
|
+
type CustomOutput = {
|
|
197
|
+
/** Custom writer used when path is not provided */
|
|
198
|
+
writer: (payload: string | Uint8Array) => Promise<void> | void;
|
|
199
|
+
path?: undefined;
|
|
185
200
|
};
|
|
186
201
|
/**
|
|
187
202
|
* Output file options.
|
|
188
203
|
*/
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
204
|
+
type OutputOptions = NodeJSOutput | CustomOutput;
|
|
205
|
+
/**
|
|
206
|
+
* Runtime configuration for the library.
|
|
207
|
+
*/
|
|
208
|
+
type ShamelaConfig = {
|
|
209
|
+
/** API key used to authenticate against Shamela services */
|
|
210
|
+
apiKey?: string;
|
|
211
|
+
/** Endpoint used for book metadata */
|
|
212
|
+
booksEndpoint?: string;
|
|
213
|
+
/** Endpoint used for master metadata */
|
|
214
|
+
masterPatchEndpoint?: string;
|
|
215
|
+
/** Optional override for the sql.js wasm asset location */
|
|
216
|
+
sqlJsWasmUrl?: string;
|
|
217
|
+
/** Optional custom fetch implementation for environments without a global fetch */
|
|
218
|
+
fetchImplementation?: typeof fetch;
|
|
219
|
+
};
|
|
220
|
+
/**
|
|
221
|
+
* Valid configuration keys.
|
|
222
|
+
*/
|
|
223
|
+
type ShamelaConfigKey = keyof ShamelaConfig;
|
|
224
|
+
//#endregion
|
|
225
|
+
//#region src/api.d.ts
|
|
194
226
|
/**
|
|
195
227
|
* Retrieves metadata for a specific book from the Shamela API.
|
|
196
228
|
*
|
|
@@ -320,10 +352,79 @@ declare const downloadMasterDatabase: (options: DownloadMasterOptions) => Promis
|
|
|
320
352
|
* ```
|
|
321
353
|
*/
|
|
322
354
|
declare const getBook: (id: number) => Promise<BookData>;
|
|
323
|
-
|
|
355
|
+
/**
|
|
356
|
+
* Retrieves complete master data including authors, books, and categories.
|
|
357
|
+
*
|
|
358
|
+
* This convenience function downloads the master database archive, builds an in-memory
|
|
359
|
+
* SQLite database, and returns structured data for immediate consumption alongside
|
|
360
|
+
* the version number of the snapshot.
|
|
361
|
+
*
|
|
362
|
+
* @returns A promise that resolves to the complete master dataset and its version
|
|
363
|
+
*/
|
|
364
|
+
declare const getMaster: () => Promise<MasterData>;
|
|
365
|
+
//#endregion
|
|
366
|
+
//#region src/utils/logger.d.ts
|
|
367
|
+
/**
|
|
368
|
+
* Signature accepted by logger methods.
|
|
369
|
+
*/
|
|
370
|
+
type LogFunction = (...args: unknown[]) => void;
|
|
371
|
+
/**
|
|
372
|
+
* Contract expected from logger implementations consumed by the library.
|
|
373
|
+
*/
|
|
374
|
+
interface Logger {
|
|
375
|
+
debug: LogFunction;
|
|
376
|
+
error: LogFunction;
|
|
377
|
+
info: LogFunction;
|
|
378
|
+
warn: LogFunction;
|
|
379
|
+
}
|
|
380
|
+
//#endregion
|
|
381
|
+
//#region src/config.d.ts
|
|
382
|
+
/**
|
|
383
|
+
* Runtime configuration options accepted by {@link configure}.
|
|
384
|
+
*/
|
|
385
|
+
type ConfigureOptions = Partial<ShamelaConfig> & {
|
|
386
|
+
logger?: Logger;
|
|
387
|
+
};
|
|
388
|
+
/**
|
|
389
|
+
* Updates the runtime configuration for the library.
|
|
390
|
+
*
|
|
391
|
+
* This function merges the provided options with existing overrides and optionally
|
|
392
|
+
* configures a custom logger implementation.
|
|
393
|
+
*
|
|
394
|
+
* @param config - Runtime configuration overrides and optional logger instance
|
|
395
|
+
*/
|
|
396
|
+
declare const configure: (config: ConfigureOptions) => void;
|
|
397
|
+
/**
|
|
398
|
+
* Clears runtime configuration overrides and restores the default logger.
|
|
399
|
+
*/
|
|
400
|
+
declare const resetConfig: () => void;
|
|
401
|
+
/**
|
|
402
|
+
* Creates a default configuration for Node.js environments.
|
|
403
|
+
* Automatically sets the correct sqlJsWasmUrl path for bundled environments.
|
|
404
|
+
*
|
|
405
|
+
* This helper is optional - the library will auto-detect the WASM file location
|
|
406
|
+
* in most cases. Use this if you want explicit control or are experiencing issues.
|
|
407
|
+
*
|
|
408
|
+
* @param config - Your API configuration
|
|
409
|
+
* @returns Complete configuration with sqlJsWasmUrl set for Node.js
|
|
410
|
+
*
|
|
411
|
+
* @example
|
|
412
|
+
* ```typescript
|
|
413
|
+
* import { configure, createNodeConfig } from 'shamela';
|
|
414
|
+
*
|
|
415
|
+
* configure(createNodeConfig({
|
|
416
|
+
* apiKey: process.env.SHAMELA_API_KEY,
|
|
417
|
+
* booksEndpoint: process.env.SHAMELA_BOOKS_ENDPOINT,
|
|
418
|
+
* masterPatchEndpoint: process.env.SHAMELA_MASTER_ENDPOINT,
|
|
419
|
+
* }));
|
|
420
|
+
* ```
|
|
421
|
+
*/
|
|
422
|
+
declare const createNodeConfig: (config: Omit<ShamelaConfig, "sqlJsWasmUrl">) => ShamelaConfig;
|
|
423
|
+
//#endregion
|
|
424
|
+
//#region src/content.d.ts
|
|
324
425
|
type Line = {
|
|
325
|
-
|
|
326
|
-
|
|
426
|
+
id?: string;
|
|
427
|
+
text: string;
|
|
327
428
|
};
|
|
328
429
|
declare const parseContentRobust: (content: string) => Line[];
|
|
329
430
|
/**
|
|
@@ -336,14 +437,6 @@ declare const sanitizePageContent: (text: string, rules?: Record<string, string>
|
|
|
336
437
|
declare const splitPageBodyFromFooter: (content: string, footnoteMarker?: string) => readonly [string, string];
|
|
337
438
|
declare const removeArabicNumericPageMarkers: (text: string) => string;
|
|
338
439
|
declare const removeTagsExceptSpan: (content: string) => string;
|
|
339
|
-
|
|
340
|
-
type
|
|
341
|
-
|
|
342
|
-
debug: LogFunction;
|
|
343
|
-
error: LogFunction;
|
|
344
|
-
info: LogFunction;
|
|
345
|
-
warn: LogFunction;
|
|
346
|
-
}
|
|
347
|
-
declare const setLogger: (newLogger?: Logger) => void;
|
|
348
|
-
|
|
349
|
-
export { type Author, type Book, type BookData, type Category, type DownloadBookOptions, type DownloadMasterOptions, type GetBookMetadataOptions, type GetBookMetadataResponsePayload, type GetMasterMetadataResponsePayload, type Line, type MasterData, type OutputOptions, type Page, type Title, downloadBook, downloadMasterDatabase, getBook, getBookMetadata, getCoverUrl, getMasterMetadata, parseContentRobust, removeArabicNumericPageMarkers, removeTagsExceptSpan, sanitizePageContent, setLogger, splitPageBodyFromFooter };
|
|
440
|
+
//#endregion
|
|
441
|
+
export { Author, Book, BookData, Category, type ConfigureOptions, DownloadBookOptions, DownloadMasterOptions, GetBookMetadataOptions, GetBookMetadataResponsePayload, GetMasterMetadataResponsePayload, Line, type Logger, MasterData, OutputOptions, Page, ShamelaConfig, ShamelaConfigKey, Title, configure, createNodeConfig, downloadBook, downloadMasterDatabase, getBook, getBookMetadata, getCoverUrl, getMaster, getMasterMetadata, parseContentRobust, removeArabicNumericPageMarkers, removeTagsExceptSpan, resetConfig, sanitizePageContent, splitPageBodyFromFooter };
|
|
442
|
+
//# sourceMappingURL=index.d.ts.map
|
package/dist/index.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import
|
|
1
|
+
import e from"sql.js";import{unzipSync as t}from"fflate";var n=(e=>typeof require<`u`?require:typeof Proxy<`u`?new Proxy(e,{get:(e,t)=>(typeof require<`u`?require:e)[t]}):e)(function(e){if(typeof require<`u`)return require.apply(this,arguments);throw Error('Calling `require` for "'+e+"\" in an environment that doesn't expose the `require` function.")});const r=Object.freeze({debug:()=>{},error:()=>{},info:()=>{},warn:()=>{}});let i=r;const a=e=>{if(!e){i=r;return}let t=[`debug`,`error`,`info`,`warn`].find(t=>typeof e[t]!=`function`);if(t)throw Error(`Logger must implement debug, error, info, and warn methods. Missing: ${String(t)}`);i=e},o=()=>i,s=()=>{i=r};var c=new Proxy({},{get:(e,t)=>{let n=o(),r=n[t];return typeof r==`function`?(...e)=>r.apply(n,e):r}});let l={};const u={apiKey:`SHAMELA_API_KEY`,booksEndpoint:`SHAMELA_API_BOOKS_ENDPOINT`,masterPatchEndpoint:`SHAMELA_API_MASTER_PATCH_ENDPOINT`,sqlJsWasmUrl:`SHAMELA_SQLJS_WASM_URL`},ee=typeof process<`u`&&!!process?.env,d=e=>{let t=l[e];if(t!==void 0)return t;let n=u[e];if(ee)return process.env[n]},te=e=>{let{logger:t,...n}=e;`logger`in e&&a(t),l={...l,...n}},f=e=>e===`fetchImplementation`?l.fetchImplementation:d(e),p=()=>({apiKey:d(`apiKey`),booksEndpoint:d(`booksEndpoint`),fetchImplementation:l.fetchImplementation,masterPatchEndpoint:d(`masterPatchEndpoint`),sqlJsWasmUrl:d(`sqlJsWasmUrl`)}),m=e=>{if(e===`fetchImplementation`)throw Error(`fetchImplementation must be provided via configure().`);let t=f(e);if(!t)throw Error(`${u[e]} environment variable not set`);return t},ne=()=>{l={},s()},re=e=>{let{join:t}=n(`node:path`);return{...e,sqlJsWasmUrl:t(process.cwd(),`node_modules`,`sql.js`,`dist`,`sql-wasm.wasm`)}};let h=function(e){return e.Authors=`author`,e.Books=`book`,e.Categories=`category`,e.Page=`page`,e.Title=`title`,e}({});const g=(e,t)=>e.query(`PRAGMA table_info(${t})`).all(),_=(e,t)=>!!e.query(`SELECT name FROM sqlite_master WHERE type='table' AND name = ?1`).get(t),v=(e,t)=>_(e,t)?e.query(`SELECT * FROM ${t}`).all():[],y=e=>String(e.is_deleted)===`1`,b=(e,t,n)=>{let r={};for(let i of n){if(i===`id`){r.id=(t??e)?.id??null;continue}if(t&&i in t){let e=t[i];if(e!==`#`&&e!=null){r[i]=e;continue}}if(e&&i in e){r[i]=e[i];continue}r[i]=null}return r},ie=(e,t,n)=>{let r=new Set,i=new Map;for(let t of e)r.add(String(t.id));for(let e of t)i.set(String(e.id),e);let a=[];for(let t of e){let e=i.get(String(t.id));e&&y(e)||a.push(b(t,e,n))}for(let e of t){let t=String(e.id);r.has(t)||y(e)||a.push(b(void 0,e,n))}return a},ae=(e,t,n,r)=>{if(r.length===0)return;let i=n.map(()=>`?`).join(`,`),a=e.prepare(`INSERT INTO ${t} (${n.join(`,`)}) VALUES (${i})`);r.forEach(e=>{let t=n.map(t=>t in e?e[t]:null);a.run(...t)}),a.finalize()},oe=(e,t,n)=>{let r=t.query(`SELECT sql FROM sqlite_master WHERE type='table' AND name = ?1`).get(n);return r?.sql?(e.run(`DROP TABLE IF EXISTS ${n}`),e.run(r.sql),!0):(c.warn(`${n} table definition missing in source database`),!1)},x=(e,t,n,r)=>{if(!_(t,r)){c.warn(`${r} table missing in source database`);return}if(!oe(e,t,r))return;let i=g(t,r),a=n&&_(n,r)?g(n,r):[],o=i.map(e=>e.name);for(let t of a)if(!o.includes(t.name)){let n=t.type&&t.type.length>0?t.type:`TEXT`;e.run(`ALTER TABLE ${r} ADD COLUMN ${t.name} ${n}`),o.push(t.name)}ae(e,r,o,ie(v(t,r),n?v(n,r):[],o))},se=(e,t,n)=>{e.transaction(()=>{x(e,t,n,h.Page),x(e,t,n,h.Title)})()},ce=(e,t)=>{e.transaction(()=>{x(e,t,null,h.Page),x(e,t,null,h.Title)})()},S=e=>{e.run(`CREATE TABLE ${h.Page} (
|
|
2
2
|
id INTEGER,
|
|
3
3
|
content TEXT,
|
|
4
4
|
part TEXT,
|
|
@@ -6,20 +6,21 @@ import{Database as W}from"bun:sqlite";import{promises as m}from"fs";import g fro
|
|
|
6
6
|
number TEXT,
|
|
7
7
|
services TEXT,
|
|
8
8
|
is_deleted TEXT
|
|
9
|
-
)`),e.run(`CREATE TABLE
|
|
9
|
+
)`),e.run(`CREATE TABLE ${h.Title} (
|
|
10
10
|
id INTEGER,
|
|
11
11
|
content TEXT,
|
|
12
12
|
page INTEGER,
|
|
13
13
|
parent INTEGER,
|
|
14
14
|
is_deleted TEXT
|
|
15
|
-
)`)},
|
|
15
|
+
)`)},C=e=>e.query(`SELECT * FROM ${h.Page}`).all(),le=e=>e.query(`SELECT * FROM ${h.Title}`).all(),w=e=>({pages:C(e),titles:le(e)}),T=e=>{try{return n(`node:fs`).existsSync(e)}catch{return!1}},ue=()=>{if(n!==void 0&&n.resolve!==void 0)try{let e=n.resolve(`sql.js`),t=n(`node:path`),r=t.dirname(e),i=t.join(r,`dist`,`sql-wasm.wasm`);if(T(i))return i}catch{}if(typeof process<`u`&&process.cwd)try{let e=n(`node:path`),t=process.cwd(),r=[e.join(t,`node_modules`,`sql.js`,`dist`,`sql-wasm.wasm`),e.join(t,`..`,`node_modules`,`sql.js`,`dist`,`sql-wasm.wasm`),e.join(t,`../..`,`node_modules`,`sql.js`,`dist`,`sql-wasm.wasm`),e.join(t,`.next`,`server`,`node_modules`,`sql.js`,`dist`,`sql-wasm.wasm`)];for(let e of r)if(T(e))return e}catch{}if(n!==void 0&&n.resolve!==void 0&&n.resolve.paths)try{let e=n(`node:path`),t=n.resolve.paths(`sql.js`)||[];for(let n of t){let t=e.join(n,`sql.js`,`dist`,`sql-wasm.wasm`);if(T(t))return t}}catch{}try{if(import.meta.url){let e=new URL(`../../node_modules/sql.js/dist/sql-wasm.wasm`,import.meta.url),t=decodeURIComponent(e.pathname),n=process.platform===`win32`&&t.startsWith(`/`)?t.slice(1):t;if(T(n))return n}}catch{}return null};var de=class{constructor(e){this.statement=e}run=(...e)=>{e.length>0&&this.statement.bind(e),this.statement.step(),this.statement.reset()};finalize=()=>{this.statement.free()}},E=class{constructor(e){this.db=e}run=(e,t=[])=>{this.db.run(e,t)};prepare=e=>new de(this.db.prepare(e));query=e=>({all:(...t)=>this.all(e,t),get:(...t)=>this.get(e,t)});transaction=e=>()=>{this.db.run(`BEGIN TRANSACTION`);try{e(),this.db.run(`COMMIT`)}catch(e){throw this.db.run(`ROLLBACK`),e}};close=()=>{this.db.close()};export=()=>this.db.export();all=(e,t)=>{let n=this.db.prepare(e);try{t.length>0&&n.bind(t);let e=[];for(;n.step();)e.push(n.getAsObject());return e}finally{n.free()}};get=(e,t)=>this.all(e,t)[0]};let D=null,O=null;const fe=typeof process<`u`&&!!process?.versions?.node,pe=()=>{if(!O){let e=f(`sqlJsWasmUrl`);if(e)O=e;else if(fe){let e=ue();if(e)O=e;else{let e=[`Unable to automatically locate sql-wasm.wasm file.`,`This can happen in bundled environments (Next.js, webpack, etc.).`,``,`Quick fix - add this to your code before using shamela:`,``,` import { configure, createNodeConfig } from "shamela";`,` configure(createNodeConfig({`,` apiKey: process.env.SHAMELA_API_KEY,`,` booksEndpoint: process.env.SHAMELA_BOOKS_ENDPOINT,`,` masterPatchEndpoint: process.env.SHAMELA_MASTER_ENDPOINT,`,` }));`,``,`Or manually specify the path:`,``,` import { configure } from "shamela";`,` import { join } from "node:path";`,` configure({`,` sqlJsWasmUrl: join(process.cwd(), "node_modules", "sql.js", "dist", "sql-wasm.wasm")`,` });`].join(`
|
|
16
|
+
`);throw Error(e)}}else O=`https://cdn.jsdelivr.net/npm/sql.js@1.13.0/dist/sql-wasm.wasm`}return O},k=()=>(D||=e({locateFile:()=>pe()}),D),A=async()=>new E(new(await(k())).Database),j=async e=>new E(new(await(k())).Database(e)),me=(e,t,n)=>{let r=t.query(`SELECT sql FROM sqlite_master WHERE type='table' AND name = ?1`).get(n);if(!r?.sql)throw Error(`Missing table definition for ${n} in source database`);e.run(`DROP TABLE IF EXISTS ${n}`),e.run(r.sql)},he=async(e,t)=>{let n={author:h.Authors,book:h.Books,category:h.Categories},r={};for(let e of t){let t=n[(e.name.split(`/`).pop()?.split(`\\`).pop()??e.name).replace(/\.(sqlite|db)$/i,``).toLowerCase()];t&&(r[t]=await j(e.data))}try{let t=Object.entries(r);e.transaction(()=>{for(let[n,r]of t){me(e,r,n);let t=r.query(`PRAGMA table_info(${n})`).all().map(e=>e.name);if(t.length===0)continue;let i=r.query(`SELECT * FROM ${n}`).all();if(i.length===0)continue;let a=t.map(()=>`?`).join(`,`),o=t.map(e=>e===`order`?`"order"`:e),s=e.prepare(`INSERT INTO ${n} (${o.join(`,`)}) VALUES (${a})`);try{for(let e of i){let n=t.map(t=>t in e?e[t]:null);s.run(...n)}}finally{s.finalize()}}})()}finally{Object.values(r).forEach(e=>e?.close())}},M=(e,t,n)=>{e.run(`DROP VIEW IF EXISTS ${t}`),e.run(`CREATE VIEW ${t} AS SELECT * FROM ${n}`)},ge=e=>{e.run(`CREATE TABLE ${h.Authors} (
|
|
16
17
|
id INTEGER,
|
|
17
18
|
is_deleted TEXT,
|
|
18
19
|
name TEXT,
|
|
19
20
|
biography TEXT,
|
|
20
21
|
death_text TEXT,
|
|
21
22
|
death_number TEXT
|
|
22
|
-
)`),e.run(`CREATE TABLE
|
|
23
|
+
)`),e.run(`CREATE TABLE ${h.Books} (
|
|
23
24
|
id INTEGER,
|
|
24
25
|
name TEXT,
|
|
25
26
|
is_deleted TEXT,
|
|
@@ -34,14 +35,14 @@ import{Database as W}from"bun:sqlite";import{promises as m}from"fs";import g fro
|
|
|
34
35
|
hint TEXT,
|
|
35
36
|
pdf_links TEXT,
|
|
36
37
|
metadata TEXT
|
|
37
|
-
)`),e.run(`CREATE TABLE
|
|
38
|
+
)`),e.run(`CREATE TABLE ${h.Categories} (
|
|
38
39
|
id INTEGER,
|
|
39
40
|
is_deleted TEXT,
|
|
40
41
|
"order" TEXT,
|
|
41
42
|
name TEXT
|
|
42
|
-
)`),
|
|
43
|
+
)`),M(e,`authors`,h.Authors),M(e,`books`,h.Books),M(e,`categories`,h.Categories)},_e=e=>e.query(`SELECT * FROM ${h.Authors}`).all(),N=e=>e.query(`SELECT * FROM ${h.Books}`).all(),P=e=>e.query(`SELECT * FROM ${h.Categories}`).all(),F=(e,t)=>({authors:_e(e),books:N(e),categories:P(e),version:t}),I=(e,t=[`api_key`,`token`,`password`,`secret`,`auth`])=>{let n=typeof e==`string`?new URL(e):new URL(e.toString());return t.forEach(e=>{let t=n.searchParams.get(e);if(t&&t.length>6){let r=`${t.slice(0,3)}***${t.slice(-3)}`;n.searchParams.set(e,r)}else t&&n.searchParams.set(e,`***`)}),n.toString()},ve=e=>({content:e.content,id:e.id,...e.number&&{number:e.number},...e.page&&{page:Number(e.page)},...e.part&&{part:e.part}}),ye=e=>{let t=Number(e.parent);return{content:e.content,id:e.id,page:Number(e.page),...t&&{parent:t}}},L={"<img[^>]*>>":``,舄:``,"﵀":`رَحِمَهُ ٱللَّٰهُ`,"﵁":`رضي الله عنه`,"﵂":`رَضِيَ ٱللَّٰهُ عَنْهَا`,"﵃":`رَضِيَ اللَّهُ عَنْهُمْ`,"﵄":`رَضِيَ ٱللَّٰهُ عَنْهُمَا`,"﵅":`رَضِيَ اللَّهُ عَنْهُنَّ`,"﵌":`صلى الله عليه وآله وسلم`,"﵏":`رَحِمَهُمُ ٱللَّٰهُ`},R=e=>{let t=new URL(e);return t.protocol=`https`,t.toString()},z=e=>/\.(sqlite|db)$/i.test(e.name),B=e=>e.find(z),V=e=>{let t=/\.([^.]+)$/.exec(e);return t?`.${t[1].toLowerCase()}`:``},H=(e,t,n=!0)=>{let r=new URL(e),i=new URLSearchParams;return Object.entries(t).forEach(([e,t])=>{i.append(e,t.toString())}),n&&i.append(`api_key`,m(`apiKey`)),r.search=i.toString(),r},U=async(e,t={})=>{let n=typeof e==`string`?e:e.toString(),r=await(t.fetchImpl??p().fetchImplementation??fetch)(n);if(!r.ok)throw Error(`Error making request: ${r.status} ${r.statusText}`);if((r.headers.get(`content-type`)??``).includes(`application/json`))return await r.json();let i=await r.arrayBuffer();return new Uint8Array(i)},be=typeof process<`u`&&!!process?.versions?.node,xe=async()=>{if(!be)throw Error(`File system operations are only supported in Node.js environments`);return import(`node:fs/promises`)},Se=async e=>{let[t,n]=await Promise.all([xe(),import(`node:path`)]),r=n.dirname(e);return await t.mkdir(r,{recursive:!0}),t},W=async e=>{let n=await U(e),r=n instanceof Uint8Array?n.length:n&&typeof n.byteLength==`number`?n.byteLength:0;return c.debug(`unzipFromUrl:bytes`,r),new Promise((e,r)=>{let i=n instanceof Uint8Array?n:new Uint8Array(n);try{let n=t(i),r=Object.entries(n).map(([e,t])=>({data:t,name:e}));c.debug(`unzipFromUrl:entries`,r.map(e=>e.name)),e(r)}catch(e){r(Error(`Error processing URL: ${e.message}`))}})},G=async(e,t)=>{if(e.writer){await e.writer(t);return}if(!e.path)throw Error(`Output options must include either a writer or a path`);let n=await Se(e.path);typeof t==`string`?await n.writeFile(e.path,t,`utf-8`):await n.writeFile(e.path,t)},Ce=[`author.sqlite`,`book.sqlite`,`category.sqlite`],K=()=>{let{apiKey:e,booksEndpoint:t,masterPatchEndpoint:n}=p(),r=[[`apiKey`,e],[`booksEndpoint`,t],[`masterPatchEndpoint`,n]].filter(([,e])=>!e).map(([e])=>e);if(r.length)throw Error(`${r.join(`, `)} environment variables not set`)},we=e=>{let t=new Set(e.map(e=>e.match(/[^\\/]+$/)?.[0]??e).map(e=>e.toLowerCase()));return Ce.every(e=>t.has(e.toLowerCase()))},q=async(e,t)=>{c.info(`Setting up book database for ${e}`);let n=t||await Y(e),r=n.minorReleaseUrl?W(n.minorReleaseUrl):Promise.resolve([]),[i,a]=await Promise.all([W(n.majorReleaseUrl),r]),o=B(i);if(!o)throw Error(`Unable to locate book database in archive`);let s=await A();try{c.info(`Creating tables`),S(s);let e=await j(o.data);try{let t=B(a);if(t){c.info(`Applying patches from ${t.name} to ${o.name}`);let n=await j(t.data);try{se(s,e,n)}finally{n.close()}}else c.info(`Copying table data from ${o.name}`),ce(s,e)}finally{e.close()}return{cleanup:async()=>{s.close()},client:s}}catch(e){throw s.close(),e}},J=async e=>{c.info(`Setting up master database`);let t=e||await X(0);c.info(`Downloading master database ${t.version} from: ${I(t.url)}`);let n=await W(R(t.url));if(c.debug?.(`sourceTables downloaded: ${n.map(e=>e.name).toString()}`),!we(n.map(e=>e.name)))throw c.error(`Some source tables were not found: ${n.map(e=>e.name).toString()}`),Error(`Expected tables not found!`);let r=await A();try{return c.info(`Creating master tables`),ge(r),c.info(`Copying data to master table`),await he(r,n.filter(z)),{cleanup:async()=>{r.close()},client:r,version:t.version}}catch(e){throw r.close(),e}},Y=async(e,t)=>{K();let n=H(`${m(`booksEndpoint`)}/${e}`,{major_release:(t?.majorVersion||0).toString(),minor_release:(t?.minorVersion||0).toString()});c.info(`Fetching shamela.ws book link: ${I(n)}`);try{let e=await U(n);return{majorRelease:e.major_release,majorReleaseUrl:R(e.major_release_url),...e.minor_release_url&&{minorReleaseUrl:R(e.minor_release_url)},...e.minor_release_url&&{minorRelease:e.minor_release}}}catch(e){throw Error(`Error fetching book metadata: ${e.message}`)}},Te=async(e,t)=>{if(c.info(`downloadBook ${e} ${JSON.stringify(t)}`),!t.outputFile.path)throw Error(`outputFile.path must be provided to determine output format`);let n=V(t.outputFile.path).toLowerCase(),{client:r,cleanup:i}=await q(e,t?.bookMetadata);try{if(n===`.json`){let e=await w(r);await G(t.outputFile,JSON.stringify(e,null,2))}else if(n===`.db`||n===`.sqlite`){let e=r.export();await G(t.outputFile,e)}else throw Error(`Unsupported output extension: ${n}`)}finally{await i()}return t.outputFile.path},X=async(e=0)=>{K();let t=H(m(`masterPatchEndpoint`),{version:e.toString()});c.info(`Fetching shamela.ws master database patch link: ${I(t)}`);try{let e=await U(t);return{url:e.patch_url,version:e.version}}catch(e){throw Error(`Error fetching master patch: ${e.message}`)}},Ee=e=>{let t=m(`masterPatchEndpoint`),{origin:n}=new URL(t);return`${n}/covers/${e}.jpg`},De=async e=>{if(c.info(`downloadMasterDatabase ${JSON.stringify(e)}`),!e.outputFile.path)throw Error(`outputFile.path must be provided to determine output format`);let t=V(e.outputFile.path),{client:n,cleanup:r,version:i}=await J(e.masterMetadata);try{if(t===`.json`){let t=F(n,i);await G(e.outputFile,JSON.stringify(t,null,2))}else if(t===`.db`||t===`.sqlite`)await G(e.outputFile,n.export());else throw Error(`Unsupported output extension: ${t}`)}finally{await r()}return e.outputFile.path},Oe=async e=>{c.info(`getBook ${e}`);let{client:t,cleanup:n}=await q(e);try{let e=await w(t);return{pages:e.pages.map(ve),titles:e.titles.map(ye)}}finally{await n()}},ke=async()=>{c.info(`getMaster`);let{client:e,cleanup:t,version:n}=await J();try{return F(e,n)}finally{await t()}},Ae=/^[)\]\u00BB"”'’.,?!:\u061B\u060C\u061F\u06D4\u2026]+$/,je=/[[({«“‘]$/,Me=e=>{let t=[];for(let n of e){let e=t[t.length-1];e?.id&&Ae.test(n.text)?e.text+=n.text:t.push(n)}return t},Ne=e=>{let t=e.replace(/\r\n/g,`
|
|
43
44
|
`).replace(/\r/g,`
|
|
44
45
|
`);return/\n/.test(t)||(t=t.replace(/([.?!\u061F\u061B\u06D4\u2026]["“”'’»«)\]]?)\s+(?=[\u0600-\u06FF])/,`$1
|
|
45
46
|
`)),t.split(`
|
|
46
|
-
`).map(
|
|
47
|
+
`).map(e=>e.replace(/^\*+/,``).trim()).filter(Boolean)},Z=e=>Ne(e).map(e=>({text:e})),Q=(e,t)=>{let n=RegExp(`${t}\\s*=\\s*("([^"]*)"|'([^']*)'|([^s>]+))`,`i`),r=e.match(n);if(r)return r[2]??r[3]??r[4]},Pe=e=>{let t=[],n=/<[^>]+>/g,r=0,i;for(i=n.exec(e);i;){i.index>r&&t.push({type:`text`,value:e.slice(r,i.index)});let a=i[0],o=/^<\//.test(a),s=a.match(/^<\/?\s*([a-zA-Z0-9:-]+)/),c=s?s[1].toLowerCase():``;if(o)t.push({name:c,type:`end`});else{let e={};e.id=Q(a,`id`),e[`data-type`]=Q(a,`data-type`),t.push({attributes:e,name:c,type:`start`})}r=n.lastIndex,i=n.exec(e)}return r<e.length&&t.push({type:`text`,value:e.slice(r)}),t},$=(e,t)=>{let n=e[e.length-1];return!t||!n||!n.id||!je.test(n.text)||/\n/.test(t)?!1:(n.text+=t.replace(/^\s+/,``),!0)},Fe=e=>{if(!/<span[^>]*>/i.test(e))return Z(e);let t=Pe(`<root>${e}</root>`),n=[],r=0,i=null,a=e=>{if(!e)return;if(r>0&&i){let t=r===1?e.replace(/^\s+/,``):e;i.text+=t;return}if($(n,e))return;let t=e.trim();t&&n.push(...Z(t))};for(let e of t)e.type===`text`?a(e.value):e.type===`start`&&e.name===`span`?e.attributes[`data-type`]===`title`&&(r===0&&(i={id:e.attributes.id?.replace(/^toc-/,``)??``,text:``},n.push(i)),r+=1):e.type===`end`&&e.name===`span`&&r>0&&(--r,r===0&&(i=null));return Me(n.map(e=>e.id?e:{...e,text:e.text.trim()}).map(e=>e.id?e:{...e,text:e.text})).filter(e=>e.text.length>0)},Ie=Object.entries(L).map(([e,t])=>({regex:new RegExp(e,`g`),replacement:t})),Le=e=>{if(e===L)return Ie;let t=[];for(let n in e)t.push({regex:new RegExp(n,`g`),replacement:e[n]});return t},Re=(e,t=L)=>{let n=Le(t),r=e;for(let e=0;e<n.length;e++){let{regex:t,replacement:i}=n[e];r=r.replace(t,i)}return r},ze=(e,t=`_________`)=>{let n=``,r=e.lastIndexOf(t);return r>=0&&(n=e.slice(r+t.length),e=e.slice(0,r)),[e,n]},Be=e=>e.replace(/\s?⦗[\u0660-\u0669]+⦘\s?/,` `),Ve=e=>(e=e.replace(/<a[^>]*>(.*?)<\/a>/g,`$1`),e=e.replace(/<hadeeth[^>]*>|<\/hadeeth>|<hadeeth-\d+>/g,``),e);export{te as configure,re as createNodeConfig,Te as downloadBook,De as downloadMasterDatabase,Oe as getBook,Y as getBookMetadata,Ee as getCoverUrl,ke as getMaster,X as getMasterMetadata,Fe as parseContentRobust,Be as removeArabicNumericPageMarkers,Ve as removeTagsExceptSpan,ne as resetConfig,Re as sanitizePageContent,ze as splitPageBodyFromFooter};
|
|
47
48
|
//# sourceMappingURL=index.js.map
|