shamela 1.3.2 → 1.3.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +204 -55
- package/dist/content-B60R0uYQ.js +8 -0
- package/dist/content-B60R0uYQ.js.map +1 -0
- package/dist/content-CwjMtCQl.d.ts +54 -0
- package/dist/content.d.ts +2 -0
- package/dist/content.js +1 -0
- package/dist/index.d.ts +4 -241
- package/dist/index.js +7 -11
- package/dist/index.js.map +1 -1
- package/dist/types-C693UiUs.d.ts +226 -0
- package/dist/types.d.ts +2 -0
- package/dist/types.js +0 -0
- package/package.json +24 -14
package/dist/index.d.ts
CHANGED
|
@@ -1,228 +1,8 @@
|
|
|
1
|
-
|
|
1
|
+
import { a as removeTagsExceptSpan, i as removeArabicNumericPageMarkers, n as normalizeHtml, o as sanitizePageContent, r as parseContentRobust, s as splitPageBodyFromFooter, t as Line } from "./content-CwjMtCQl.js";
|
|
2
|
+
import { a as DownloadBookOptions, c as GetBookMetadataResponsePayload, d as OutputOptions, f as Page, h as Title, i as Category, l as GetMasterMetadataResponsePayload, m as ShamelaConfigKey, n as Book, o as DownloadMasterOptions, p as ShamelaConfig, r as BookData, s as GetBookMetadataOptions, t as Author, u as MasterData } from "./types-C693UiUs.js";
|
|
2
3
|
|
|
3
|
-
/**
|
|
4
|
-
* A record that can be deleted by patches.
|
|
5
|
-
*/
|
|
6
|
-
type Deletable = {
|
|
7
|
-
/** Indicates if it was deleted in the patch if it is set to '1 */
|
|
8
|
-
is_deleted?: string;
|
|
9
|
-
};
|
|
10
|
-
type Unique = {
|
|
11
|
-
/** Unique identifier */
|
|
12
|
-
id: number;
|
|
13
|
-
};
|
|
14
|
-
/**
|
|
15
|
-
* Database row structure for the author table.
|
|
16
|
-
*/
|
|
17
|
-
type AuthorRow = Deletable & Unique & {
|
|
18
|
-
/** Author biography */
|
|
19
|
-
biography: string;
|
|
20
|
-
/** Death year */
|
|
21
|
-
death_number: string;
|
|
22
|
-
/** The death year as a text */
|
|
23
|
-
death_text: string;
|
|
24
|
-
/** Author name */
|
|
25
|
-
name: string;
|
|
26
|
-
};
|
|
27
|
-
/**
|
|
28
|
-
* Database row structure for the book table.
|
|
29
|
-
*/
|
|
30
|
-
type BookRow = Deletable & Unique & {
|
|
31
|
-
/** Serialized author ID(s) "2747, 3147" or "513" */
|
|
32
|
-
author: string;
|
|
33
|
-
/** Bibliography information */
|
|
34
|
-
bibliography: string;
|
|
35
|
-
/** Category ID */
|
|
36
|
-
category: string;
|
|
37
|
-
/** Publication date (or 99999 for unavailable) */
|
|
38
|
-
date: string;
|
|
39
|
-
/** Hint or description */
|
|
40
|
-
hint: string;
|
|
41
|
-
/** Major version */
|
|
42
|
-
major_release: string;
|
|
43
|
-
/** Serialized metadata */
|
|
44
|
-
metadata: string;
|
|
45
|
-
/** Minor version */
|
|
46
|
-
minor_release: string;
|
|
47
|
-
/** Book name */
|
|
48
|
-
name: string;
|
|
49
|
-
/** Serialized PDF links */
|
|
50
|
-
pdf_links: string;
|
|
51
|
-
/** Printed flag */
|
|
52
|
-
printed: string;
|
|
53
|
-
/** Book type */
|
|
54
|
-
type: string;
|
|
55
|
-
};
|
|
56
|
-
/**
|
|
57
|
-
* Database row structure for the category table.
|
|
58
|
-
*/
|
|
59
|
-
type CategoryRow = Deletable & Unique & {
|
|
60
|
-
/** Category name */
|
|
61
|
-
name: string;
|
|
62
|
-
/** Category order in the list to show. */
|
|
63
|
-
order: string;
|
|
64
|
-
};
|
|
65
|
-
/**
|
|
66
|
-
* Database row structure for the page table.
|
|
67
|
-
*/
|
|
68
|
-
type PageRow = Deletable & Unique & {
|
|
69
|
-
/** Page content */
|
|
70
|
-
content: string;
|
|
71
|
-
/** Page number */
|
|
72
|
-
number: string | null;
|
|
73
|
-
/** Page reference */
|
|
74
|
-
page: string | null;
|
|
75
|
-
/** Part number */
|
|
76
|
-
part: string | null;
|
|
77
|
-
/** Additional metadata */
|
|
78
|
-
services: string | null;
|
|
79
|
-
};
|
|
80
|
-
/**
|
|
81
|
-
* Database row structure for the title table.
|
|
82
|
-
*/
|
|
83
|
-
type TitleRow = Deletable & Unique & {
|
|
84
|
-
/** Title content */
|
|
85
|
-
content: string;
|
|
86
|
-
/** Page number */
|
|
87
|
-
page: string;
|
|
88
|
-
/** Parent title ID */
|
|
89
|
-
parent: string | null;
|
|
90
|
-
};
|
|
91
|
-
//#endregion
|
|
92
|
-
//#region src/types.d.ts
|
|
93
|
-
/**
|
|
94
|
-
* Represents an author entity.
|
|
95
|
-
*/
|
|
96
|
-
type Author = AuthorRow;
|
|
97
|
-
/**
|
|
98
|
-
* Represents a book entity.
|
|
99
|
-
*/
|
|
100
|
-
type Book = BookRow;
|
|
101
|
-
/**
|
|
102
|
-
* A category for a book.
|
|
103
|
-
*/
|
|
104
|
-
type Category = CategoryRow;
|
|
105
|
-
/**
|
|
106
|
-
* A page in a book.
|
|
107
|
-
*/
|
|
108
|
-
type Page = Pick<PageRow, 'id' | 'content'> & {
|
|
109
|
-
page?: number;
|
|
110
|
-
part?: string;
|
|
111
|
-
number?: string;
|
|
112
|
-
};
|
|
113
|
-
/**
|
|
114
|
-
* A title heading in a book.
|
|
115
|
-
*/
|
|
116
|
-
type Title = Pick<TitleRow, 'id' | 'content'> & {
|
|
117
|
-
page: number;
|
|
118
|
-
parent?: number;
|
|
119
|
-
};
|
|
120
|
-
/**
|
|
121
|
-
* Represents book content data.
|
|
122
|
-
*/
|
|
123
|
-
type BookData = {
|
|
124
|
-
/** Array of pages in the book */
|
|
125
|
-
pages: Page[];
|
|
126
|
-
/** Array of titles/chapters */
|
|
127
|
-
titles: Title[];
|
|
128
|
-
};
|
|
129
|
-
/**
|
|
130
|
-
* Master data structure containing all core entities.
|
|
131
|
-
*/
|
|
132
|
-
type MasterData = {
|
|
133
|
-
/** Array of all authors */
|
|
134
|
-
authors: Author[];
|
|
135
|
-
/** Array of all books */
|
|
136
|
-
books: Book[];
|
|
137
|
-
/** Array of all categories */
|
|
138
|
-
categories: Category[];
|
|
139
|
-
/** Version number for the downloaded master database */
|
|
140
|
-
version: number;
|
|
141
|
-
};
|
|
142
|
-
/**
|
|
143
|
-
* Options for downloading a book.
|
|
144
|
-
*/
|
|
145
|
-
type DownloadBookOptions = {
|
|
146
|
-
/** Optional book metadata */
|
|
147
|
-
bookMetadata?: GetBookMetadataResponsePayload;
|
|
148
|
-
/** Output file configuration */
|
|
149
|
-
outputFile: OutputOptions;
|
|
150
|
-
};
|
|
151
|
-
/**
|
|
152
|
-
* Options for downloading master data.
|
|
153
|
-
*/
|
|
154
|
-
type DownloadMasterOptions = {
|
|
155
|
-
/** Optional master metadata */
|
|
156
|
-
masterMetadata?: GetMasterMetadataResponsePayload;
|
|
157
|
-
/** Output file configuration */
|
|
158
|
-
outputFile: OutputOptions;
|
|
159
|
-
};
|
|
160
|
-
/**
|
|
161
|
-
* Options for getting book metadata.
|
|
162
|
-
*/
|
|
163
|
-
type GetBookMetadataOptions = {
|
|
164
|
-
/** Major version number */
|
|
165
|
-
majorVersion: number;
|
|
166
|
-
/** Minor version number */
|
|
167
|
-
minorVersion: number;
|
|
168
|
-
};
|
|
169
|
-
/**
|
|
170
|
-
* Response payload for book metadata requests.
|
|
171
|
-
*/
|
|
172
|
-
type GetBookMetadataResponsePayload = {
|
|
173
|
-
/** Major release version */
|
|
174
|
-
majorRelease: number;
|
|
175
|
-
/** URL for major release download */
|
|
176
|
-
majorReleaseUrl: string;
|
|
177
|
-
/** Optional minor release version */
|
|
178
|
-
minorRelease?: number;
|
|
179
|
-
/** Optional URL for minor release download */
|
|
180
|
-
minorReleaseUrl?: string;
|
|
181
|
-
};
|
|
182
|
-
/**
|
|
183
|
-
* Response payload for master metadata requests.
|
|
184
|
-
*/
|
|
185
|
-
type GetMasterMetadataResponsePayload = {
|
|
186
|
-
/** Download URL */
|
|
187
|
-
url: string;
|
|
188
|
-
/** Version number */
|
|
189
|
-
version: number;
|
|
190
|
-
};
|
|
191
|
-
type NodeJSOutput = {
|
|
192
|
-
/** Output file path (Node.js only) */
|
|
193
|
-
path: string;
|
|
194
|
-
writer?: never;
|
|
195
|
-
};
|
|
196
|
-
type CustomOutput = {
|
|
197
|
-
/** Custom writer used when path is not provided */
|
|
198
|
-
writer: (payload: string | Uint8Array) => Promise<void> | void;
|
|
199
|
-
path?: undefined;
|
|
200
|
-
};
|
|
201
|
-
/**
|
|
202
|
-
* Output file options.
|
|
203
|
-
*/
|
|
204
|
-
type OutputOptions = NodeJSOutput | CustomOutput;
|
|
205
|
-
/**
|
|
206
|
-
* Runtime configuration for the library.
|
|
207
|
-
*/
|
|
208
|
-
type ShamelaConfig = {
|
|
209
|
-
/** API key used to authenticate against Shamela services */
|
|
210
|
-
apiKey?: string;
|
|
211
|
-
/** Endpoint used for book metadata */
|
|
212
|
-
booksEndpoint?: string;
|
|
213
|
-
/** Endpoint used for master metadata */
|
|
214
|
-
masterPatchEndpoint?: string;
|
|
215
|
-
/** Optional override for the sql.js wasm asset location */
|
|
216
|
-
sqlJsWasmUrl?: string;
|
|
217
|
-
/** Optional custom fetch implementation for environments without a global fetch */
|
|
218
|
-
fetchImplementation?: typeof fetch;
|
|
219
|
-
};
|
|
220
|
-
/**
|
|
221
|
-
* Valid configuration keys.
|
|
222
|
-
*/
|
|
223
|
-
type ShamelaConfigKey = keyof ShamelaConfig;
|
|
224
|
-
//#endregion
|
|
225
4
|
//#region src/api.d.ts
|
|
5
|
+
|
|
226
6
|
/**
|
|
227
7
|
* Retrieves metadata for a specific book from the Shamela API.
|
|
228
8
|
*
|
|
@@ -399,22 +179,5 @@ declare const configure: (config: ConfigureOptions) => void;
|
|
|
399
179
|
*/
|
|
400
180
|
declare const resetConfig: () => void;
|
|
401
181
|
//#endregion
|
|
402
|
-
|
|
403
|
-
type Line = {
|
|
404
|
-
id?: string;
|
|
405
|
-
text: string;
|
|
406
|
-
};
|
|
407
|
-
declare const parseContentRobust: (content: string) => Line[];
|
|
408
|
-
/**
|
|
409
|
-
* Sanitizes page content by applying regex replacement rules
|
|
410
|
-
* @param text - The text to sanitize
|
|
411
|
-
* @param rules - Optional custom rules (defaults to DEFAULT_SANITIZATION_RULES)
|
|
412
|
-
* @returns The sanitized text
|
|
413
|
-
*/
|
|
414
|
-
declare const sanitizePageContent: (text: string, rules?: Record<string, string>) => string;
|
|
415
|
-
declare const splitPageBodyFromFooter: (content: string, footnoteMarker?: string) => readonly [string, string];
|
|
416
|
-
declare const removeArabicNumericPageMarkers: (text: string) => string;
|
|
417
|
-
declare const removeTagsExceptSpan: (content: string) => string;
|
|
418
|
-
//#endregion
|
|
419
|
-
export { Author, Book, BookData, Category, type ConfigureOptions, DownloadBookOptions, DownloadMasterOptions, GetBookMetadataOptions, GetBookMetadataResponsePayload, GetMasterMetadataResponsePayload, Line, type Logger, MasterData, OutputOptions, Page, ShamelaConfig, ShamelaConfigKey, Title, configure, downloadBook, downloadMasterDatabase, getBook, getBookMetadata, getCoverUrl, getMaster, getMasterMetadata, parseContentRobust, removeArabicNumericPageMarkers, removeTagsExceptSpan, resetConfig, sanitizePageContent, splitPageBodyFromFooter };
|
|
182
|
+
export { Author, Book, BookData, Category, type ConfigureOptions, DownloadBookOptions, DownloadMasterOptions, GetBookMetadataOptions, GetBookMetadataResponsePayload, GetMasterMetadataResponsePayload, Line, type Logger, MasterData, OutputOptions, Page, ShamelaConfig, ShamelaConfigKey, Title, configure, downloadBook, downloadMasterDatabase, getBook, getBookMetadata, getCoverUrl, getMaster, getMasterMetadata, normalizeHtml, parseContentRobust, removeArabicNumericPageMarkers, removeTagsExceptSpan, resetConfig, sanitizePageContent, splitPageBodyFromFooter };
|
|
420
183
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/index.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import e from"sql.js";import{unzipSync as
|
|
1
|
+
import{a as e,i as t,n,o as r,r as i,s as a,t as o}from"./content-B60R0uYQ.js";import"./types.js";import s from"sql.js";import{unzipSync as ee}from"fflate";var c=(e=>typeof require<`u`?require:typeof Proxy<`u`?new Proxy(e,{get:(e,t)=>(typeof require<`u`?require:e)[t]}):e)(function(e){if(typeof require<`u`)return require.apply(this,arguments);throw Error('Calling `require` for "'+e+"\" in an environment that doesn't expose the `require` function.")});const l=Object.freeze({debug:()=>{},error:()=>{},info:()=>{},warn:()=>{}});let u=l;const te=e=>{if(!e){u=l;return}let t=[`debug`,`error`,`info`,`warn`].find(t=>typeof e[t]!=`function`);if(t)throw Error(`Logger must implement debug, error, info, and warn methods. Missing: ${String(t)}`);u=e},ne=()=>u,re=()=>{u=l};var d=new Proxy({},{get:(e,t)=>{let n=ne(),r=n[t];return typeof r==`function`?(...e)=>r.apply(n,e):r}});let f={};const p={apiKey:`SHAMELA_API_KEY`,booksEndpoint:`SHAMELA_API_BOOKS_ENDPOINT`,masterPatchEndpoint:`SHAMELA_API_MASTER_PATCH_ENDPOINT`,sqlJsWasmUrl:`SHAMELA_SQLJS_WASM_URL`},m=typeof process<`u`&&!!process?.env,h=e=>{let t=f[e];if(t!==void 0)return t;let n=p[e];if(m)return process.env[n]},ie=e=>{let{logger:t,...n}=e;`logger`in e&&te(t),f={...f,...n}},g=e=>e===`fetchImplementation`?f.fetchImplementation:h(e),_=()=>({apiKey:h(`apiKey`),booksEndpoint:h(`booksEndpoint`),fetchImplementation:f.fetchImplementation,masterPatchEndpoint:h(`masterPatchEndpoint`),sqlJsWasmUrl:h(`sqlJsWasmUrl`)}),v=e=>{if(e===`fetchImplementation`)throw Error(`fetchImplementation must be provided via configure().`);let t=g(e);if(!t)throw Error(`${p[e]} environment variable not set`);return t},ae=()=>{f={},re()};let y=function(e){return e.Authors=`author`,e.Books=`book`,e.Categories=`category`,e.Page=`page`,e.Title=`title`,e}({});const b=(e,t)=>e.query(`PRAGMA table_info(${t})`).all(),x=(e,t)=>!!e.query(`SELECT name FROM sqlite_master WHERE type='table' AND name = ?1`).get(t),S=(e,t)=>x(e,t)?e.query(`SELECT * FROM ${t}`).all():[],C=e=>String(e.is_deleted)===`1`,w=(e,t,n)=>{let r={};for(let i of n){if(i===`id`){r.id=(t??e)?.id??null;continue}if(t&&i in t){let e=t[i];if(e!==`#`&&e!=null){r[i]=e;continue}}if(e&&i in e){r[i]=e[i];continue}r[i]=null}return r},oe=(e,t,n)=>{let r=new Set,i=new Map;for(let t of e)r.add(String(t.id));for(let e of t)i.set(String(e.id),e);let a=[];for(let t of e){let e=i.get(String(t.id));e&&C(e)||a.push(w(t,e,n))}for(let e of t){let t=String(e.id);r.has(t)||C(e)||a.push(w(void 0,e,n))}return a},se=(e,t,n,r)=>{if(r.length===0)return;let i=n.map(()=>`?`).join(`,`),a=e.prepare(`INSERT INTO ${t} (${n.join(`,`)}) VALUES (${i})`);r.forEach(e=>{let t=n.map(t=>t in e?e[t]:null);a.run(...t)}),a.finalize()},ce=(e,t,n)=>{let r=t.query(`SELECT sql FROM sqlite_master WHERE type='table' AND name = ?1`).get(n);return r?.sql?(e.run(`DROP TABLE IF EXISTS ${n}`),e.run(r.sql),!0):(d.warn(`${n} table definition missing in source database`),!1)},T=(e,t,n,r)=>{if(!x(t,r)){d.warn(`${r} table missing in source database`);return}if(!ce(e,t,r))return;let i=b(t,r),a=n&&x(n,r)?b(n,r):[],o=i.map(e=>e.name);for(let t of a)if(!o.includes(t.name)){let n=t.type&&t.type.length>0?t.type:`TEXT`;e.run(`ALTER TABLE ${r} ADD COLUMN ${t.name} ${n}`),o.push(t.name)}se(e,r,o,oe(S(t,r),n?S(n,r):[],o))},le=(e,t,n)=>{e.transaction(()=>{T(e,t,n,y.Page),T(e,t,n,y.Title)})()},E=(e,t)=>{e.transaction(()=>{T(e,t,null,y.Page),T(e,t,null,y.Title)})()},D=e=>{e.run(`CREATE TABLE ${y.Page} (
|
|
2
2
|
id INTEGER,
|
|
3
3
|
content TEXT,
|
|
4
4
|
part TEXT,
|
|
@@ -6,21 +6,21 @@ import e from"sql.js";import{unzipSync as t}from"fflate";var n=(e=>typeof requir
|
|
|
6
6
|
number TEXT,
|
|
7
7
|
services TEXT,
|
|
8
8
|
is_deleted TEXT
|
|
9
|
-
)`),e.run(`CREATE TABLE ${
|
|
9
|
+
)`),e.run(`CREATE TABLE ${y.Title} (
|
|
10
10
|
id INTEGER,
|
|
11
11
|
content TEXT,
|
|
12
12
|
page INTEGER,
|
|
13
13
|
parent INTEGER,
|
|
14
14
|
is_deleted TEXT
|
|
15
|
-
)`)},
|
|
16
|
-
`);throw Error(e)}}else
|
|
15
|
+
)`)},ue=e=>e.query(`SELECT * FROM ${y.Page}`).all(),de=e=>e.query(`SELECT * FROM ${y.Title}`).all(),O=e=>({pages:ue(e),titles:de(e)}),k=e=>{try{return c(`node:fs`).existsSync(e)}catch{return!1}},fe=()=>{if(c!==void 0&&c.resolve!==void 0)try{let e=c.resolve(`sql.js`),t=c(`node:path`),n=t.dirname(e),r=t.join(n,`dist`,`sql-wasm.wasm`);if(k(r))return r}catch{}if(typeof process<`u`&&process.cwd)try{let e=c(`node:path`),t=process.cwd(),n=[e.join(t,`node_modules`,`sql.js`,`dist`,`sql-wasm.wasm`),e.join(t,`..`,`node_modules`,`sql.js`,`dist`,`sql-wasm.wasm`),e.join(t,`../..`,`node_modules`,`sql.js`,`dist`,`sql-wasm.wasm`),e.join(t,`.next`,`server`,`node_modules`,`sql.js`,`dist`,`sql-wasm.wasm`)];for(let e of n)if(k(e))return e}catch{}if(c!==void 0&&c.resolve!==void 0&&c.resolve.paths)try{let e=c(`node:path`),t=c.resolve.paths(`sql.js`)||[];for(let n of t){let t=e.join(n,`sql.js`,`dist`,`sql-wasm.wasm`);if(k(t))return t}}catch{}try{if(import.meta.url){let e=new URL(`../../node_modules/sql.js/dist/sql-wasm.wasm`,import.meta.url),t=decodeURIComponent(e.pathname),n=process.platform===`win32`&&t.startsWith(`/`)?t.slice(1):t;if(k(n))return n}}catch{}return null};var pe=class{constructor(e){this.statement=e}run=(...e)=>{e.length>0&&this.statement.bind(e),this.statement.step(),this.statement.reset()};finalize=()=>{this.statement.free()}},A=class{constructor(e){this.db=e}run=(e,t=[])=>{this.db.run(e,t)};prepare=e=>new pe(this.db.prepare(e));query=e=>({all:(...t)=>this.all(e,t),get:(...t)=>this.get(e,t)});transaction=e=>()=>{this.db.run(`BEGIN TRANSACTION`);try{e(),this.db.run(`COMMIT`)}catch(e){throw this.db.run(`ROLLBACK`),e}};close=()=>{this.db.close()};export=()=>this.db.export();all=(e,t)=>{let n=this.db.prepare(e);try{t.length>0&&n.bind(t);let e=[];for(;n.step();)e.push(n.getAsObject());return e}finally{n.free()}};get=(e,t)=>this.all(e,t)[0]};let j=null,M=null;const me=typeof process<`u`&&!!process?.versions?.node,he=()=>{if(!M){let e=g(`sqlJsWasmUrl`);if(e)M=e;else if(me){let e=fe();if(e)M=e;else{let e=[`Unable to automatically locate sql-wasm.wasm file.`,`This can happen in bundled environments (Next.js, webpack, etc.).`,``,`Quick fix - add this to your code before using shamela:`,``,` import { configure, createNodeConfig } from "shamela";`,` configure(createNodeConfig({`,` apiKey: process.env.SHAMELA_API_KEY,`,` booksEndpoint: process.env.SHAMELA_BOOKS_ENDPOINT,`,` masterPatchEndpoint: process.env.SHAMELA_MASTER_ENDPOINT,`,` }));`,``,`Or manually specify the path:`,``,` import { configure } from "shamela";`,` import { join } from "node:path";`,` configure({`,` sqlJsWasmUrl: join(process.cwd(), "node_modules", "sql.js", "dist", "sql-wasm.wasm")`,` });`].join(`
|
|
16
|
+
`);throw Error(e)}}else M=`https://cdn.jsdelivr.net/npm/sql.js@1.13.0/dist/sql-wasm.wasm`}return M},N=()=>(j||=s({locateFile:()=>he()}),j),P=async()=>new A(new(await(N())).Database),F=async e=>new A(new(await(N())).Database(e)),I=(e,t,n)=>{let r=t.query(`SELECT sql FROM sqlite_master WHERE type='table' AND name = ?1`).get(n);if(!r?.sql)throw Error(`Missing table definition for ${n} in source database`);e.run(`DROP TABLE IF EXISTS ${n}`),e.run(r.sql)},L=async(e,t)=>{let n={author:y.Authors,book:y.Books,category:y.Categories},r={};for(let e of t){let t=n[(e.name.split(`/`).pop()?.split(`\\`).pop()??e.name).replace(/\.(sqlite|db)$/i,``).toLowerCase()];t&&(r[t]=await F(e.data))}try{let t=Object.entries(r);e.transaction(()=>{for(let[n,r]of t){I(e,r,n);let t=r.query(`PRAGMA table_info(${n})`).all().map(e=>e.name);if(t.length===0)continue;let i=r.query(`SELECT * FROM ${n}`).all();if(i.length===0)continue;let a=t.map(()=>`?`).join(`,`),o=t.map(e=>e===`order`?`"order"`:e),s=e.prepare(`INSERT INTO ${n} (${o.join(`,`)}) VALUES (${a})`);try{for(let e of i){let n=t.map(t=>t in e?e[t]:null);s.run(...n)}}finally{s.finalize()}}})()}finally{Object.values(r).forEach(e=>e?.close())}},R=(e,t,n)=>{e.run(`DROP VIEW IF EXISTS ${t}`),e.run(`CREATE VIEW ${t} AS SELECT * FROM ${n}`)},ge=e=>{e.run(`CREATE TABLE ${y.Authors} (
|
|
17
17
|
id INTEGER,
|
|
18
18
|
is_deleted TEXT,
|
|
19
19
|
name TEXT,
|
|
20
20
|
biography TEXT,
|
|
21
21
|
death_text TEXT,
|
|
22
22
|
death_number TEXT
|
|
23
|
-
)`),e.run(`CREATE TABLE ${
|
|
23
|
+
)`),e.run(`CREATE TABLE ${y.Books} (
|
|
24
24
|
id INTEGER,
|
|
25
25
|
name TEXT,
|
|
26
26
|
is_deleted TEXT,
|
|
@@ -35,14 +35,10 @@ import e from"sql.js";import{unzipSync as t}from"fflate";var n=(e=>typeof requir
|
|
|
35
35
|
hint TEXT,
|
|
36
36
|
pdf_links TEXT,
|
|
37
37
|
metadata TEXT
|
|
38
|
-
)`),e.run(`CREATE TABLE ${
|
|
38
|
+
)`),e.run(`CREATE TABLE ${y.Categories} (
|
|
39
39
|
id INTEGER,
|
|
40
40
|
is_deleted TEXT,
|
|
41
41
|
"order" TEXT,
|
|
42
42
|
name TEXT
|
|
43
|
-
)`),
|
|
44
|
-
`).replace(/\r/g,`
|
|
45
|
-
`);return/\n/.test(t)||(t=t.replace(/([.?!\u061F\u061B\u06D4\u2026]["“”'’»«)\]]?)\s+(?=[\u0600-\u06FF])/,`$1
|
|
46
|
-
`)),t.split(`
|
|
47
|
-
`).map(e=>e.replace(/^\*+/,``).trim()).filter(Boolean)},Q=e=>je(e).map(e=>({text:e})),$=(e,t)=>{let n=RegExp(`${t}\\s*=\\s*("([^"]*)"|'([^']*)'|([^s>]+))`,`i`),r=e.match(n);if(r)return r[2]??r[3]??r[4]},Me=e=>{let t=[],n=/<[^>]+>/g,r=0,i;for(i=n.exec(e);i;){i.index>r&&t.push({type:`text`,value:e.slice(r,i.index)});let a=i[0],o=/^<\//.test(a),s=a.match(/^<\/?\s*([a-zA-Z0-9:-]+)/),c=s?s[1].toLowerCase():``;if(o)t.push({name:c,type:`end`});else{let e={};e.id=$(a,`id`),e[`data-type`]=$(a,`data-type`),t.push({attributes:e,name:c,type:`start`})}r=n.lastIndex,i=n.exec(e)}return r<e.length&&t.push({type:`text`,value:e.slice(r)}),t},Ne=(e,t)=>{let n=e[e.length-1];return!t||!n||!n.id||!Z.test(n.text)||/\n/.test(t)?!1:(n.text+=t.replace(/^\s+/,``),!0)},Pe=e=>{if(!/<span[^>]*>/i.test(e))return Q(e);let t=Me(`<root>${e}</root>`),n=[],r=0,i=null,a=e=>{if(!e)return;if(r>0&&i){let t=r===1?e.replace(/^\s+/,``):e;i.text+=t;return}if(Ne(n,e))return;let t=e.trim();t&&n.push(...Q(t))};for(let e of t)e.type===`text`?a(e.value):e.type===`start`&&e.name===`span`?e.attributes[`data-type`]===`title`&&(r===0&&(i={id:e.attributes.id?.replace(/^toc-/,``)??``,text:``},n.push(i)),r+=1):e.type===`end`&&e.name===`span`&&r>0&&(--r,r===0&&(i=null));return Ae(n.map(e=>e.id?e:{...e,text:e.text.trim()}).map(e=>e.id?e:{...e,text:e.text})).filter(e=>e.text.length>0)},Fe=Object.entries(L).map(([e,t])=>({regex:new RegExp(e,`g`),replacement:t})),Ie=e=>{if(e===L)return Fe;let t=[];for(let n in e)t.push({regex:new RegExp(n,`g`),replacement:e[n]});return t},Le=(e,t=L)=>{let n=Ie(t),r=e;for(let e=0;e<n.length;e++){let{regex:t,replacement:i}=n[e];r=r.replace(t,i)}return r},Re=(e,t=`_________`)=>{let n=``,r=e.lastIndexOf(t);return r>=0&&(n=e.slice(r+t.length),e=e.slice(0,r)),[e,n]},ze=e=>e.replace(/\s?⦗[\u0660-\u0669]+⦘\s?/,` `),Be=e=>(e=e.replace(/<a[^>]*>(.*?)<\/a>/g,`$1`),e=e.replace(/<hadeeth[^>]*>|<\/hadeeth>|<hadeeth-\d+>/g,``),e);export{te as configure,we as downloadBook,Ee as downloadMasterDatabase,De as getBook,Y as getBookMetadata,Te as getCoverUrl,Oe as getMaster,X as getMasterMetadata,Pe as parseContentRobust,ze as removeArabicNumericPageMarkers,Be as removeTagsExceptSpan,ne as resetConfig,Le as sanitizePageContent,Re as splitPageBodyFromFooter};
|
|
43
|
+
)`),R(e,`authors`,y.Authors),R(e,`books`,y.Books),R(e,`categories`,y.Categories)},_e=e=>e.query(`SELECT * FROM ${y.Authors}`).all(),ve=e=>e.query(`SELECT * FROM ${y.Books}`).all(),ye=e=>e.query(`SELECT * FROM ${y.Categories}`).all(),z=(e,t)=>({authors:_e(e),books:ve(e),categories:ye(e),version:t}),B=(e,t=[`api_key`,`token`,`password`,`secret`,`auth`])=>{let n=typeof e==`string`?new URL(e):new URL(e.toString());return t.forEach(e=>{let t=n.searchParams.get(e);if(t&&t.length>6){let r=`${t.slice(0,3)}***${t.slice(-3)}`;n.searchParams.set(e,r)}else t&&n.searchParams.set(e,`***`)}),n.toString()},be=e=>({content:e.content,id:e.id,...e.number&&{number:e.number},...e.page&&{page:Number(e.page)},...e.part&&{part:e.part}}),xe=e=>{let t=Number(e.parent);return{content:e.content,id:e.id,page:Number(e.page),...t&&{parent:t}}},V=e=>{let t=new URL(e);return t.protocol=`https`,t.toString()},H=e=>/\.(sqlite|db)$/i.test(e.name),U=e=>e.find(H),W=e=>{let t=/\.([^.]+)$/.exec(e);return t?`.${t[1].toLowerCase()}`:``},G=(e,t,n=!0)=>{let r=new URL(e),i=new URLSearchParams;return Object.entries(t).forEach(([e,t])=>{i.append(e,t.toString())}),n&&i.append(`api_key`,v(`apiKey`)),r.search=i.toString(),r},K=async(e,t={})=>{let n=typeof e==`string`?e:e.toString(),r=await(t.fetchImpl??_().fetchImplementation??fetch)(n);if(!r.ok)throw Error(`Error making request: ${r.status} ${r.statusText}`);if((r.headers.get(`content-type`)??``).includes(`application/json`))return await r.json();let i=await r.arrayBuffer();return new Uint8Array(i)},Se=typeof process<`u`&&!!process?.versions?.node,Ce=async()=>{if(!Se)throw Error(`File system operations are only supported in Node.js environments`);return import(`node:fs/promises`)},we=async e=>{let[t,n]=await Promise.all([Ce(),import(`node:path`)]),r=n.dirname(e);return await t.mkdir(r,{recursive:!0}),t},q=async e=>{let t=await K(e),n=t instanceof Uint8Array?t.length:t&&typeof t.byteLength==`number`?t.byteLength:0;return d.debug(`unzipFromUrl:bytes`,n),new Promise((e,n)=>{let r=t instanceof Uint8Array?t:new Uint8Array(t);try{let t=ee(r),n=Object.entries(t).map(([e,t])=>({data:t,name:e}));d.debug(`unzipFromUrl:entries`,n.map(e=>e.name)),e(n)}catch(e){n(Error(`Error processing URL: ${e.message}`))}})},J=async(e,t)=>{if(e.writer){await e.writer(t);return}if(!e.path)throw Error(`Output options must include either a writer or a path`);let n=await we(e.path);typeof t==`string`?await n.writeFile(e.path,t,`utf-8`):await n.writeFile(e.path,t)},Te=[`author.sqlite`,`book.sqlite`,`category.sqlite`],Y=()=>{let{apiKey:e,booksEndpoint:t,masterPatchEndpoint:n}=_(),r=[[`apiKey`,e],[`booksEndpoint`,t],[`masterPatchEndpoint`,n]].filter(([,e])=>!e).map(([e])=>e);if(r.length)throw Error(`${r.join(`, `)} environment variables not set`)},Ee=e=>{let t=new Set(e.map(e=>e.match(/[^\\/]+$/)?.[0]??e).map(e=>e.toLowerCase()));return Te.every(e=>t.has(e.toLowerCase()))},X=async(e,t)=>{d.info(`Setting up book database for ${e}`);let n=t||await Q(e),r=n.minorReleaseUrl?q(n.minorReleaseUrl):Promise.resolve([]),[i,a]=await Promise.all([q(n.majorReleaseUrl),r]),o=U(i);if(!o)throw Error(`Unable to locate book database in archive`);let s=await P();try{d.info(`Creating tables`),D(s);let e=await F(o.data);try{let t=U(a);if(t){d.info(`Applying patches from ${t.name} to ${o.name}`);let n=await F(t.data);try{le(s,e,n)}finally{n.close()}}else d.info(`Copying table data from ${o.name}`),E(s,e)}finally{e.close()}return{cleanup:async()=>{s.close()},client:s}}catch(e){throw s.close(),e}},Z=async e=>{d.info(`Setting up master database`);let t=e||await $(a);d.info(`Downloading master database ${t.version} from: ${B(t.url)}`);let n=await q(V(t.url));if(d.debug?.(`sourceTables downloaded: ${n.map(e=>e.name).toString()}`),!Ee(n.map(e=>e.name)))throw d.error(`Some source tables were not found: ${n.map(e=>e.name).toString()}`),Error(`Expected tables not found!`);let r=await P();try{return d.info(`Creating master tables`),ge(r),d.info(`Copying data to master table`),await L(r,n.filter(H)),{cleanup:async()=>{r.close()},client:r,version:t.version}}catch(e){throw r.close(),e}},Q=async(e,t)=>{Y();let n=G(`${v(`booksEndpoint`)}/${e}`,{major_release:(t?.majorVersion||0).toString(),minor_release:(t?.minorVersion||0).toString()});d.info(`Fetching shamela.ws book link: ${B(n)}`);try{let e=await K(n);return{majorRelease:e.major_release,majorReleaseUrl:V(e.major_release_url),...e.minor_release_url&&{minorReleaseUrl:V(e.minor_release_url)},...e.minor_release_url&&{minorRelease:e.minor_release}}}catch(e){throw Error(`Error fetching book metadata: ${e.message}`)}},De=async(e,t)=>{if(d.info(`downloadBook ${e} ${JSON.stringify(t)}`),!t.outputFile.path)throw Error(`outputFile.path must be provided to determine output format`);let n=W(t.outputFile.path).toLowerCase(),{client:r,cleanup:i}=await X(e,t?.bookMetadata);try{if(n===`.json`){let e=await O(r);await J(t.outputFile,JSON.stringify(e,null,2))}else if(n===`.db`||n===`.sqlite`){let e=r.export();await J(t.outputFile,e)}else throw Error(`Unsupported output extension: ${n}`)}finally{await i()}return t.outputFile.path},$=async(e=0)=>{Y();let t=G(v(`masterPatchEndpoint`),{version:e.toString()});d.info(`Fetching shamela.ws master database patch link: ${B(t)}`);try{let e=await K(t);return{url:e.patch_url,version:e.version}}catch(e){throw Error(`Error fetching master patch: ${e.message}`)}},Oe=e=>{let t=v(`masterPatchEndpoint`),{origin:n}=new URL(t);return`${n}/covers/${e}.jpg`},ke=async e=>{if(d.info(`downloadMasterDatabase ${JSON.stringify(e)}`),!e.outputFile.path)throw Error(`outputFile.path must be provided to determine output format`);let t=W(e.outputFile.path),{client:n,cleanup:r,version:i}=await Z(e.masterMetadata);try{if(t===`.json`){let t=z(n,i);await J(e.outputFile,JSON.stringify(t,null,2))}else if(t===`.db`||t===`.sqlite`)await J(e.outputFile,n.export());else throw Error(`Unsupported output extension: ${t}`)}finally{await r()}return e.outputFile.path},Ae=async e=>{d.info(`getBook ${e}`);let{client:t,cleanup:n}=await X(e);try{let e=await O(t);return{pages:e.pages.map(be),titles:e.titles.map(xe)}}finally{await n()}},je=async()=>{d.info(`getMaster`);let{client:e,cleanup:t,version:n}=await Z();try{return z(e,n)}finally{await t()}};export{ie as configure,De as downloadBook,ke as downloadMasterDatabase,Ae as getBook,Q as getBookMetadata,Oe as getCoverUrl,je as getMaster,$ as getMasterMetadata,o as normalizeHtml,n as parseContentRobust,i as removeArabicNumericPageMarkers,t as removeTagsExceptSpan,ae as resetConfig,e as sanitizePageContent,r as splitPageBodyFromFooter};
|
|
48
44
|
//# sourceMappingURL=index.js.map
|